From 200ec8a2cd0d8b09f57dab5d398a1106f8b23e8b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 14 Aug 2023 14:49:48 +0200 Subject: [PATCH] Fix build by replacing msgpack --- BioD/bio/std/hts/bam/cigar.d | 2 +- BioD/bio/std/hts/bam/read.d | 8 +- BioD/bio/std/hts/bam/tagvalue.d | 84 +- BioD/bio/std/hts/sam/header.d | 2 +- BioD/bio/std/hts/thirdparty/msgpack.d | 5171 ----------------- BioD/contrib/msgpack-d | 1 + Makefile | 2 +- Makefile.guix | 6 +- guix.scm | 101 + meson.build | 1 - sambamba/utils/view/alignmentrangeprocessor.d | 2 +- sambamba/utils/view/headerserializer.d | 4 +- 12 files changed, 157 insertions(+), 5227 deletions(-) delete mode 100644 BioD/bio/std/hts/thirdparty/msgpack.d create mode 160000 BioD/contrib/msgpack-d create mode 100644 guix.scm diff --git a/BioD/bio/std/hts/bam/cigar.d b/BioD/bio/std/hts/bam/cigar.d index 78dbfd4b..e9a7d8be 100644 --- a/BioD/bio/std/hts/bam/cigar.d +++ b/BioD/bio/std/hts/bam/cigar.d @@ -50,7 +50,7 @@ import bio.std.hts.utils.array; import bio.std.hts.utils.value; import bio.core.utils.switchendianness; -import bio.std.hts.thirdparty.msgpack : Packer, unpack; +import msgpack : Packer, unpack; /** Represents single CIGAR operation diff --git a/BioD/bio/std/hts/bam/read.d b/BioD/bio/std/hts/bam/read.d index ec7cc12d..9ccdd0b4 100644 --- a/BioD/bio/std/hts/bam/read.d +++ b/BioD/bio/std/hts/bam/read.d @@ -62,7 +62,7 @@ import bio.std.hts.utils.array; import bio.std.hts.utils.value; import bio.core.utils.switchendianness; -import bio.std.hts.thirdparty.msgpack : Packer, unpack; +import msgpack : Packer, unpack, packer; import std.algorithm; import std.range; @@ -631,7 +631,7 @@ struct BamRead { /// Packs message in the following format: /// $(BR) - /// MsgPack array with elements + /// Msack array with elements /// $(OL /// $(LI name - string) /// $(LI flag - ushort) @@ -1353,8 +1353,8 @@ unittest { { import std.typecons; - static import bio.std.hts.thirdparty.msgpack; - auto packer = bio.std.hts.thirdparty.msgpack.packer(Appender!(ubyte[])()); + // static import msgpack.packer; + auto packer = packer(Appender!(ubyte[])()); read.toMsgpack(packer); auto data = packer.stream.data; auto rec = unpack(data).via.array; diff --git a/BioD/bio/std/hts/bam/tagvalue.d b/BioD/bio/std/hts/bam/tagvalue.d index e8a18c13..b0b7da3d 100644 --- a/BioD/bio/std/hts/bam/tagvalue.d +++ b/BioD/bio/std/hts/bam/tagvalue.d @@ -8,10 +8,10 @@ the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -55,16 +55,16 @@ import std.format; import std.array; import bio.core.utils.format; -import bio.std.hts.thirdparty.msgpack; +import msgpack; struct CharToType(char c, T) { /** symbol */ enum ch = c; /** type which corresponds to the symbol - according to SAM/BAM specification + according to SAM/BAM specification */ - alias T ValueType; + alias T ValueType; } /** @@ -76,22 +76,22 @@ class UnknownTagTypeException : Exception { alias TypeTuple!(CharToType!('A', char), - CharToType!('c', byte), + CharToType!('c', byte), CharToType!('C', ubyte), - CharToType!('s', short), + CharToType!('s', short), CharToType!('S', ushort), - CharToType!('i', int), + CharToType!('i', int), CharToType!('I', uint), CharToType!('f', float)) PrimitiveTagValueTypes; -alias TypeTuple!(CharToType!('Z', string), +alias TypeTuple!(CharToType!('Z', string), CharToType!('H', string)) StringTagValueTypes; -alias TypeTuple!(CharToType!('c', byte), +alias TypeTuple!(CharToType!('c', byte), CharToType!('C', ubyte), - CharToType!('s', short), + CharToType!('s', short), CharToType!('S', ushort), - CharToType!('i', int), + CharToType!('i', int), CharToType!('I', uint), CharToType!('f', float)) ArrayElementTagValueTypes; @@ -112,14 +112,14 @@ uint charToSizeof(char c) { } return "switch (c) { " ~ cases.idup ~ " default: " ~ - " throw new UnknownTagTypeException(to!string(c));"~ + " throw new UnknownTagTypeException(to!string(c));"~ "}"; } mixin(charToSizeofHelper()); } /* - Pair of type and its ubyte identifier. + Pair of type and its ubyte identifier. (Currently, ubyte is enough, but that might change in the future.) */ @@ -131,12 +131,12 @@ struct TypeId(T, ubyte id) { /* Structure of type identifier: - 0 1 + 0 1 primitive array/string something null/nothing numeric string - numeric char 0 0 Z H - integer float 0 [see left 0 0 + numeric char 0 0 Z H + integer float 0 [see left 0 0 unsigned signed 0 0 branch] 0 0 [ size in bytes] [size in bytes] 0 [element size] 1 1 @@ -144,25 +144,25 @@ unsigned signed 0 0 branch] 0 */ alias TypeTuple!(TypeId!(char, 0b001_00_1_00), - - TypeId!(ubyte, 0b001_0_0000), - TypeId!(ushort, 0b010_0_0000), - TypeId!(uint, 0b100_0__0__0__0__0), -/* Let's take 4 u i n s p - uint as an n n u o r - example b s t m m i + + TypeId!(ubyte, 0b001_0_0000), + TypeId!(ushort, 0b010_0_0000), + TypeId!(uint, 0b100_0__0__0__0__0), +/* Let's take 4 u i n s p + uint as an n n u o r + example b s t m m i y i e e e m t g g r t i e n e i h t s e r c i i d n v g e -*/ - +*/ + TypeId!(byte, 0b001_1_0000), - TypeId!(short, 0b010_1_0000), - TypeId!(int, 0b100_1_0000), + TypeId!(short, 0b010_1_0000), + TypeId!(int, 0b100_1_0000), TypeId!(float, 0b100_01_000), @@ -190,7 +190,7 @@ private template GetType(U) { /// Get tag for type T. /// /// Useful for comparison with tag field of Value struct. -/// +/// /// Example: /// ----------------------------------- /// Value v = "zzz"; @@ -251,10 +251,10 @@ string injectOpAssign() { string injectOpCast() { char[] cs = "static if".dup; - string injectSwitchPrimitive(string requested_type) + string injectSwitchPrimitive(string requested_type) { char[] cs = `switch (_tag) {`.dup; - + foreach (t2; PrimitiveTagValueTypes) { cs ~= `case GetTypeId!`~t2.ValueType.stringof~`: `~ ` return to!T(u.`~t2.ch~`);`.dup; @@ -266,10 +266,10 @@ string injectOpCast() { return cs.idup; } - string injectSwitchArrayElement(string requested_type) + string injectSwitchArrayElement(string requested_type) { char[] cs = `switch (_tag) {`.dup; - + foreach (t2; ArrayElementTagValueTypes) { cs ~= `case GetTypeId!(`~t2.ValueType.stringof~`[]): `~ ` return to!T(u.B`~t2.ch~`);`.dup; @@ -309,10 +309,10 @@ string injectOpCast() { } /** - Struct for representing tag values. + Struct for representing tag values. - Tagged union, allows to store - 8/16/32-bit integers, floats, chars, strings, + Tagged union, allows to store + 8/16/32-bit integers, floats, chars, strings, and arrays of integers/floats. */ struct Value { @@ -338,7 +338,7 @@ struct Value { Currently, type identifier for (u)int requires 8 bits. Fortunately, SAM/BAM specification doesn't use bigger integer types. However, in case of need to extend the hierarchy, the type - should be changed from ubyte to something bigger. + should be changed from ubyte to something bigger. */ ubyte _tag; @@ -382,16 +382,16 @@ struct Value { this(T)(T value) { opAssign(value); } - + /// sets 'H' tag instead of default 'Z'. Is not expected to be used much. void setHexadecimalFlag() { enforce(this.is_string); - + bam_typeid = 'H'; _tag = hexStringTag; - if (_tag != 0b111) { + if (_tag != 0b111) { u.H = u.Z; } } @@ -465,7 +465,7 @@ struct Value { } /// ditto - void toSam(Sink)(auto ref Sink sink) const + void toSam(Sink)(auto ref Sink sink) const if (isSomeSink!Sink) { if (is_integer) { @@ -512,7 +512,7 @@ struct Value { } /// ditto - void toJson(Sink)(auto ref Sink sink) const + void toJson(Sink)(auto ref Sink sink) const if (isSomeSink!Sink) { switch (_tag) { diff --git a/BioD/bio/std/hts/sam/header.d b/BioD/bio/std/hts/sam/header.d index 38fb4c0e..bf49e867 100644 --- a/BioD/bio/std/hts/sam/header.d +++ b/BioD/bio/std/hts/sam/header.d @@ -23,7 +23,7 @@ */ module bio.std.hts.sam.header; -import bio.std.hts.thirdparty.msgpack; +import msgpack; import bio.core.utils.format; import std.algorithm; diff --git a/BioD/bio/std/hts/thirdparty/msgpack.d b/BioD/bio/std/hts/thirdparty/msgpack.d deleted file mode 100644 index 84824e90..00000000 --- a/BioD/bio/std/hts/thirdparty/msgpack.d +++ /dev/null @@ -1,5171 +0,0 @@ -/** - * MessagePack serializer and deserializer implementation. - * - * MessagePack is a binary-based serialization specification. - * - * Example: - * ----- - * auto data = tuple("MessagePack!", [1, 2], true); - * - * auto serialized = pack(data); - * - * // ... - * - * typeof(data) deserialized; - * - * unpack(serialized, deserialized); - * - * assert(data == deserialized); - * ----- - * - * See_Also: - * $(LINK2 http://msgpack.org/, The MessagePack Project)$(BR) - * $(LINK2 http://wiki.msgpack.org/display/MSGPACK/Design+of+Serialization, MessagePack Design concept)$(BR) - * $(LINK2 http://wiki.msgpack.org/display/MSGPACK/Format+specification, MessagePack data format) - * - * Copyright: Copyright Masahiro Nakagawa 2010-. - * License: Boost License 1.0. - * Authors: Masahiro Nakagawa - */ -module bio.std.hts.thirdparty.msgpack; - -import std.array; -import std.exception; -import std.range; -import std.stdio; -import std.traits; -import std.typecons; -import std.typetuple; - -// for RefBuffer -version(Posix) -{ - import core.sys.posix.sys.uio : iovec; -} -else -{ - /** - * from core.sys.posix.sys.uio.iovec for compatibility with posix. - */ - struct iovec - { - void* iov_base; - size_t iov_len; - } -} - -// for Converting Endian using ntohs and ntohl; -version(Windows) -{ - import core.stdc.windows.winsock; -} -else -{ - import core.sys.posix.arpa.inet; -} - -version(EnableReal) -{ - enum EnableReal = true; -} -else -{ - enum EnableReal = false; -} - -static if (real.sizeof == double.sizeof) { - // for 80bit real inter-operation on non-x86 CPU - version = NonX86; - - import std.numeric; -} - -version(unittest) import std.file, core.stdc.string; - - -@trusted: - - -public: - - -// Convenient functions - - -/** - * Serializes $(D_PARAM args). - * - * Assumes single object if the length of $(D_PARAM args) == 1, - * otherwise array object. - * - * Params: - * args = the contents to serialize. - * - * Returns: - * a serialized data. - */ -ubyte[] pack(bool withFieldName = false, Args...)(in Args args) -{ - auto packer = Packer(withFieldName); - - static if (Args.length == 1) - packer.pack(args[0]); - else - packer.packArray(args); - - return packer.stream.data; -} - - -unittest -{ - auto serialized = pack(false); - - assert(serialized[0] == Format.FALSE); - - auto deserialized = unpack(pack(1, true, "Foo")); - - assert(deserialized.type == Value.Type.array); - assert(deserialized.via.array[0].type == Value.Type.unsigned); - assert(deserialized.via.array[1].type == Value.Type.boolean); - assert(deserialized.via.array[2].type == Value.Type.raw); -} - - -/** - * Deserializes $(D_PARAM buffer) using stream deserializer. - * - * Params: - * buffer = the buffer to deserialize. - * - * Returns: - * a $(D Unpacked) contains deserialized object. - * - * Throws: - * UnpackException if deserialization doesn't succeed. - */ -Unpacked unpack(in ubyte[] buffer) -{ - auto unpacker = StreamingUnpacker(buffer); - - if (!unpacker.execute()) - throw new UnpackException("Deserialization failure"); - - return unpacker.unpacked; -} - - -/** - * Deserializes $(D_PARAM buffer) using direct-conversion deserializer. - * - * Assumes single object if the length of $(D_PARAM args) == 1, - * otherwise array object. - * - * Params: - * buffer = the buffer to deserialize. - * args = the references of values to assign. - */ -void unpack(bool withFieldName = false, Args...)(in ubyte[] buffer, ref Args args) -{ - auto unpacker = Unpacker(buffer, buffer.length, withFieldName); - - static if (Args.length == 1) - unpacker.unpack(args[0]); - else - unpacker.unpackArray(args); -} - - -/** - * Return value version - */ -Type unpack(Type, bool withFieldName = false)(in ubyte[] buffer) -{ - auto unpacker = Unpacker(buffer, buffer.length, withFieldName); - - Type result; - unpacker.unpack(result); - return result; -} - - -unittest -{ - { // stream - auto result = unpack(pack(false)); - - assert(result.via.boolean == false); - } - { // direct conversion - Tuple!(uint, string) result; - Tuple!(uint, string) test = tuple(1, "Hi!"); - - unpack(pack(test), result); - assert(result == test); - - test.field[0] = 2; - test.field[1] = "Hey!"; - unpack(pack(test.field[0], test.field[1]), result.field[0], result.field[1]); - assert(result == test); - } - { // return value direct conversion - Tuple!(uint, string) test = tuple(1, "Hi!"); - - auto data = pack(test); - assert(data.unpack!(Tuple!(uint, string)) == test); - } - { // serialize object as a Map - static class C - { - int num; - - this(int num) { this.num = num; } - } - - auto test = new C(10); - auto result = new C(100); - - unpack!(true)(pack!(true)(test), result); - assert(result.num == 10, "Unpacking with field names failed"); - } -} - - -unittest -{ - // unittest for https://github.com/msgpack/msgpack-d/issues/8 - foreach (Type; TypeTuple!(byte, short, int, long)) { - foreach (i; [-33, -20, -1, 0, 1, 20, 33]) { - Type a = cast(Type)i; - Type b; - unpack(pack(a), b); - assert(a == b); - } - } -} - - -/** - * $(D MessagePackException) is a root Exception for MessagePack related operation. - */ -class MessagePackException : Exception -{ - pure this(string message) - { - super(message); - } -} - - -/** - * Attribute for specifying non pack/unpack field. - * This is an alternative approach of MessagePackable mixin. - * - * Example: - * ----- - * struct S - * { - * int num; - * // Packer/Unpacker ignores this field; - * @nonPacked string str; - * } - * ----- - */ -struct nonPacked {} - -template isPackedField(alias field) -{ - enum isPackedField = (staticIndexOf!(nonPacked, __traits(getAttributes, field)) == -1) && (!isSomeFunction!(typeof(field))); -} - - -// Serializing routines - - -/** - * $(D Packer) is a $(D MessagePack) serializer - * - * Example: - * ----- - * auto packer = packer(Appender!(ubyte[])()); - * - * packer.packArray(false, 100, 1e-10, null); - * - * stdout.rawWrite(packer.buffer.data); - * ----- - * - * NOTE: - * Current implementation can't deal with a circular reference. - * If you try to serialize a object that has circular reference, runtime raises 'Stack Overflow'. - */ -struct PackerImpl(Stream) if (isOutputRange!(Stream, ubyte) && isOutputRange!(Stream, ubyte[])) -{ - private: - static @system - { - alias void delegate(ref PackerImpl, void*) PackHandler; - PackHandler[TypeInfo] packHandlers; - - public void registerHandler(T, alias Handler)() - { - packHandlers[typeid(T)] = delegate(ref PackerImpl packer, void* obj) { - Handler(packer, *cast(T*)obj); - }; - } - } - - enum size_t Offset = 1; // type-information offset - - Stream stream_; // the stream to write - ubyte[Offset + RealSize] store_; // stores serialized value - bool withFieldName_; - - - public: - /** - * Constructs a packer with $(D_PARAM stream). - * - * Params: - * stream = the stream to write. - * withFieldName = serialize class / struct with field name - */ - this(Stream stream, bool withFieldName = false) - { - stream_ = stream; - withFieldName_ = withFieldName; - } - - - /** - * Constructs a packer with $(D_PARAM withFieldName). - * - * Params: - * withFieldName = serialize class / struct with field name - */ - this(bool withFieldName) - { - withFieldName_ = withFieldName; - } - - - /** - * Forwards to stream. - * - * Returns: - * the stream. - */ - @property @safe - nothrow ref Stream stream() - { - return stream_; - } - - - /** - * Serializes argument and writes to stream. - * - * If the argument is the pointer type, dereferences the pointer and serializes pointed value. - * ----- - * int a = 10; - * int* b = &b; - * - * packer.pack(b); // serializes 10, not address of a - * ----- - * Serializes nil if the argument of nullable type is null. - * - * NOTE: - * MessagePack doesn't define $(D_KEYWORD real) type format. - * Don't serialize $(D_KEYWORD real) if you communicate with other languages. - * Transfer $(D_KEYWORD double) serialization if $(D_KEYWORD real) on your environment equals $(D_KEYWORD double). - * - * Params: - * value = the content to serialize. - * - * Returns: - * self, i.e. for method chaining. - */ - ref PackerImpl pack(T)(in T value) if (is(Unqual!T == bool)) - { - if (value) - stream_.put(Format.TRUE); - else - stream_.put(Format.FALSE); - - return this; - } - - - /// ditto - ref PackerImpl pack(T)(in T value) if (isUnsigned!T && !is(Unqual!T == enum)) - { - // ulong < ulong is slower than uint < uint - static if (!is(Unqual!T == ulong)) { - enum Bits = T.sizeof * 8; - - if (value < (1 << 8)) { - if (value < (1 << 7)) { - // fixnum - stream_.put(take8from!Bits(value)); - } else { - // uint 8 - store_[0] = Format.UINT8; - store_[1] = take8from!Bits(value); - stream_.put(store_[0..Offset + ubyte.sizeof]); - } - } else { - if (value < (1 << 16)) { - // uint 16 - const temp = convertEndianTo!16(value); - - store_[0] = Format.UINT16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } else { - // uint 32 - const temp = convertEndianTo!32(value); - - store_[0] = Format.UINT32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } - } - } else { - if (value < (1UL << 8)) { - if (value < (1UL << 7)) { - // fixnum - stream_.put(take8from!64(value)); - } else { - // uint 8 - store_[0] = Format.UINT8; - store_[1] = take8from!64(value); - stream_.put(store_[0..Offset + ubyte.sizeof]); - } - } else { - if (value < (1UL << 16)) { - // uint 16 - const temp = convertEndianTo!16(value); - - store_[0] = Format.UINT16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } else if (value < (1UL << 32)){ - // uint 32 - const temp = convertEndianTo!32(value); - - store_[0] = Format.UINT32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } else { - // uint 64 - const temp = convertEndianTo!64(value); - - store_[0] = Format.UINT64; - *cast(ulong*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ulong.sizeof]); - } - } - } - - return this; - } - - - /// ditto - ref PackerImpl pack(T)(in T value) if (isSigned!T && isIntegral!T && !is(Unqual!T == enum)) - { - // long < long is slower than int < int - static if (!is(Unqual!T == long)) { - enum Bits = T.sizeof * 8; - - if (value < -(1 << 5)) { - if (value < -(1 << 15)) { - // int 32 - const temp = convertEndianTo!32(value); - - store_[0] = Format.INT32; - *cast(int*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + int.sizeof]); - } else if (value < -(1 << 7)) { - // int 16 - const temp = convertEndianTo!16(value); - - store_[0] = Format.INT16; - *cast(short*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + short.sizeof]); - } else { - // int 8 - store_[0] = Format.INT8; - store_[1] = take8from!Bits(value); - stream_.put(store_[0..Offset + byte.sizeof]); - } - } else if (value < (1 << 7)) { - // fixnum - stream_.put(take8from!Bits(value)); - } else { - if (value < (1 << 8)) { - // uint 8 - store_[0] = Format.UINT8; - store_[1] = take8from!Bits(value); - stream_.put(store_[0..Offset + ubyte.sizeof]); - } else if (value < (1 << 16)) { - // uint 16 - const temp = convertEndianTo!16(value); - - store_[0] = Format.UINT16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } else { - // uint 32 - const temp = convertEndianTo!32(value); - - store_[0] = Format.UINT32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } - } - } else { - if (value < -(1L << 5)) { - if (value < -(1L << 15)) { - if (value < -(1L << 31)) { - // int 64 - const temp = convertEndianTo!64(value); - - store_[0] = Format.INT64; - *cast(long*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + long.sizeof]); - } else { - // int 32 - const temp = convertEndianTo!32(value); - - store_[0] = Format.INT32; - *cast(int*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + int.sizeof]); - } - } else { - if (value < -(1L << 7)) { - // int 16 - const temp = convertEndianTo!16(value); - - store_[0] = Format.INT16; - *cast(short*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + short.sizeof]); - } else { - // int 8 - store_[0] = Format.INT8; - store_[1] = take8from!64(value); - stream_.put(store_[0..Offset + byte.sizeof]); - } - } - } else if (value < (1L << 7)) { - // fixnum - stream_.put(take8from!64(value)); - } else { - if (value < (1L << 16)) { - if (value < (1L << 8)) { - // uint 8 - store_[0] = Format.UINT8; - store_[1] = take8from!64(value); - stream_.put(store_[0..Offset + ubyte.sizeof]); - } else { - // uint 16 - const temp = convertEndianTo!16(value); - - store_[0] = Format.UINT16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } - } else { - if (value < (1L << 32)) { - // uint 32 - const temp = convertEndianTo!32(value); - - store_[0] = Format.UINT32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } else { - // uint 64 - const temp = convertEndianTo!64(value); - - store_[0] = Format.UINT64; - *cast(ulong*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ulong.sizeof]); - } - } - } - } - - return this; - } - - - /// ditto - ref PackerImpl pack(T)(in T value) if (isFloatingPoint!T && !is(Unqual!T == enum)) - { - static if (is(Unqual!T == float)) { - const temp = convertEndianTo!32(_f(value).i); - - store_[0] = Format.FLOAT; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } else static if (is(Unqual!T == double)) { - const temp = convertEndianTo!64(_d(value).i); - - store_[0] = Format.DOUBLE; - *cast(ulong*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ulong.sizeof]); - } else { - static if ((real.sizeof > double.sizeof) && EnableReal) { - store_[0] = Format.REAL; - const temp = _r(value); - const fraction = convertEndianTo!64(temp.fraction); - const exponent = convertEndianTo!16(temp.exponent); - - *cast(Unqual!(typeof(fraction))*)&store_[Offset] = fraction; - *cast(Unqual!(typeof(exponent))*)&store_[Offset + fraction.sizeof] = exponent; - stream_.put(store_[0..$]); - } else { // Non-x86 CPUs, real type equals double type. - pack(cast(double)value); - } - } - - return this; - } - - - /// ditto - ref PackerImpl pack(T)(in T value) if (is(Unqual!T == enum)) - { - pack(cast(OriginalType!T)value); - - return this; - } - - - /// Overload for pack(null) for 2.057 or later - static if (!is(typeof(null) == void*)) - { - ref PackerImpl pack(T)(in T value) if (is(Unqual!T == typeof(null))) - { - return packNil(); - } - } - - - /// ditto - ref PackerImpl pack(T)(in T value) if (isPointer!T) - { - static if (is(Unqual!T == void*)) { // for pack(null) for 2.056 or earlier - enforce(value is null, "Can't serialize void type"); - stream_.put(Format.NIL); - } else { - if (value is null) - stream_.put(Format.NIL); - else - pack(mixin(AsteriskOf!T ~ "value")); - } - - return this; - } - - - /// ditto - ref PackerImpl pack(T)(in T array) if (isArray!T) - { - alias typeof(T.init[0]) U; - - /* - * Serializes raw type-information to stream. - */ - void beginRaw(in size_t length) - { - if (length < 32) { - const ubyte temp = Format.RAW | cast(ubyte)length; - stream_.put(take8from(temp)); - } else if (length < 65536) { - const temp = convertEndianTo!16(length); - - store_[0] = Format.RAW16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } else { - const temp = convertEndianTo!32(length); - - store_[0] = Format.RAW32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } - } - - if (array.empty) - return packNil(); - - // Raw bytes - static if (isByte!(U) || isSomeChar!(U)) { - ubyte[] raw = cast(ubyte[])array; - - beginRaw(raw.length); - stream_.put(raw); - } else { - beginArray(array.length); - foreach (elem; array) - pack(elem); - } - - return this; - } - - - /// ditto - ref PackerImpl pack(T)(in T array) if (isAssociativeArray!T) - { - if (array is null) - return packNil(); - - beginMap(array.length); - foreach (key, value; array) { - pack(key); - pack(value); - } - - return this; - } - - - /// ditto - ref PackerImpl pack(Types...)(auto ref const Types objects) if (Types.length > 1) - { - foreach (i, T; Types) - pack(objects[i]); - - return this; - } - - - /** - * Serializes $(D_PARAM object) and writes to stream. - * - * Calling $(D toMsgpack) if $(D_KEYWORD class) and $(D_KEYWORD struct) implement $(D toMsgpack) method. $(D toMsgpack) signature is: - * ----- - * void toMsgpack(Packer)(ref Packer packer) const - * ----- - * This method serializes all members of T object if $(D_KEYWORD class) and $(D_KEYWORD struct) don't implement $(D toMsgpack). - * - * An object that doesn't implement $(D toMsgpack) is serialized to Array type. - * ----- - * packer.pack(tuple(true, 1, "Hi!")) // -> '[true, 1, "Hi!"]', not 'ture, 1, "Hi!"' - * - * struct Foo - * { - * int num = 10; - * string msg = "D!"; - * } - * packer.pack(Foo()); // -> '[10, "D!"]' - * - * class Base - * { - * bool flag = true; - * } - * class Derived : Base - * { - * double = 0.5f; - * } - * packer.pack(new Derived()); // -> '[true, 0.5f]' - * ----- - * - * Params: - * object = the content to serialize. - * - * Returns: - * self, i.e. for method chaining. - */ - ref PackerImpl pack(T)(in T object) if (is(Unqual!T == class)) - { - if (object is null) - return packNil(); - - static if (hasMember!(T, "toMsgpack")) - { - static if (__traits(compiles, { T t; t.toMsgpack(this, withFieldName_); })) { - object.toMsgpack(this, withFieldName_); - } else static if (__traits(compiles, { T t; t.toMsgpack(this); })) { // backward compatible - object.toMsgpack(this); - } else { - static assert(0, "Failed to invoke 'toMsgpack' on type '" ~ Unqual!T.stringof ~ "'"); - } - } else { - if (auto handler = object.classinfo in packHandlers) { - (*handler)(this, cast(void*)&object); - return this; - } - if (T.classinfo !is object.classinfo) { - throw new MessagePackException("Can't pack derived class through reference to base class."); - } - - alias SerializingClasses!(T) Classes; - - immutable memberNum = SerializingMemberNumbers!(Classes); - if (withFieldName_) - beginMap(memberNum); - else - beginArray(memberNum); - - foreach (Class; Classes) { - Class obj = cast(Class)object; - if (withFieldName_) { - foreach (i, f ; obj.tupleof) { - static if (isPackedField!(Class.tupleof[i])) { - pack(getFieldName!(Class, i)); - pack(f); - } - } - } else { - foreach (i, f ; obj.tupleof) { - static if (isPackedField!(Class.tupleof[i])) - pack(f); - } - } - } - } - - return this; - } - - - /// ditto - @trusted - ref PackerImpl pack(T)(auto ref T object) if (is(Unqual!T == struct)) - { - static if (hasMember!(T, "toMsgpack")) - { - static if (__traits(compiles, { T t; t.toMsgpack(this, withFieldName_); })) { - object.toMsgpack(this, withFieldName_); - } else static if (__traits(compiles, { T t; t.toMsgpack(this); })) { // backward compatible - object.toMsgpack(this); - } else { - static assert(0, "Failed to invoke 'toMsgpack' on type '" ~ Unqual!T.stringof ~ "'"); - } - } else static if (isTuple!T) { - beginArray(object.field.length); - foreach (f; object.field) - pack(f); - } else { // simple struct - if (auto handler = typeid(T) in packHandlers) { - (*handler)(this, cast(void*)&object); - return this; - } - - immutable memberNum = SerializingMemberNumbers!(T); - if (withFieldName_) - beginMap(memberNum); - else - beginArray(memberNum); - - if (withFieldName_) { - foreach (i, f; object.tupleof) { - static if (isPackedField!(T.tupleof[i]) && __traits(compiles, { pack(f); })) - { - pack(getFieldName!(T, i)); - pack(f); - } - } - } else { - foreach (i, f; object.tupleof) { - static if (isPackedField!(T.tupleof[i]) && __traits(compiles, { pack(f); })) - pack(f); - } - } - } - - return this; - } - - - /** - * Serializes the arguments as container to stream. - * - * ----- - * packer.packArray(true, 1); // -> [true, 1] - * packer.packMap("Hi", 100); // -> ["Hi":100] - * ----- - * - * In packMap, the number of arguments must be even. - * - * Params: - * objects = the contents to serialize. - * - * Returns: - * self, i.e. for method chaining. - */ - ref PackerImpl packArray(Types...)(auto ref const Types objects) - { - beginArray(Types.length); - foreach (i, T; Types) - pack(objects[i]); - //pack(objects); // slow :( - - return this; - } - - - /// ditto - ref PackerImpl packMap(Types...)(auto ref const Types objects) - { - static assert(Types.length % 2 == 0, "The number of arguments must be even"); - - beginMap(Types.length / 2); - foreach (i, T; Types) - pack(objects[i]); - - return this; - } - - - /** - * Serializes the type-information to stream. - * - * These methods don't serialize contents. - * You need to call pack method to serialize contents at your own risk. - * ----- - * packer.beginArray(3).pack(true, 1); // -> [true, 1, - * - * // other operation - * - * packer.pack("Hi!"); // -> [true, 1, "Hi!"] - * ----- - * - * Params: - * length = the length of container. - * - * Returns: - * self, i.e. for method chaining. - */ - ref PackerImpl beginArray(in size_t length) - { - if (length < 16) { - const ubyte temp = Format.ARRAY | cast(ubyte)length; - stream_.put(take8from(temp)); - } else if (length < 65536) { - const temp = convertEndianTo!16(length); - - store_[0] = Format.ARRAY16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } else { - const temp = convertEndianTo!32(length); - - store_[0] = Format.ARRAY32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } - - return this; - } - - - /// ditto - ref PackerImpl beginMap(in size_t length) - { - if (length < 16) { - const ubyte temp = Format.MAP | cast(ubyte)length; - stream_.put(take8from(temp)); - } else if (length < 65536) { - const temp = convertEndianTo!16(length); - - store_[0] = Format.MAP16; - *cast(ushort*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + ushort.sizeof]); - } else { - const temp = convertEndianTo!32(length); - - store_[0] = Format.MAP32; - *cast(uint*)&store_[Offset] = temp; - stream_.put(store_[0..Offset + uint.sizeof]); - } - - return this; - } - - - private: - /* - * Serializes the nil value. - */ - ref PackerImpl packNil() - { - stream_.put(Format.NIL); - return this; - } -} - - -/// Default serializer -alias PackerImpl!(Appender!(ubyte[])) Packer; // should be pure struct? - - -/** - * Register a serialization handler for $(D_PARAM T) type - * - * Example: - * ----- - * registerPackHandler!(Foo, fooPackHandler); - * ----- - */ -void registerPackHandler(T, alias Handler, Stream = Appender!(ubyte[]))() -{ - PackerImpl!(Stream).registerHandler!(T, Handler); -} - - -/** - * Helper for $(D Packer) construction. - * - * Params: - * stream = the stream to write. - * withFieldName = serialize class / struct with field name - * - * Returns: - * a $(D Packer) object instantiated and initialized according to the arguments. - */ -PackerImpl!(Stream) packer(Stream)(Stream stream, bool withFieldName = false) -{ - return typeof(return)(stream, withFieldName); -} - - -// Buffer implementations - - -/** - * $(D RefBuffer) is a reference stored buffer for more efficient serialization - * - * Example: - * ----- - * auto packer = packer(RefBuffer(16)); // threshold is 16 - * - * // packs data - * - * writev(fd, cast(void*)packer.buffer.vector.ptr, packer.buffer.vector.length); - * ----- - */ -struct RefBuffer -{ - private: - static struct Chunk - { - ubyte[] data; // storing serialized value - size_t used; // used size of data - } - - immutable size_t Threshold; - immutable size_t ChunkSize; - - // for putCopy - Chunk[] chunks_; // memory chunk for buffer - size_t index_; // index for cunrrent chunk - - // for putRef - iovec[] vecList_; // reference to large data or copied data. - - - public: - /** - * Constructs a buffer. - * - * Params: - * threshold = the threshold of writing value or stores reference. - * chunkSize = the default size of chunk for allocation. - */ - @safe - this(in size_t threshold, in size_t chunkSize = 8192) - { - Threshold = threshold; - ChunkSize = chunkSize; - - chunks_.length = 1; - chunks_[index_].data.length = chunkSize; - } - - - /** - * Returns the buffer contents that excluding references. - * - * Returns: - * the non-contiguous copied contents. - */ - @property @safe - nothrow ubyte[] data() - { - ubyte[] result; - - foreach (ref chunk; chunks_) - result ~= chunk.data[0..chunk.used]; - - return result; - } - - - /** - * Forwards to all buffer contents. - * - * Returns: - * the array of iovec struct that stores references. - */ - @property @safe - nothrow ref auto vector() - { - return vecList_; - } - - - /** - * Writes the argument to buffer and stores the reference of writed content - * if the argument size is smaller than threshold, - * otherwise stores the reference of argument directly. - * - * Params: - * value = the content to write. - */ - @safe - void put(in ubyte value) - { - ubyte[1] values = [value]; - putCopy(values); - } - - - /// ditto - @safe - void put(in ubyte[] value) - { - if (value.length < Threshold) - putCopy(value); - else - putRef(value); - } - - - private: - /* - * Stores the reference of $(D_PARAM value). - * - * Params: - * value = the content to write. - */ - @trusted - void putRef(in ubyte[] value) - { - vecList_.length += 1; - vecList_[$ - 1] = iovec(cast(void*)value.ptr, value.length); - } - - - /* - * Writes $(D_PARAM value) to buffer and appends to its reference. - * - * Params: - * value = the contents to write. - */ - @trusted - void putCopy(in ubyte[] value) - { - /* - * Helper for expanding new space. - */ - void expand(in size_t size) - { - const newSize = size < ChunkSize ? ChunkSize : size; - - index_++; - chunks_.length = 1; - chunks_[index_].data.length = newSize; - } - - const size = value.length; - - // lacks current chunk? - if (chunks_[index_].data.length - chunks_[index_].used < size) - expand(size); - - const base = chunks_[index_].used; // start index - auto data = chunks_[index_].data[base..base + size]; // chunk to write - - data[] = value[]; - chunks_[index_].used += size; - - // Optimization for avoiding iovec allocation. - if (vecList_.length && data.ptr == (vecList_[$ - 1].iov_base + - vecList_[$ - 1].iov_len)) - vecList_[$ - 1].iov_len += size; - else - putRef(data); - } -} - - -unittest -{ - static assert(isOutputRange!(RefBuffer, ubyte) && - isOutputRange!(RefBuffer, ubyte[])); - - auto buffer = RefBuffer(2, 4); - - ubyte[] tests = [1, 2]; - foreach (v; tests) - buffer.put(v); - buffer.put(tests); - - assert(buffer.data == tests, "putCopy failed"); - - iovec[] vector = buffer.vector; - ubyte[] result; - - assert(vector.length == 2, "Optimization failed"); - - foreach (v; vector) - result ~= (cast(ubyte*)v.iov_base)[0..v.iov_len]; - - assert(result == tests ~ tests); -} - - -version (unittest) -{ - mixin template DefinePacker() - { - Packer packer; - } - - mixin template DefineDictionalPacker() - { - Packer packer = Packer(false); - } -} - -unittest -{ - { // unique value - mixin DefinePacker; - - ubyte[] result = [Format.NIL, Format.TRUE, Format.FALSE]; - - packer.pack(null, true, false); - foreach (i, value; packer.stream.data) - assert(value == result[i]); - } - { // uint * - static struct UTest { ubyte format; ulong value; } - - enum : ulong { A = ubyte.max, B = ushort.max, C = uint.max, D = ulong.max } - - static UTest[][] utests = [ - [{Format.UINT8, A}], - [{Format.UINT8, A}, {Format.UINT16, B}], - [{Format.UINT8, A}, {Format.UINT16, B}, {Format.UINT32, C}], - [{Format.UINT8, A}, {Format.UINT16, B}, {Format.UINT32, C}, {Format.UINT64, D}], - ]; - - foreach (I, T; TypeTuple!(ubyte, ushort, uint, ulong)) { - foreach (i, test; utests[I]) { - mixin DefinePacker; - - packer.pack(cast(T)test.value); - assert(packer.stream.data[0] == test.format); - - switch (i) { - case 0: - auto answer = take8from!(T.sizeof * 8)(test.value); - assert(memcmp(&packer.stream.data[1], &answer, ubyte.sizeof) == 0); - break; - case 1: - auto answer = convertEndianTo!16(test.value); - assert(memcmp(&packer.stream.data[1], &answer, ushort.sizeof) == 0); - break; - case 2: - auto answer = convertEndianTo!32(test.value); - assert(memcmp(&packer.stream.data[1], &answer, uint.sizeof) == 0); - break; - default: - auto answer = convertEndianTo!64(test.value); - assert(memcmp(&packer.stream.data[1], &answer, ulong.sizeof) == 0); - } - } - } - } - { // int * - static struct STest { ubyte format; long value; } - - enum : long { A = byte.min, B = short.min, C = int.min, D = long.min } - - static STest[][] stests = [ - [{Format.INT8, A}], - [{Format.INT8, A}, {Format.INT16, B}], - [{Format.INT8, A}, {Format.INT16, B}, {Format.INT32, C}], - [{Format.INT8, A}, {Format.INT16, B}, {Format.INT32, C}, {Format.INT64, D}], - ]; - - foreach (I, T; TypeTuple!(byte, short, int, long)) { - foreach (i, test; stests[I]) { - mixin DefinePacker; - - packer.pack(cast(T)test.value); - assert(packer.stream.data[0] == test.format); - - switch (i) { - case 0: - auto answer = take8from!(T.sizeof * 8)(test.value); - assert(memcmp(&packer.stream.data[1], &answer, byte.sizeof) == 0); - break; - case 1: - auto answer = convertEndianTo!16(test.value); - assert(memcmp(&packer.stream.data[1], &answer, short.sizeof) == 0); - break; - case 2: - auto answer = convertEndianTo!32(test.value); - assert(memcmp(&packer.stream.data[1], &answer, int.sizeof) == 0); - break; - default: - auto answer = convertEndianTo!64(test.value); - assert(memcmp(&packer.stream.data[1], &answer, long.sizeof) == 0); - } - } - } - } - { // fload, double - static if ((real.sizeof == double.sizeof) || !EnableReal) - { - alias TypeTuple!(float, double, double) FloatingTypes; - static struct FTest { ubyte format; double value; } - - static FTest[] ftests = [ - {Format.FLOAT, float.min_normal}, - {Format.DOUBLE, double.max}, - {Format.DOUBLE, double.max}, - ]; - } - else - { - alias TypeTuple!(float, double, real) FloatingTypes; - static struct FTest { ubyte format; real value; } - - static FTest[] ftests = [ - {Format.FLOAT, float.min_normal}, - {Format.DOUBLE, double.max}, - {Format.REAL, real.max}, - ]; - } - - foreach (I, T; FloatingTypes) { - mixin DefinePacker; - - packer.pack(cast(T)ftests[I].value); - assert(packer.stream.data[0] == ftests[I].format); - - switch (I) { - case 0: - const answer = convertEndianTo!32(_f(cast(T)ftests[I].value).i); - assert(memcmp(&packer.stream.data[1], &answer, float.sizeof) == 0); - break; - case 1: - const answer = convertEndianTo!64(_d(cast(T)ftests[I].value).i); - assert(memcmp(&packer.stream.data[1], &answer, double.sizeof) == 0); - break; - default: - static if (EnableReal) - { - const t = _r(cast(T)ftests[I].value); - const f = convertEndianTo!64(t.fraction); - const e = convertEndianTo!16(t.exponent); - assert(memcmp(&packer.stream.data[1], &f, f.sizeof) == 0); - assert(memcmp(&packer.stream.data[1 + f.sizeof], &e, e.sizeof) == 0); - } - else - { - const answer = convertEndianTo!64(_d(cast(T)ftests[I].value).i); - assert(memcmp(&packer.stream.data[1], &answer, double.sizeof) == 0); - } - } - } - } - { // pointer - static struct PTest - { - ubyte format; - - union - { - ulong* p0; - long* p1; - double* p2; - } - } - - PTest[] ptests = [PTest(Format.UINT64), PTest(Format.INT64), PTest(Format.DOUBLE)]; - - ulong v0 = ulong.max; - long v1 = long.min; - double v2 = double.max; - - foreach (I, Index; TypeTuple!("0", "1", "2")) { - mixin DefinePacker; - - mixin("ptests[I].p" ~ Index ~ " = &v" ~ Index ~ ";"); - - packer.pack(mixin("ptests[I].p" ~ Index)); - assert(packer.stream.data[0] == ptests[I].format); - - switch (I) { - case 0: - auto answer = convertEndianTo!64(*ptests[I].p0); - assert(memcmp(&packer.stream.data[1], &answer, ulong.sizeof) == 0); - break; - case 1: - auto answer = convertEndianTo!64(*ptests[I].p1); - assert(memcmp(&packer.stream.data[1], &answer, long.sizeof) == 0); - break; - default: - const answer = convertEndianTo!64(_d(*ptests[I].p2).i); - assert(memcmp(&packer.stream.data[1], &answer, double.sizeof) == 0); - } - } - } - { // enum - enum E : ubyte { A = ubyte.max } - - mixin DefinePacker; E e = E.A; - - packer.pack(e); - assert(packer.stream.data[0] == Format.UINT8); - - auto answer = E.A; - assert(memcmp(&packer.stream.data[1], &answer, (OriginalType!E).sizeof) == 0); - } - { // container - static struct CTest { ubyte format; size_t value; } - - enum : ulong { A = 16 / 2, B = ushort.max, C = uint.max } - - static CTest[][] ctests = [ - [{Format.ARRAY | A, Format.ARRAY | A}, {Format.ARRAY16, B}, {Format.ARRAY32, C}], - [{Format.MAP | A, Format.MAP | A}, {Format.MAP16, B}, {Format.MAP32, C}], - ]; - - foreach (I, Name; TypeTuple!("Array", "Map")) { - auto test = ctests[I]; - - foreach (i, T; TypeTuple!(ubyte, ushort, uint)) { - mixin DefinePacker; - mixin("packer.begin" ~ Name ~ "(i ? test[i].value : A);"); - - assert(packer.stream.data[0] == test[i].format); - - switch (i) { - case 0: - auto answer = take8from(test[i].value); - assert(memcmp(&packer.stream.data[0], &answer, ubyte.sizeof) == 0); - break; - case 1: - auto answer = convertEndianTo!16(test[i].value); - assert(memcmp(&packer.stream.data[1], &answer, ushort.sizeof) == 0); - break; - default: - auto answer = convertEndianTo!32(test[i].value); - assert(memcmp(&packer.stream.data[1], &answer, uint.sizeof) == 0); - } - } - } - } - { // user defined - { - static struct S - { - uint num = uint.max; - - void toMsgpack(P)(ref P p) const { p.packArray(num); } - } - - mixin DefinePacker; S test; - - packer.pack(test); - - assert(packer.stream.data[0] == (Format.ARRAY | 1)); - assert(packer.stream.data[1] == Format.UINT32); - assert(memcmp(&packer.stream.data[2], &test.num, uint.sizeof) == 0); - } - { - mixin DefinePacker; auto test = tuple(true, false, uint.max); - - packer.pack(test); - - assert(packer.stream.data[0] == (Format.ARRAY | 3)); - assert(packer.stream.data[1] == Format.TRUE); - assert(packer.stream.data[2] == Format.FALSE); - assert(packer.stream.data[3] == Format.UINT32); - assert(memcmp(&packer.stream.data[4], &test.field[2], uint.sizeof) == 0); - } - { - static class C - { - uint num; - - this(uint n) { num = n; } - - void toMsgpack(P)(ref P p) const { p.packArray(num); } - } - - mixin DefinePacker; C test = new C(ushort.max); - - packer.pack(test); - - assert(packer.stream.data[0] == (Format.ARRAY | 1)); - assert(packer.stream.data[1] == Format.UINT16); - assert(memcmp(&packer.stream.data[2], &test.num, ushort.sizeof) == 0); - } - } - { // simple struct and class - { - static struct Simple - { - uint num = uint.max; - } - - static struct SimpleWithNonPacked1 - { - uint num = uint.max; - @nonPacked string str = "ignored"; - } - - static struct SimpleWithNonPacked2 - { - @nonPacked string str = "ignored"; - uint num = uint.max; - } - - static struct SimpleWithSkippedTypes - { - int function(int) fn; - int delegate(int) dg; - uint num = uint.max; - } - - foreach (Type; TypeTuple!(Simple, SimpleWithNonPacked1, SimpleWithNonPacked2, SimpleWithSkippedTypes)) { - mixin DefinePacker; - - Type test; - packer.pack(test); - - assert(packer.stream.data[0] == (Format.ARRAY | 1)); - assert(packer.stream.data[1] == Format.UINT32); - assert(memcmp(&packer.stream.data[2], &test.num, uint.sizeof) == 0); - } - } - - static class SimpleA - { - bool flag = true; - } - - static class SimpleB : SimpleA - { - ubyte type = 100; - } - - static class SimpleC : SimpleB - { - uint num = uint.max; - } - - static class SimpleCWithNonPacked1 : SimpleB - { - uint num = uint.max; - @nonPacked string str = "ignored"; - } - - static class SimpleCWithNonPacked2 : SimpleB - { - @nonPacked string str = "ignored"; - uint num = uint.max; - } - - static class SimpleCWithSkippedTypes : SimpleB - { - uint num = uint.max; - int function(int) fn; - int delegate(int) dg; - } - - { // from derived class - foreach (Type; TypeTuple!(SimpleC, SimpleCWithNonPacked1, SimpleCWithNonPacked2, SimpleCWithSkippedTypes)) { - mixin DefinePacker; - - Type test = new Type(); - packer.pack(test); - - assert(packer.stream.data[0] == (Format.ARRAY | 3)); - assert(packer.stream.data[1] == Format.TRUE); - assert(packer.stream.data[2] == 100); - assert(packer.stream.data[3] == Format.UINT32); - assert(memcmp(&packer.stream.data[4], &test.num, uint.sizeof) == 0); - } - } - { // from base class - mixin DefinePacker; SimpleB test = new SimpleC(); - - try { - packer.pack(test); - assert(false); - } catch (Exception e) { } - } - } -} - - -// deserializing routines - - -/** - * $(D UnpackException) is thrown on deserialization failure - */ -class UnpackException : MessagePackException -{ - this(string message) - { - super(message); - } -} - - -version (D_Ddoc) -{ - /** - * Internal buffer and related operations for Unpacker - * - * Following Unpackers mixin this template. So, Unpacker can use following methods. - * - * ----- - * //buffer image: - * +-------------------------------------------+ - * | [object] | [obj | unparsed... | unused... | - * +-------------------------------------------+ - * ^ offset - * ^ current - * ^ used - * ^ buffer.length - * ----- - * - * This mixin template is a private. - */ - mixin template InternalBuffer() - { - private: - ubyte[] buffer_; // internal buffer - size_t used_; // index that buffer cosumed - size_t offset_; // index that buffer parsed - size_t parsed_; // total size of parsed message - bool hasRaw_; // indicates whether Raw object has been deserialized - - - public: - /** - * Forwards to internal buffer. - * - * Returns: - * the reference of internal buffer. - */ - @property @safe - nothrow ubyte[] buffer(); - - - /** - * Fills internal buffer with $(D_PARAM target). - * - * Params: - * target = new serialized buffer to deserialize. - */ - @safe void feed(in ubyte[] target); - - - /** - * Consumes buffer. This method is helper for buffer property. - * You must use this method if you write bytes to buffer directly. - * - * Params: - * size = the number of consuming. - */ - @safe - nothrow void bufferConsumed(in size_t size); - - - /** - * Removes unparsed buffer. - */ - @safe - nothrow void removeUnparsed(); - - - /** - * Returns: - * the total size including unparsed buffer size. - */ - @property @safe - nothrow size_t size() const; - - - /** - * Returns: - * the parsed size of buffer. - */ - @property @safe - nothrow size_t parsedSize() const; - - - /** - * Returns: - * the unparsed size of buffer. - */ - @property @safe - nothrow size_t unparsedSize() const; - - - private: - @safe - void initializeBuffer(in ubyte[] target, in size_t bufferSize = 8192); - } -} -else -{ - private mixin template InternalBuffer() - { - private: - ubyte[] buffer_; // internal buffer - size_t used_; // index that buffer cosumed - size_t offset_; // index that buffer parsed - size_t parsed_; // total size of parsed message - bool hasRaw_; // indicates whether Raw object has been deserialized - - - public: - @property @safe - nothrow ubyte[] buffer() - { - return buffer_; - } - - - @safe - void feed(in ubyte[] target) - in - { - assert(target.length); - } - do - { - /* - * Expands internal buffer. - * - * Params: - * size = new buffer size to append. - */ - void expandBuffer(in size_t size) - { - // rewinds buffer(completed deserialization) - if (used_ == offset_ && !hasRaw_) { - used_ = offset_ = 0; - - if (buffer_.length < size) - buffer_.length = size; - - return; - } - - // deserializing state is mid-flow(buffer has non-parsed data yet) - auto unparsed = buffer_[offset_..used_]; - auto restSize = buffer_.length - used_ + offset_; - auto newSize = size > restSize ? unparsedSize + size : buffer_.length; - - if (hasRaw_) { - hasRaw_ = false; - buffer_ = new ubyte[](newSize); - } else { - buffer_.length = newSize; - - // avoids overlapping copy - auto area = buffer_[0..unparsedSize]; - unparsed = area.overlap(unparsed) ? unparsed.dup : unparsed; - } - - buffer_[0..unparsedSize] = unparsed[]; - used_ = unparsedSize; - offset_ = 0; - } - - const size = target.length; - - // lacks current buffer? - if (buffer_.length - used_ < size) - expandBuffer(size); - - buffer_[used_..used_ + size] = target[]; - used_ += size; - } - - - @safe - nothrow void bufferConsumed(in size_t size) - { - if (used_ + size > buffer_.length) - used_ = buffer_.length; - else - used_ += size; - } - - - @safe - nothrow void removeUnparsed() - { - used_ = offset_; - } - - - @property @safe - nothrow size_t size() const - { - return parsed_ - offset_ + used_; - } - - - @property @safe - nothrow size_t parsedSize() const - { - return parsed_; - } - - - @property @safe - nothrow size_t unparsedSize() const - { - return used_ - offset_; - } - - - private: - @safe - nothrow void initializeBuffer(in ubyte[] target, in size_t bufferSize = 8192) - { - const size = target.length; - - buffer_ = new ubyte[](size > bufferSize ? size : bufferSize); - used_ = size; - buffer_[0..size] = target[]; - } - } -} - - -/** - * This $(D Unpacker) is a $(D MessagePack) direct-conversion deserializer - * - * This implementation is suitable for fixed data. - * - * Example: - * ----- - * // serializedData is [10, 0.1, false] - * auto unpacker = Unpacker(serializedData); - * - * uint n; - * double d; - * bool b; - * - * unpacker.unpackArray(n, d, b); - * - * // using Tuple - * Tuple!(uint, double, bool) record; - * unpacker.unpack(record); // record is [10, 0.1, false] - * ----- - * - * NOTE: - * Unpacker becomes template struct if Phobos supports truly IO module. - */ -struct Unpacker -{ - private: - static @system - { - alias void delegate(ref Unpacker, void*) UnpackHandler; - UnpackHandler[TypeInfo] unpackHandlers; - - public void registerHandler(T, alias Handler)() - { - unpackHandlers[typeid(T)] = delegate(ref Unpacker unpacker, void* obj) { - Handler(unpacker, *cast(T*)obj); - }; - } - } - - enum Offset = 1; - - mixin InternalBuffer; - - bool withFieldName_; - - - public: - /** - * Constructs a $(D Unpacker). - * - * Params: - * target = byte buffer to deserialize - * bufferSize = size limit of buffer size - */ - this(in ubyte[] target, in size_t bufferSize = 8192, bool withFieldName = false) - { - initializeBuffer(target, bufferSize); - withFieldName_ = withFieldName; - } - - - /** - * Clears states for next deserialization. - */ - @safe - nothrow void clear() - { - parsed_ = 0; - } - - - /** - * Deserializes $(D_PARAM T) object and assigns to $(D_PARAM value). - * - * If the argument is pointer, dereferences pointer and assigns deserialized value. - * ----- - * int* a; - * unpacker.unpack(a) // enforce throws Exception because a is null or - * // no throw if deserialized value is nil - * - * int b; a = &b; - * unpacker.unpack(b) // b is deserialized value or - * // assigns null if deserialized value is nil - * ----- - * - * Params: - * value = the reference of value to assign. - * - * Returns: - * self, i.e. for method chaining. - * - * Throws: - * UnpackException when doesn't read from buffer or precision loss occurs and - * MessagePackException when $(D_PARAM T) type doesn't match serialized type. - */ - ref Unpacker unpack(T)(ref T value) if (is(Unqual!T == bool)) - { - canRead(Offset, 0); - const header = read(); - - switch (header) { - case Format.TRUE: - value = true; - break; - case Format.FALSE: - value = false; - break; - default: - rollback(); - } - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(ref T value) if (isUnsigned!T && !is(Unqual!T == enum)) - { - canRead(Offset, 0); - const header = read(); - - if (0x00 <= header && header <= 0x7f) { - value = header; - } else { - switch (header) { - case Format.UINT8: - canRead(ubyte.sizeof); - value = read(); - break; - case Format.UINT16: - canRead(ushort.sizeof); - auto us = load16To!ushort(read(ushort.sizeof)); - if (us > T.max) - rollback(ushort.sizeof); - value = cast(T)us; - break; - case Format.UINT32: - canRead(uint.sizeof); - auto ui = load32To!uint(read(uint.sizeof)); - if (ui > T.max) - rollback(uint.sizeof); - value = cast(T)ui; - break; - case Format.UINT64: - canRead(ulong.sizeof); - auto ul = load64To!ulong(read(ulong.sizeof)); - if (ul > T.max) - rollback(ulong.sizeof); - value = cast(T)ul; - break; - default: - rollback(); - } - } - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(ref T value) if (isSigned!T && isIntegral!T && !is(Unqual!T == enum)) - { - canRead(Offset, 0); - const header = read(); - - if (0x00 <= header && header <= 0x7f) { - value = cast(T)header; - } else if (0xe0 <= header && header <= 0xff) { - value = -(cast(T)-header); - } else { - switch (header) { - case Format.UINT8: - canRead(ubyte.sizeof); - auto ub = read(); - if (ub > T.max) - rollback(ubyte.sizeof); - value = cast(T)ub; - break; - case Format.UINT16: - canRead(ushort.sizeof); - auto us = load16To!ushort(read(ushort.sizeof)); - if (us > T.max) - rollback(ushort.sizeof); - value = cast(T)us; - break; - case Format.UINT32: - canRead(uint.sizeof); - auto ui = load32To!uint(read(uint.sizeof)); - if (ui > T.max) - rollback(uint.sizeof); - value = cast(T)ui; - break; - case Format.UINT64: - canRead(ulong.sizeof); - auto ul = load64To!ulong(read(ulong.sizeof)); - if (ul > T.max) - rollback(ulong.sizeof); - value = cast(T)ul; - break; - case Format.INT8: - canRead(byte.sizeof); - value = cast(byte)read(); - break; - case Format.INT16: - canRead(short.sizeof); - auto s = load16To!short(read(short.sizeof)); - if (s < T.min || T.max < s) - rollback(short.sizeof); - value = cast(T)s; - break; - case Format.INT32: - canRead(int.sizeof); - auto i = load32To!int(read(int.sizeof)); - if (i < T.min || T.max < i) - rollback(int.sizeof); - value = cast(T)i; - break; - case Format.INT64: - canRead(long.sizeof); - auto l = load64To!long(read(long.sizeof)); - if (l < T.min || T.max < l) - rollback(long.sizeof); - value = cast(T)l; - break; - default: - rollback(); - } - } - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(ref T value) if (isFloatingPoint!T && !is(Unqual!T == enum)) - { - canRead(Offset, 0); - const header = read(); - - switch (header) { - case Format.FLOAT: - _f temp; - - canRead(uint.sizeof); - temp.i = load32To!uint(read(uint.sizeof)); - value = temp.f; - break; - case Format.DOUBLE: - // check precision loss - static if (is(Unqual!T == float)) - rollback(); - - _d temp; - - canRead(ulong.sizeof); - temp.i = load64To!ulong(read(ulong.sizeof)); - value = temp.f; - break; - case Format.REAL: - static if (!EnableReal) - { - rollback(); - } - else - { - // check precision loss - static if (is(Unqual!T == float) || is(Unqual!T == double)) - rollback(); - - canRead(RealSize); - - version (NonX86) - { - CustomFloat!80 temp; - - const frac = load64To!ulong (read(ulong.sizeof)); - const exp = load16To!ushort(read(ushort.sizeof)); - - temp.significand = frac; - temp.exponent = exp & 0x7fff; - temp.sign = exp & 0x8000 ? true : false; - - // NOTE: temp.get!real is inf on non-x86 when deserialized value is larger than double.max. - value = temp.get!real; - } - else - { - _r temp; - - temp.fraction = load64To!(typeof(temp.fraction))(read(temp.fraction.sizeof)); - temp.exponent = load16To!(typeof(temp.exponent))(read(temp.exponent.sizeof)); - - value = temp.f; - } - } - - break; - default: - rollback(); - } - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(ref T value) if (is(Unqual!T == enum)) - { - OriginalType!T temp; - - unpack(temp); - - value = cast(T)temp; - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(T value) if (isPointer!T) - { - static if (is(Unqual!T == void*)) { - enforce(value !is null, "Can't deserialize void type"); - unpackNil(value); - } else { - if (checkNil()) - unpackNil(value); - else - enforce(value !is null, T.stringof ~ " is null pointer"); - - unpack(mixin(AsteriskOf!T ~ "value")); - } - - return this; - } - - - /// ditto - ref Unpacker unpack(Types...)(ref Types objects) if (Types.length > 1) - { - foreach (i, T; Types) - unpack!(T)(objects[i]); - - return this; - } - - - /** - * Deserializes $(D_PARAM T) object and assigns to $(D_PARAM array). - * - * This is convenient method for array deserialization. - * Rollback will be completely successful if you deserialize raw type((u)byte[] or string types). - * But, Rollback will be one element(e.g. int) if you deserialize other types(e.g. int[], int[int]) - * - * No assign if the length of deserialized object is 0. - * - * In a static array, this method checks the length. Do rollback and throw exception - * if length of $(D_PARAM array) is different from length of deserialized object. - * - * Params: - * array = the reference of array to assign. - * - * Returns: - * self, i.e. for method chaining. - * - * Throws: - * UnpackException when doesn't read from buffer or precision loss occurs and - * MessagePackException when $(D_PARAM T) type doesn't match serialized type. - */ - ref Unpacker unpack(T)(ref T array) if (isArray!T && !is(Unqual!T == enum)) - { - alias typeof(T.init[0]) U; - - /* - * Deserializes type-information of raw type. - */ - @safe - size_t beginRaw() - { - canRead(Offset, 0); - const header = read(); - size_t length; - - if (0xa0 <= header && header <= 0xbf) { - length = header & 0x1f; - } else { - switch (header) { - case Format.BIN8, Format.STR8: - canRead(ubyte.sizeof); - length = read(); - break; - case Format.BIN16, Format.RAW16: - canRead(ushort.sizeof); - length = load16To!size_t(read(ushort.sizeof)); - break; - case Format.BIN32, Format.RAW32: - canRead(uint.sizeof); - length = load32To!size_t(read(uint.sizeof)); - break; - case Format.NIL: - break; - default: - rollback(); - } - } - - return length; - } - - - if (checkNil()) { - static if (isStaticArray!T) { - onInvalidType(); - } else { - return unpackNil(array); - } - } - - // Raw bytes - static if (isByte!U || isSomeChar!U) { - auto length = beginRaw(); - auto offset = calculateSize!(true)(length); - if (length == 0) - return this; - - static if (isStaticArray!T) { - if (length != array.length) - rollback(offset); - } - - canRead(length, offset + Offset); - static if (isStaticArray!T) { - array[] = (cast(U[])read(length))[0 .. T.length]; - } else { - array = cast(T)read(length); - } - - static if (isDynamicArray!T) - hasRaw_ = true; - } else { - auto length = beginArray(); - if (length == 0) - return this; - - static if (isStaticArray!T) { - if (length != array.length) - rollback(calculateSize(length)); - } else { - array.length = length; - } - - foreach (i; 0..length) - unpack(array[i]); - } - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(ref T array) if (isAssociativeArray!T) - { - alias typeof(T.init.keys[0]) K; - alias typeof(T.init.values[0]) V; - - if (checkNil()) - return unpackNil(array); - - auto length = beginMap(); - if (length == 0) - return this; - - foreach (i; 0..length) { - K k; unpack(k); - V v; unpack(v); - array[k] = v; - } - - return this; - } - - - /** - * Deserializes $(D_PARAM T) object and assigns to $(D_PARAM object). - * - * Calling $(D fromMsgpack) if $(D_KEYWORD class) and $(D_KEYWORD struct) implement $(D fromMsgpack) method. $(D fromMsgpack) signature is: - * ----- - * void fromMsgpack(ref Unpacker unpacker) - * ----- - * Assumes $(D std.typecons.Tuple) or simple struct if $(D_KEYWORD struct) doesn't implement $(D fromMsgpack). - * Checks length if $(D_PARAM T) is a $(D std.typecons.Tuple) or simple struct. - * - * Params: - * object = the reference of object to assign. - * args = the arguments to class constructor(class only). - * This is used at new statement if $(D_PARAM object) is $(D_KEYWORD null). - * - * Returns: - * self, i.e. for method chaining. - */ - ref Unpacker unpack(T, Args...)(ref T object, auto ref Args args) if (is(Unqual!T == class)) - { - if (checkNil()) - return unpackNil(object); - - if (object is null) { - //static if (is(typeof(new T(args)))) - static if (__traits(compiles, { new T(args); })) - object = new T(args); - else - throw new MessagePackException("Don't know how to construct class type '" ~ Unqual!T.stringof ~ "' with argument types '" ~ Args.stringof ~ "'."); - } - - static if (hasMember!(T, "fromMsgpack")) - { - static if (__traits(compiles, { T t; t.fromMsgpack(this, withFieldName_); })) { - object.fromMsgpack(this, withFieldName_); - } else static if (__traits(compiles, { T t; t.fromMsgpack(this); })) { // backward compatible - object.fromMsgpack(this); - } else { - static assert(0, "Failed to invoke 'fromMsgpack' on type '" ~ Unqual!T.stringof ~ "'"); - } - } else { - if (auto handler = object.classinfo in unpackHandlers) { - (*handler)(this, cast(void*)&object); - return this; - } - if (T.classinfo !is object.classinfo) { - throw new MessagePackException("Can't unpack derived class through reference to base class."); - } - - alias SerializingClasses!(T) Classes; - - size_t length = withFieldName_ ? beginMap() : beginArray(); - if (length == 0) - return this; - - if (length != SerializingMemberNumbers!(Classes)) - rollback(calculateSize(length)); - - if (withFieldName_) { - foreach (_; 0..length) { - string fieldName; - unpack(fieldName); - - foreach (Class; Classes) { - Class obj = cast(Class)object; - - foreach (i, member; obj.tupleof) { - static if (isPackedField!(Class.tupleof[i])) - { - if (fieldName == getFieldName!(Class, i)) { - unpack(obj.tupleof[i]); - goto endLoop; - } - } - } - } - assert(false, "Invalid field name: '" ~ fieldName~"' "); - - endLoop: - continue; - } - } else { - foreach (Class; Classes) { - Class obj = cast(Class)object; - - foreach (i, member; obj.tupleof) { - static if (isPackedField!(Class.tupleof[i])) - unpack(obj.tupleof[i]); - } - } - } - } - - return this; - } - - - /// ditto - ref Unpacker unpack(T)(ref T object) if (is(Unqual!T == struct)) - { - static if (hasMember!(T, "fromMsgpack")) - { - static if (__traits(compiles, { T t; t.fromMsgpack(this); })) { - object.fromMsgpack(this); - } else { - static assert(0, "Failed to invoke 'fromMsgpack' on type '" ~ Unqual!T.stringof ~ "'"); - } - } else { - if (auto handler = typeid(T) in unpackHandlers) { - (*handler)(this, cast(void*)&object); - return this; - } - - auto length = beginArray(); - if (length == 0) - return this; - - static if (isTuple!T) { - if (length != T.Types.length) - rollback(calculateSize(length)); - - foreach (i, Type; T.Types) - unpack(object.field[i]); - } else { // simple struct - //if (length != object.tupleof.length) - if (length != SerializingMemberNumbers!(T)) - rollback(calculateSize(length)); - - foreach (i, member; object.tupleof) { - static if (isPackedField!(T.tupleof[i])) - unpack(object.tupleof[i]); - } - } - } - - return this; - } - - - /** - * Deserializes the container object and assigns to each argument. - * - * These methods check the length. Do rollback if - * the length of arguments is different from length of deserialized object. - * - * In unpackMap, the number of arguments must be even. - * - * Params: - * objects = the references of object to assign. - * - * Returns: - * self, i.e. for method chaining. - */ - ref Unpacker unpackArray(Types...)(ref Types objects) - { - auto length = beginArray(); - if (length != Types.length) - rollback(calculateSize(length)); - - foreach (i, T; Types) - unpack(objects[i]); - // unpack(objects); // slow :( - - return this; - } - - - /// ditto - ref Unpacker unpackMap(Types...)(ref Types objects) - { - static assert(Types.length % 2 == 0, "The number of arguments must be even"); - - auto length = beginMap(); - if (length != Types.length / 2) - rollback(calculateSize(length)); - - foreach (i, T; Types) - unpack(objects[i]); - - return this; - } - - - /** - * Deserializes the type-information of container. - * - * These methods don't deserialize contents. - * You need to call unpack method to deserialize contents at your own risk. - * ----- - * // serialized data is [1, "Hi!"]; - * int num; - * unpacker.beginArray(2).unpack(num); // num is 1 - * - * // other operation - * - * string str; - * unpacker.unpack(str); // str is "Hi!" - * ----- - * - * Returns: - * the container size. - */ - @safe - size_t beginArray() - { - canRead(Offset, 0); - const header = read(); - size_t length; - - if (0x90 <= header && header <= 0x9f) { - length = header & 0x0f; - } else { - switch (header) { - case Format.ARRAY16: - canRead(ushort.sizeof); - length = load16To!size_t(read(ushort.sizeof)); - break; - case Format.ARRAY32: - canRead(uint.sizeof); - length = load32To!size_t(read(uint.sizeof)); - break; - case Format.NIL: - break; - default: - rollback(); - } - } - - return length; - } - - - /// ditto - @safe - size_t beginMap() - { - canRead(Offset, 0); - const header = read(); - size_t length; - - if (0x80 <= header && header <= 0x8f) { - length = header & 0x0f; - } else { - switch (header) { - case Format.MAP16: - canRead(ushort.sizeof); - length = load16To!size_t(read(ushort.sizeof)); - break; - case Format.MAP32: - canRead(uint.sizeof); - length = load32To!size_t(read(uint.sizeof)); - break; - case Format.NIL: - break; - default: - rollback(); - } - } - - return length; - } - - - /** - * Scans an entire buffer and converts each objects. - * - * This method is used for unpacking record-like objects. - * - * Example: - * ----- - * // serialized data is "[1, 2][3, 4][5, 6][...". - * auto unpacker = Unpacker(serializedData); - * foreach (n, d; &unpacker.scan!(int, int)) // == "foreach (int n, int d; unpacker)" - * writeln(n, d); // 1st loop "1, 2", 2nd loop "3, 4"... - * ----- - */ - int scan(Types...)(scope int delegate(ref Types) dg) - { - return opApply!(Types)(delegate int(ref Types objects) { return dg(objects); }); - } - - - /// ditto - int opApply(Types...)(scope int delegate(ref Types) dg) - { - int result; - - while (used_ - offset_) { - auto length = beginArray(); - if (length != Types.length) - rollback(calculateSize(length)); - - Types objects; - foreach (i, T; Types) - unpack(objects[i]); - - result = dg(objects); - if (result) - return result; - } - - return result; - } - - - private: - /* - * Deserializes nil object and assigns to $(D_PARAM value). - * - * Params: - * value = the reference of value to assign. - * - * Returns: - * self, i.e. for method chaining. - * - * Throws: - * UnpackException when doesn't read from buffer or precision loss occurs and - * MessagePackException when $(D_PARAM T) type doesn't match serialized type. - */ - @safe - ref Unpacker unpackNil(T)(ref T value) - { - canRead(Offset, 0); - const header = read(); - - if (header == Format.NIL) - value = null; - else - rollback(); - - return this; - } - - - /* - * Next object is nil? - * - * Returns: - * true if next object is nil. - */ - @safe - bool checkNil() - { - canRead(Offset, 0); - - return buffer_[offset_] == Format.NIL; - } - - - /* - * Calculates the format size of container length. - */ - size_t calculateSize(bool rawType = false)(in size_t length) - { - static if (rawType) - return length < 32 ? 0 : length < 65536 ? ushort.sizeof : uint.sizeof; - else - return length < 16 ? 0 : length < 65536 ? ushort.sizeof : uint.sizeof; - } - - - /* - * Reading test. - * - * Params: - * size = the size to read. - * offset = the offset to subtract when doesn't read from buffer. - * - * Throws: - * UnpackException when doesn't read from buffer. - */ - @safe - void canRead(in size_t size, in size_t offset = Offset) - { - if (used_ - offset_ < size) { - if (offset) - offset_ -= offset; - - throw new UnpackException("Insufficient buffer"); - } - } - - - /* - * Reads value from buffer and advances offset. - */ - @safe - nothrow ubyte read() - { - return buffer_[offset_++]; - } - - - /* - * Reads value from buffer and advances offset. - */ - @safe - nothrow ubyte[] read(in size_t size) - { - auto result = buffer_[offset_..offset_ + size]; - - offset_ += size; - - return result; - } - - - /* - * Do rollback and throws exception. - */ - @safe - void rollback(in size_t size = 0) - { - offset_ -= size + Offset; - onInvalidType(); - } -} - - -/** - * Register a deserialization handler for $(D_PARAM T) type - * - * Example: - * ----- - * registerUnackHandler!(Foo, fooUnackHandler); - * ----- - */ -void registerUnpackHandler(T, alias Handler)() -{ - Unpacker.registerHandler!(T, Handler); -} - - -unittest -{ - { // unique - mixin DefinePacker; - - Tuple!(bool, bool) result; - Tuple!(bool, bool) test = tuple(true, false); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - - unpacker.unpack(result); - assert(test == result); - } - { // uint * - mixin DefinePacker; - - Tuple!(ubyte, ushort, uint, ulong) result; - Tuple!(ubyte, ushort, uint, ulong) test = tuple(cast(ubyte)ubyte.max, cast(ushort)ushort.max, - cast(uint)uint.max, cast(ulong)ulong.max); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - - unpacker.unpack(result); - assert(test == result); - } - { // int * - mixin DefinePacker; - - Tuple!(byte, short, int, long) result; - Tuple!(byte, short, int, long) test = tuple(cast(byte)byte.min, cast(short)short.min, - cast(int)int.min, cast(long)long.min); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - - unpacker.unpack(result); - assert(test == result); - } - { // floating point - mixin DefinePacker; - - static if (real.sizeof == double.sizeof || !EnableReal) - { - Tuple!(float, double, double) result; - Tuple!(float, double, double) test = tuple(cast(float)float.min_normal, cast(double)double.max, cast(real)double.min_normal); - } - else - { - Tuple!(float, double, real) result; - Tuple!(float, double, real) test = tuple(cast(float)float.min_normal, cast(double)double.max, cast(real)real.min_normal); - } - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - - unpacker.unpack(result); - assert(test == result); - } - { // pointer - mixin DefinePacker; - - Tuple!(ulong, long, double) origin; - Tuple!(ulong, long, double) values = tuple(ulong.max, long.min, double.min_normal); - Tuple!(ulong*, long*, double*) result = tuple(&origin.field[0], &origin.field[1], &origin.field[2]); - Tuple!(ulong*, long*, double*) test = tuple(&values.field[0], &values.field[1], &values.field[2]); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - - unpacker.unpack(result); - foreach (i, v; test.field) - assert(*v == *result.field[i]); - assert(origin == values); - } - { // enum - enum : float { D = 0.5 } - enum E : ulong { U = 100 } - - mixin DefinePacker; - - float f = D, resultF; - E e = E.U, resultE; - - packer.pack(D, e); - - auto unpacker = Unpacker(packer.stream.data); - - unpacker.unpack(resultF, resultE); - assert(f == resultF); - assert(e == resultE); - } - { // container - mixin DefinePacker; - - Tuple!(ulong[], double[uint], string, bool[2], char[2]) test - = tuple([1UL, 2], [3U:4.0, 5:6.0, 7:8.0], "MessagePack is nice!", [true, false], "D!"); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - Tuple!(ulong[], double[uint], string, bool[2], char[2]) result; - - unpacker.unpack(result); - assert(test == result); - } - { // user defined - { - static struct S - { - uint num; - - void toMsgpack(P)(ref P p) const { p.packArray(num); } - void fromMsgpack(ref Unpacker u) - { - assert(u.beginArray() == 1); - u.unpack(num); - } - } - - mixin DefinePacker; S result, test = S(uint.max); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - unpacker.unpack(result); - - assert(test.num == result.num); - } - { - static class C - { - uint num; - - this(uint n) { num = n; } - - void toMsgpack(P)(ref P p) const { p.packArray(num - 1); } - void fromMsgpack(ref Unpacker u) - { - assert(u.beginArray() == 1); - u.unpack(num); - } - } - - mixin DefinePacker; C result, test = new C(ushort.max); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - unpacker.unpack(result, ushort.max); - - assert(test.num == result.num + 1); - } - } - { // simple struct and class - { - static struct Simple - { - uint num; - @nonPacked string str; - } - - static struct Simple2 - { - @nonPacked string str; - uint num; - } - - foreach (Type; TypeTuple!(Simple, Simple2)) { - mixin DefinePacker; - Type result, test; - test.num = uint.max; - test.str = "ignored"; - - packer.pack(test); - auto unpacker = Unpacker(packer.stream.data); - unpacker.unpack(result); - - assert(test.num == result.num); - assert(test.str != result.str); - } - } - - static class SimpleA - { - bool flag = true; - } - - static class SimpleB : SimpleA - { - ubyte type = 100; - } - - static class SimpleC : SimpleB - { - uint num = uint.max; - @nonPacked string str; - } - - static class SimpleC2 : SimpleB - { - @nonPacked string str; - uint num = uint.max; - } - - { // from derived class - foreach (Type; TypeTuple!(SimpleC, SimpleC2)) { - mixin DefinePacker; - Type result, test = new Type(); - test.flag = false; - test.type = 99; - test.num = uint.max / 2; - test.str = "ignored"; - - packer.pack(test); - auto unpacker = Unpacker(packer.stream.data); - unpacker.unpack(result); - - assert(test.flag == result.flag); - assert(test.type == result.type); - assert(test.num == result.num); - assert(test.str != result.str); - } - } - { // from base class - mixin DefinePacker; SimpleC test = new SimpleC(); - - packer.pack(test); - - SimpleB result = new SimpleC(); - auto unpacker = Unpacker(packer.stream.data); - - try { - unpacker.unpack(result); - assert(false); - } catch (Exception e) { } - } - { // https://github.com/msgpack/msgpack-d/issues/16 - static class Issue16 - { - int i; - this(int i) { this.i = i; } - } - - Issue16 c1 = new Issue16(10); - - try { - Issue16 c2 = null; - unpack(pack(c1), c2); - assert(false); - } catch (Exception e) {} - - Issue16 c3 = new Issue16(20); - unpack(pack(c1), c3); - assert(c3.i == c1.i); - } - } - { // variadic - mixin DefinePacker; - - Tuple!(uint, long, double) test = tuple(uint.max, long.min, double.max); - - packer.pack(test); - - auto unpacker = Unpacker(packer.stream.data); - - uint u; long l; double d; - - unpacker.unpackArray(u, l, d); - assert(test == tuple(u, l, d)); - } - { // scan / opApply - ubyte[] data; - mixin DefinePacker; - - foreach (i; 0..2) - packer.pack(tuple(1, 0.5, "Hi!")); - - foreach (n, d, s; &Unpacker(packer.stream.data).scan!(int, double, string)) { - assert(n == 1); - assert(d == 0.5); - assert(s == "Hi!"); - } - } -} - - -// Static resolution routines for Stream deserializer - - -/** - * $(D Value) is a $(D MessagePack) value representation - * - * Example: - * ----- - * auto unpacker = StreamingUnpacker(pack(1, 0.1L) ~ pack(true) ~ pack("foobarbaz")); - * - * foreach (unpacked; unpacker) { - * if (unpacked.type == Value.Type.array) { - * foreach (obj; unpacked) { - * switch (obj.type) { - * case Value.Type.unsigned: writeln(obj.as!(uint)); break; - * case Value.Type.floating: writeln(obj.as!(real)); break; - * defalut: - * throw new Exception("Unknown type"); - * } - * } - * } else { - * if (unpacked.type == Value.Type.boolean) - * writeln(unpacked.as!(bool)); - * else - * writeln("Message: ", unpacked.as!(string)); - * } - * } - * ----- - */ -struct Value -{ - /** - * $(D MessagePack) value type - */ - static enum Type - { - nil, /// nil(null in D) - boolean, /// true, false - unsigned, /// positive fixnum, uint 8, uint 16, uint 32, uint 64 - signed, /// negative fixnum, int 8, int 16, int 32, int 64 - floating, /// float, double, real - array, /// fix array, array 16, array 32 - map, /// fix map, map 16, map 32 - raw /// fix raw, raw 16, raw 32 - } - - /** - * msgpack value representation - */ - static union Via - { - bool boolean; /// corresponding to Type.boolean - ulong uinteger; /// corresponding to Type.unsigned - long integer; /// corresponding to Type.signed - real floating; /// corresponding to Type.floating - Value[] array; /// corresponding to Type.array - Value[Value] map; /// corresponding to Type.map - ubyte[] raw; /// corresponding to Type.raw - } - - - Type type; /// represents value type - Via via; /// represents real value - - - /** - * Constructs a $(D Value) with arguments. - * - * Params: - * value = the real content. - * type = the type of value. - */ - @safe - this(Type type) - { - this.type = type; - } - - @safe - this(typeof(null)) - { - this(Type.nil); - } - - /// ditto - @trusted - this(bool value, Type type = Type.boolean) - { - this(type); - via.boolean = value; - } - - - /// ditto - @trusted - this(ulong value, Type type = Type.unsigned) - { - this(type); - via.uinteger = value; - } - - - /// ditto - @trusted - this(long value, Type type = Type.signed) - { - this(type); - via.integer = value; - } - - - /// ditto - @trusted - this(real value, Type type = Type.floating) - { - this(type); - via.floating = value; - } - - - /// ditto - @trusted - this(Value[] value, Type type = Type.array) - { - this(type); - via.array = value; - } - - - /// ditto - @trusted - this(Value[Value] value, Type type = Type.map) - { - this(type); - via.map = value; - } - - - /// ditto - @trusted - this(ubyte[] value, Type type = Type.raw) - { - this(type); - via.raw = value; - } - - /// This is unsafe overload because using cast internally. - @trusted - this(string value, Type type = Type.raw) - { - this(type); - via.raw = cast(ubyte[])value; - } - - /** - * Converts value to $(D_PARAM T) type. - * - * Returns: - * converted value. - * - * Throws: - * MessagePackException if type is mismatched. - * - * NOTE: - * Current implementation uses cast. - */ - @property @trusted - T as(T)() if (is(Unqual!T == bool)) - { - if (type != Type.boolean) - onCastError(); - - return via.boolean; - } - - - /// ditto - @property @trusted - T as(T)() if (isIntegral!T && !is(Unqual!T == enum)) - { - if (type == Type.unsigned) - return cast(T)via.uinteger; - - if (type == Type.signed) - return cast(T)via.integer; - - onCastError(); - - assert(false); - } - - - /// ditto - @property @trusted - T as(T)() if (isFloatingPoint!T && !is(Unqual!T == enum)) - { - if (type != Type.floating) - onCastError(); - - return cast(T)via.floating; - } - - - /// ditto - @property @trusted - T as(T)() if (is(Unqual!T == enum)) - { - return cast(T)as!(OriginalType!T); - } - - - /// ditto - @property @trusted - T as(T)() if (isArray!T && !is(Unqual!T == enum)) - { - alias typeof(T.init[0]) V; - - if (type == Type.nil) { - static if (isDynamicArray!T) { - return null; - } else { - return T.init; - } - } - - static if (isByte!V || isSomeChar!V) { - if (type != Type.raw) - onCastError(); - - static if (isDynamicArray!T) { - return cast(T)via.raw; - } else { - if (via.raw.length != T.length) - onCastError(); - - return cast(T)(via.raw[0 .. T.length]); - } - } else { - if (type != Type.array) - onCastError(); - - V[] array; - - foreach (elem; via.array) - array ~= elem.as!(V); - - return array; - } - } - - - /// ditto - @property @trusted - T as(T)() if (isAssociativeArray!T) - { - alias typeof(T.init.keys[0]) K; - alias typeof(T.init.values[0]) V; - - if (type == Type.nil) - return null; - - if (type != Type.map) - onCastError(); - - V[K] map; - - foreach (key, value; via.map) - map[key.as!(K)] = value.as!(V); - - return map; - } - - - /** - * Converts to $(D_PARAM T) type. - * - * Calling $(D fromMsgpack) if $(D_KEYWORD class) and $(D_KEYWORD struct) implement $(D fromMsgpack) method. $(D fromMsgpack) signature is: - * ----- - * void fromMsgpack(Value value) - * ----- - * This method assigns converted values to all members of T object if $(D_KEYWORD class) and $(D_KEYWORD struct) don't implement $(D fromMsgpack). - * - * Params: - * args = arguments to class constructor(class only). - * - * Returns: - * converted value. - */ - @property @trusted - T as(T, Args...)(Args args) if (is(Unqual!T == class)) - { - if (type == Type.nil) - return null; - - T object = new T(args); - - static if (hasMember!(T, "fromMsgpack")) - { - static if (__traits(compiles, { T t; t.fromMsgpack(this); })) { - object.fromMsgpack(this); - } else { - static assert(0, "Failed to invoke 'fromMsgpack' on type '" ~ Unqual!T.stringof ~ "'"); - } - } else { - alias SerializingClasses!(T) Classes; - - if (via.array.length != SerializingMemberNumbers!(Classes)) - throw new MessagePackException("The number of deserialized object member is mismatched"); - - size_t offset; - foreach (Class; Classes) { - Class obj = cast(Class)object; - foreach (i, member; obj.tupleof) { - static if (isPackedField!(Class.tupleof[i])) - obj.tupleof[i] = via.array[offset++].as!(typeof(member)); - } - } - } - - return object; - } - - - /// ditto - @property @trusted - T as(T)() if (is(Unqual!T == struct)) - { - T obj; - - static if (hasMember!(T, "fromMsgpack")) - { - static if (__traits(compiles, { T t; t.fromMsgpack(this); })) { - obj.fromMsgpack(this); - } else { - static assert(0, "Failed to invoke 'fromMsgpack' on type '" ~ Unqual!T.stringof ~ "'"); - } - } else { - static if (isTuple!T) { - if (via.array.length != T.Types.length) - throw new MessagePackException("The number of deserialized Tuple element is mismatched"); - - foreach (i, Type; T.Types) - obj.field[i] = via.array[i].as!(Type); - } else { // simple struct - if (via.array.length != SerializingMemberNumbers!T) - throw new MessagePackException("The number of deserialized struct member is mismatched"); - - size_t offset; - foreach (i, member; obj.tupleof) { - static if (isPackedField!(T.tupleof[i])) - obj.tupleof[i] = via.array[offset++].as!(typeof(member)); - } - } - } - - return obj; - } - - - /** - * Special method called by $(D Packer). - * - * Params: - * packer = a MessagePack serializer. - */ - void toMsgpack(Packer)(ref Packer packer) const - { - final switch (type) { - case Type.nil: - packer.packNil(); - break; - case Type.boolean: - packer.pack(via.boolean); - break; - case Type.unsigned: - packer.pack(via.uinteger); - break; - case Type.signed: - packer.pack(via.integer); - break; - case Type.floating: - packer.pack(via.floating); - break; - case Type.raw: - packer.pack(via.raw); - break; - case Type.array: - packer.beginArray(via.array.length); - foreach (elem; via.array) - elem.toMsgpack(packer); - break; - case Type.map: - packer.beginMap(via.map.length); - foreach (key, value; via.map) { - key.toMsgpack(packer); - value.toMsgpack(packer); - } - break; - } - } - - - /** - * Comparison for equality. @trusted for union. - */ - @trusted - bool opEquals(Tdummy = void)(ref const Value other) const - { - if (type != other.type) - return false; - - final switch (other.type) { - case Type.nil: return true; - case Type.boolean: return opEquals(other.via.boolean); - case Type.unsigned: return opEquals(other.via.uinteger); - case Type.signed: return opEquals(other.via.integer); - case Type.floating: return opEquals(other.via.floating); - case Type.raw: return opEquals(other.via.raw); - case Type.array: return opEquals(other.via.array); - case Type.map: return opEquals(other.via.map); - } - } - - - /// ditto - @trusted - bool opEquals(T : bool)(in T other) const - { - if (type != Type.boolean) - return false; - - return via.boolean == other; - } - - - /// ditto - @trusted - bool opEquals(T : ulong)(in T other) const - { - static if (__traits(isUnsigned, T)) { - if (type != Type.unsigned) - return false; - - return via.uinteger == other; - } else { - if (type != Type.signed) - return false; - - return via.integer == other; - } - } - - - /// ditto - @trusted - bool opEquals(T : real)(in T other) const - { - if (type != Type.floating) - return false; - - return via.floating == other; - } - - - /// ditto - @trusted - bool opEquals(T : const Value[])(in T other) const - { - if (type != Type.array) - return false; - - return via.array == other; - } - - - /// ditto - @trusted - bool opEquals(T : const Value[Value])(in T other) const - { - if (type != Type.map) - return false; - - // This comparison is instead of default comparison because 'via.map == other' raises "Access Violation". - foreach (key, value; via.map) { - if (key in other) { - if (other[key] != value) - return false; - } else { - return false; - } - } - - return true; - } - - - /// ditto - @trusted - bool opEquals(T : const(ubyte)[])(in T other) const - { - if (type != Type.raw) - return false; - - return via.raw == other; - } - - - /// ditto - @trusted - bool opEquals(T : string)(in T other) const - { - if (type != Type.raw) - return false; - - return via.raw == cast(ubyte[])other; - } - - @trusted - hash_t toHash() const nothrow - { - static hash_t getHash(T)(T* v) @safe nothrow - { - return typeid(T).getHash(v); - } - - final switch (type) { - case Type.nil: return 0; - case Type.boolean: return getHash(&via.boolean); - case Type.unsigned: return getHash(&via.uinteger); - case Type.signed: return getHash(&via.integer); - case Type.floating: return getHash(&via.floating); - case Type.raw: return getHash(&via.raw); - case Type.array: - hash_t ret; - foreach (elem; via.array) - ret ^= elem.toHash(); - return ret; - case Type.map: - try { - hash_t ret; - foreach (key, value; via.map) { - ret ^= key.toHash(); - ret ^= value.toHash(); - } - return ret; - } catch (Exception) assert(0); - } - } -} - - -unittest -{ - // nil - Value value = Value(null); - Value other = Value(); - - assert(value == other); - assert(value.type == Value.Type.nil); - - // boolean - value = Value(true); - other = Value(false); - - assert(value != other); - assert(value.type == Value.Type.boolean); - assert(value.as!(bool) == true); - assert(other == false); - - try { - auto b = value.as!(uint); - assert(false); - } catch (MessagePackException e) { } - - // unsigned integer - value = Value(10UL); - other = Value(10UL); - - assert(value == other); - assert(value.type == Value.Type.unsigned); - assert(value.as!(uint) == 10); - assert(other == 10UL); - - // signed integer - value = Value(-20L); - other = Value(-10L); - - assert(value != other); - assert(value.type == Value.Type.signed); - assert(value.as!(int) == -20); - assert(other == -10L); - - // enum - enum E : int { F = -20 } - - E e = value.as!(E); - assert(e == E.F); - - // floating point - value = Value(0.1e-10L); - other = Value(0.1e-20L); - - assert(value != other); - assert(value.type == Value.Type.floating); - assert(value.as!(real) == 0.1e-10L); - assert(other == 0.1e-20L); - - // raw - value = Value(cast(ubyte[])[72, 105, 33]); - other = Value(cast(ubyte[])[72, 105, 33]); - - assert(value == other); - assert(value.type == Value.Type.raw); - assert(value.as!(string) == "Hi!"); - assert(value.as!(ubyte[3]) == [72, 105, 33]); - assert(other == cast(ubyte[])[72, 105, 33]); - - // raw with string - value = Value("hello"); - other = Value("hello"); - - assert(value == other); - assert(value.type == Value.Type.raw); - assert(value.as!(string) == "hello"); - - // enum : string - enum EStr : string { elem = "hello" } - - assert(value.as!(EStr) == EStr.elem); - - // array - auto t = Value(cast(ubyte[])[72, 105, 33]); - value = Value([t]); - other = Value([t]); - - assert(value == other); - assert(value.type == Value.Type.array); - assert(value.as!(string[]) == ["Hi!"]); - assert(other == [t]); - - // map - value = Value([Value(1L):Value(2L)]); - other = Value([Value(1L):Value(1L)]); - - assert(value != other); - assert(value.type == Value.Type.map); - assert(value.as!(int[int]) == [1:2]); - assert(other == [Value(1L):Value(1L)]); - - value = Value(10UL); - - // struct - static struct S - { - ulong num; - - void fromMsgpack(Value value) { num = value.via.uinteger; } - } - - S s = value.as!(S); - assert(s.num == 10); - - value = Value([Value(0.5f), Value(cast(ubyte[])[72, 105, 33])]); - - // struct - static struct Simple - { - @nonPacked int era; - double num; - string msg; - } - - Simple simple = value.as!(Simple); - assert(simple.era == int.init); - assert(simple.num == 0.5f); - assert(simple.msg == "Hi!"); - - value = Value(10UL); - - // class - static class C - { - ulong num; - - void fromMsgpack(Value value) { num = value.via.uinteger; } - } - - C c = value.as!(C); - assert(c.num == 10); - - static class SimpleA - { - bool flag = true; - } - - static class SimpleB : SimpleA - { - ubyte type = 100; - } - - static class SimpleC : SimpleB - { - @nonPacked string str; - uint num = uint.max; - } - - value = Value([Value(false), Value(99UL), Value(cast(ulong)(uint.max / 2u))]); - - SimpleC sc = value.as!(SimpleC); - assert(sc.flag == false); - assert(sc.type == 99); - assert(sc.num == uint.max / 2); - assert(sc.str.empty); - - // std.typecons.Tuple - value = Value([Value(true), Value(1UL), Value(cast(ubyte[])"Hi!")]); - - auto tuple = value.as!(Tuple!(bool, uint, string)); - assert(tuple.field[0] == true); - assert(tuple.field[1] == 1u); - assert(tuple.field[2] == "Hi!"); - - /* - * non-MessagePackable object is stopped by static assert - * static struct NonMessagePackable {} - * auto nonMessagePackable = value.as!(NonMessagePackable); - */ -} - - -/** - * $(D Unpacked) is a $(D Range) wrapper for stream deserialization result - */ -struct Unpacked -{ - Value value; /// deserialized value - - alias value this; - - - /** - * Constructs a $(D Unpacked) with argument. - * - * Params: - * value = a deserialized value. - */ - @safe - this(ref Value value) - { - this.value = value; - } - - - /** - * InputRange primitive operation that checks iteration state. - * - * Returns: - * true if there are no more elements to be iterated. - */ - @property @trusted - nothrow bool empty() const // std.array.empty isn't nothrow function - { - return (value.type == Value.Type.array) && !value.via.array.length; - } - - - /** - * Range primitive operation that returns the length of the range. - * - * Returns: - * the number of values. - */ - @property @trusted - size_t length() - { - return value.via.array.length; - } - - - /** - * InputRange primitive operation that returns the currently iterated element. - * - * Returns: - * the deserialized $(D Value). - */ - @property @trusted - ref Value front() - { - return value.via.array.front; - } - - - /** - * InputRange primitive operation that advances the range to its next element. - */ - @trusted - void popFront() - { - value.via.array.popFront(); - } - - /** - * RandomAccessRange primitive operation. - * - * Returns: - * the deserialized $(D Value) at $(D_PARAM n) position. - */ - @trusted - nothrow ref Value opIndex(size_t n) - { - return value.via.array[n]; - } - - /** - * Returns a slice of the range. - * - * Paramas: - * from = the start point of slicing. - * to = the end point of slicing. - * - * Returns: - * the slice of Values. - */ - @trusted - Value[] opSlice(size_t from, size_t to) - { - return value.via.array[from..to]; - } - - /** - * Range primitive operation that returns the snapshot. - * - * Returns: - * the snapshot of this Value. - */ - @property @safe - Unpacked save() - { - return Unpacked(value); - } -} - - -unittest -{ - static assert(isForwardRange!Unpacked); - static assert(hasLength!Unpacked); -} - - -/** - * This $(D StreamingUnpacker) is a $(D MessagePack) streaming deserializer - * - * This implementation enables you to load multiple objects from a stream(like network). - * - * Example: - * ----- - * ... - * auto unpacker = StreamingUnpacker(serializedData); - * ... - * - * // appends new data to buffer if pre execute() call didn't finish deserialization. - * unpacker.feed(newSerializedData); - * - * while (unpacker.execute()) { - * foreach (obj; unpacker.purge()) { - * // do stuff (obj is a Value) - * } - * } - * - * if (unpacker.size) - * throw new Exception("Message is too large"); - * ----- - */ -struct StreamingUnpacker -{ - private: - /* - * Context state of deserialization - */ - enum State - { - HEADER = 0x00, - - BIN8 = 0x04, - BIN16, - BIN32, - - // Floating point, Unsigned, Signed interger (== header & 0x03) - FLOAT = 0x0a, - DOUBLE, - UINT8, - UINT16, - UINT32, - UINT64, - INT8, - INT16, - INT32, - INT64, - - // Container (== header & 0x01) - STR8 = 0x19, - RAW16 = 0x1a, - RAW32, - ARRAY16, - ARRAY36, - MAP16, - MAP32, - RAW, - - // D-specific type - REAL - } - - - /* - * Element type of container - */ - enum ContainerElement - { - ARRAY_ITEM, - MAP_KEY, - MAP_VALUE - } - - - /* - * Internal stack context - */ - static struct Context - { - static struct Container - { - ContainerElement type; // value container type - Value value; // current value - Value key; // for map value - size_t count; // container length - } - - State state; // current state of deserialization - size_t trail; // current deserializing size - size_t top; // current index of stack - Container[] stack; // storing values - } - - Context context_; // stack environment for streaming deserialization - - mixin InternalBuffer; - - - public: - /** - * Constructs a $(D StreamingUnpacker). - * - * Params: - * target = byte buffer to deserialize - * bufferSize = size limit of buffer size - */ - @safe - this(in ubyte[] target, in size_t bufferSize = 8192) - { - initializeBuffer(target, bufferSize); - initializeContext(); - } - - - /** - * Forwards to deserialized object. - * - * Returns: - * the $(D Unpacked) object contains deserialized value. - */ - @property @safe - Unpacked unpacked() - { - return Unpacked(context_.stack[0].value); - } - - - /** - * Clears some states for next deserialization. - */ - @safe - nothrow void clear() - { - initializeContext(); - - parsed_ = 0; - } - - - /** - * Convenient method for unpacking and clearing states. - * - * Example: - * ----- - * foreach (obj; unpacker.purge()) { - * // do stuff - * } - * ----- - * is equivalent to - * ----- - * foreach (obj; unpacker.unpacked) { - * // do stuff - * } - * unpacker.clear(); - * ----- - * - * Returns: - * the $(D Unpacked) object contains deserialized value. - */ - @safe - Unpacked purge() - { - auto result = Unpacked(context_.stack[0].value); - - clear(); - - return result; - } - - - /** - * Executes deserialization. - * - * Returns: - * true if deserialization has been completed, otherwise false. - * - * Throws: - * $(D UnpackException) when parse error occurs. - */ - bool execute() - { - /* - * Current implementation is very dirty(goto! goto!! goto!!!). - * This Complexity for performance(avoid function call). - */ - - bool ret; - size_t cur = offset_; - Value obj; - - // restores before state - auto state = context_.state; - auto trail = context_.trail; - auto top = context_.top; - auto stack = &context_.stack; - - /* - * Helper for container deserialization - */ - bool startContainer(string Type)(ContainerElement type, size_t length) - { - mixin("callback" ~ Type ~ "((*stack)[top].value, length);"); - - if (length == 0) - return false; - - (*stack)[top].type = type; - (*stack)[top].count = length; - (*stack).length = ++top + 1; - - return true; - } - - // non-deserialized data is nothing - if (used_ - offset_ == 0) - goto Labort; - - do { - Lstart: - if (state == State.HEADER) { - const header = buffer_[cur]; - - if (0x00 <= header && header <= 0x7f) { // positive - callbackUInt(obj, header); - goto Lpush; - } else if (0xe0 <= header && header <= 0xff) { // negative - callbackInt(obj, cast(byte)header); - goto Lpush; - } else if (0xa0 <= header && header <= 0xbf) { // fix raw - trail = header & 0x1f; - state = State.RAW; - cur++; - continue; - } else if (0x90 <= header && header <= 0x9f) { // fix array - if (!startContainer!"Array"(ContainerElement.ARRAY_ITEM, header & 0x0f)) - goto Lpush; - cur++; - continue; - } else if (0x80 <= header && header <= 0x8f) { // fix map - if (!startContainer!"Map"(ContainerElement.MAP_KEY, header & 0x0f)) - goto Lpush; - cur++; - continue; - } else { - switch (header) { - case Format.UINT8, Format.UINT16, Format.UINT32, Format.UINT64, - Format.INT8, Format.INT16, Format.INT32, Format.INT64, - Format.FLOAT, Format.DOUBLE: - trail = 1 << (header & 0x03); // computes object size - state = cast(State)(header & 0x1f); - break; - case Format.REAL: - trail = RealSize; - state = State.REAL; - break; - case Format.ARRAY16, Format.ARRAY32, - Format.MAP16, Format.MAP32: - trail = 2 << (header & 0x01); // computes container size - state = cast(State)(header & 0x1f); - break; - // raw will become str format in new spec - case Format.STR8: - case Format.RAW16: // will be STR16 - case Format.RAW32: // will be STR32 - trail = 1 << ((header & 0x03) - 1); // computes container size - state = cast(State)(header & 0x1f); - break; - case Format.BIN8, Format.BIN16, Format.BIN32: - trail = 1 << (header & 0x03); // computes container size - state = cast(State)(header & 0x1f); - break; - case Format.NIL: - callbackNil(obj); - goto Lpush; - case Format.TRUE: - callbackBool(obj, true); - goto Lpush; - case Format.FALSE: - callbackBool(obj, false); - goto Lpush; - default: - throw new UnpackException("Unknown type"); - } - - cur++; - goto Lstart; - } - } else { - // data lack for deserialization - if (used_ - cur < trail) - goto Labort; - - const base = cur; cur += trail - 1; // fix current position - - final switch (state) { - case State.FLOAT: - _f temp; - - temp.i = load32To!uint(buffer_[base..base + trail]); - callbackFloat(obj, temp.f); - goto Lpush; - case State.DOUBLE: - _d temp; - - temp.i = load64To!ulong(buffer_[base..base + trail]); - callbackFloat(obj, temp.f); - goto Lpush; - case State.REAL: - const expb = base + ulong.sizeof; - - version (NonX86) - { - CustomFloat!80 temp; - - const frac = load64To!ulong (buffer_[base..expb]); - const exp = load16To!ushort(buffer_[expb..expb + ushort.sizeof]); - - temp.significand = frac; - temp.exponent = exp & 0x7fff; - temp.sign = exp & 0x8000 ? true : false; - - // NOTE: temp.get!real is inf on non-x86 when deserialized value is larger than double.max. - callbackFloat(obj, temp.get!real); - } - else - { - _r temp; - - temp.fraction = load64To!(typeof(temp.fraction))(buffer_[base..expb]); - temp.exponent = load16To!(typeof(temp.exponent))(buffer_[expb..expb + temp.exponent.sizeof]); - - callbackFloat(obj, temp.f); - } - - goto Lpush; - case State.UINT8: - callbackUInt(obj, buffer_[base]); - goto Lpush; - case State.UINT16: - callbackUInt(obj, load16To!ulong(buffer_[base..base + trail])); - goto Lpush; - case State.UINT32: - callbackUInt(obj, load32To!ulong(buffer_[base..base + trail])); - goto Lpush; - case State.UINT64: - callbackUInt(obj, load64To!ulong(buffer_[base..base + trail])); - goto Lpush; - case State.INT8: - callbackInt(obj, cast(byte)buffer_[base]); - goto Lpush; - case State.INT16: - callbackInt(obj, load16To!long(buffer_[base..base + trail])); - goto Lpush; - case State.INT32: - callbackInt(obj, load32To!long(buffer_[base..base + trail])); - goto Lpush; - case State.INT64: - callbackInt(obj, load64To!long(buffer_[base..base + trail])); - goto Lpush; - case State.RAW: Lraw: - hasRaw_ = true; - callbackRaw(obj, buffer_[base..base + trail]); - goto Lpush; - case State.STR8, State.BIN8: - trail = buffer_[base]; - if (trail == 0) - goto Lraw; - state = State.RAW; - cur++; - goto Lstart; - case State.RAW16, State.BIN16: - trail = load16To!size_t(buffer_[base..base + trail]); - if (trail == 0) - goto Lraw; - state = State.RAW; - cur++; - goto Lstart; - case State.RAW32, State.BIN32: - trail = load32To!size_t(buffer_[base..base + trail]); - if (trail == 0) - goto Lraw; - state = State.RAW; - cur++; - goto Lstart; - case State.ARRAY16: - if (!startContainer!"Array"(ContainerElement.ARRAY_ITEM, - load16To!size_t(buffer_[base..base + trail]))) - goto Lpush; - state = State.HEADER; - cur++; - continue; - case State.ARRAY36: - if (!startContainer!"Array"(ContainerElement.ARRAY_ITEM, - load32To!size_t(buffer_[base..base + trail]))) - goto Lpush; - state = State.HEADER; - cur++; - continue; - case State.MAP16: - if (!startContainer!"Map"(ContainerElement.MAP_KEY, - load16To!size_t(buffer_[base..base + trail]))) - goto Lpush; - state = State.HEADER; - cur++; - continue; - case State.MAP32: - if (!startContainer!"Map"(ContainerElement.MAP_KEY, - load32To!size_t(buffer_[base..base + trail]))) - goto Lpush; - state = State.HEADER; - cur++; - continue; - case State.HEADER: - break; - } - } - - Lpush: - if (top == 0) - goto Lfinish; - - auto container = &(*stack)[top - 1]; - - final switch (container.type) { - case ContainerElement.ARRAY_ITEM: - container.value.via.array ~= obj; - if (--container.count == 0) { - obj = container.value; - top--; - goto Lpush; - } - break; - case ContainerElement.MAP_KEY: - container.key = obj; - container.type = ContainerElement.MAP_VALUE; - break; - case ContainerElement.MAP_VALUE: - container.value.via.map[container.key] = obj; - if (--container.count == 0) { - obj = container.value; - top--; - goto Lpush; - } - container.type = ContainerElement.MAP_KEY; - } - - state = State.HEADER; - cur++; - } while (cur < used_); - - goto Labort; - - Lfinish: - (*stack)[0].value = obj; - ret = true; - cur++; - goto Lend; - - Labort: - ret = false; - - Lend: - context_.state = state; - context_.trail = trail; - context_.top = top; - parsed_ += cur - offset_; - offset_ = cur; - - return ret; - } - - - /** - * supports foreach. One loop provides $(D Unpacked) object contains execute() result. - * This is convenient in case that $(D MessagePack) values are continuous. - */ - int opApply(scope int delegate(ref Unpacked) dg) - { - int result; - - while (execute()) { - auto unpackedResult = Unpacked(context_.stack[0].value); - result = dg(unpackedResult); - if (result) - break; - - clear(); - } - - return result; - } - - - private: - /* - * initializes internal stack environment. - */ - @safe - nothrow void initializeContext() - { - context_.state = State.HEADER; - context_.trail = 0; - context_.top = 0; - context_.stack.length = 1; - } -} - - -unittest -{ - // serialize - mixin DefinePacker; - - packer.packArray(null, true, 1, -2, "Hi!", [1], [1:1], double.max); - - // deserialize - auto unpacker = StreamingUnpacker(packer.stream.data); unpacker.execute(); - auto unpacked = unpacker.purge(); - - // Range test - foreach (unused; 0..2) { - uint i; - - foreach (obj; unpacked) - i++; - - assert(i == unpacked.via.array.length); - } - - auto result = unpacked.via.array; - - assert(result[0].type == Value.Type.nil); - assert(result[1].via.boolean == true); - assert(result[2].via.uinteger == 1); - assert(result[3].via.integer == -2); - assert(result[4].via.raw == [72, 105, 33]); - assert(result[5].as!(int[]) == [1]); - assert(result[6].as!(int[int]) == [1:1]); - assert(result[7].as!(double) == double.max); -} - - -private: - - -/* - * Sets value type and value. - * - * Params: - * value = the value to set - * number = the content to set - */ -@trusted -void callbackUInt(ref Value value, ulong number) -{ - value.type = Value.Type.unsigned; - value.via.uinteger = number; -} - - -/// ditto -@trusted -void callbackInt(ref Value value, long number) -{ - value.type = Value.Type.signed; - value.via.integer = number; -} - - -/// ditto -@trusted -void callbackFloat(ref Value value, real number) -{ - value.type = Value.Type.floating; - value.via.floating = number; -} - - -/// ditto -@trusted -void callbackRaw(ref Value value, ubyte[] raw) -{ - value.type = Value.Type.raw; - value.via.raw = raw; -} - - -/// ditto -@trusted -void callbackArray(ref Value value, size_t length) -{ - value.type = Value.Type.array; - value.via.array.length = 0; - value.via.array.reserve(length); -} - - -/// ditto -@trusted -void callbackMap(ref Value value, lazy size_t length) -{ - value.type = Value.Type.map; - value.via.map = null; // clears previous result avoiding 'Access Violation' -} - - -/// ditto -@safe -void callbackNil(ref Value value) -{ - value.type = Value.Type.nil; -} - - -/// ditto -@trusted -void callbackBool(ref Value value, bool boolean) -{ - value.type = Value.Type.boolean; - value.via.boolean = boolean; -} - - -unittest -{ - Value value; - - // Unsigned integer - callbackUInt(value, uint.max); - assert(value.type == Value.Type.unsigned); - assert(value.via.uinteger == uint.max); - - // Signed integer - callbackInt(value, int.min); - assert(value.type == Value.Type.signed); - assert(value.via.integer == int.min); - - // Floating point - callbackFloat(value, real.max); - assert(value.type == Value.Type.floating); - assert(value.via.floating == real.max); - - // Raw - callbackRaw(value, cast(ubyte[])[1]); - assert(value.type == Value.Type.raw); - assert(value.via.raw == cast(ubyte[])[1]); - - // Array - Value[] array; array.reserve(16); - - callbackArray(value, 16); - assert(value.type == Value.Type.array); - assert(value.via.array.capacity == array.capacity); - - // Map - Value[Value] map; - - callbackMap(value, 16); - assert(value.type == Value.Type.map); - assert(value.via.map == null); - - // NIL - callbackNil(value); - assert(value.type == Value.Type.nil); - - // Bool - callbackBool(value, true); - assert(value.type == Value.Type.boolean); - assert(value.via.boolean == true); -} - - -private: - - -/* - * A callback for type-mismatched error in cast conversion. - */ -@safe -pure void onCastError() -{ - throw new MessagePackException("Attempt to cast with another type"); -} - - -/* - * A callback for type-mismatched error in deserialization process. - */ -@safe -pure void onInvalidType() -{ - throw new MessagePackException("Attempt to unpack with non-compatible type"); -} - - -public: - - -/* - * Handy helper for creating MessagePackable object. - * - * toMsgpack / fromMsgpack are special methods for serialization / deserialization. - * This template provides those methods to struct/class. - * - * Example: - * ----- - * struct S - * { - * int num; string str; - * - * // http://d.puremagic.com/issues/show_bug.cgi?id = 1099 - * mixin MessagePackable; // all members - * // mixin MessagePackable!("num"); // num only - * } - * ----- - * - * Defines those methods manually if you treat complex data-structure. - */ -mixin template MessagePackable(Members...) -{ - static if (Members.length == 0) { - /** - * Serializes members using $(D_PARAM packer). - * - * Params: - * packer = the serializer to pack. - */ - void toMsgpack(Packer)(ref Packer packer, bool withFieldName = false) const - { - if (withFieldName) { - packer.beginMap(this.tupleof.length); - foreach (i, member; this.tupleof) { - packer.pack(getFieldName!(typeof(this), i)); - packer.pack(member); - } - } else { - packer.beginArray(this.tupleof.length); - foreach (member; this.tupleof) - packer.pack(member); - } - } - - - /** - * Deserializes $(D MessagePack) object to members using Value. - * - * Params: - * value = the MessagePack value to unpack. - * - * Throws: - * MessagePackException if $(D_PARAM value) is not an Array type. - */ - void fromMsgpack(Value value) - { - // enables if std.contracts.enforce is moved to object_.d - // enforceEx!MessagePackException(value.type == Value.Type.array, "Value must be Array type"); - if (value.type != Value.Type.array) - throw new MessagePackException("Value must be an Array type"); - if (value.via.array.length != this.tupleof.length) - throw new MessagePackException("The size of deserialized value is mismatched"); - - foreach (i, member; this.tupleof) - this.tupleof[i] = value.via.array[i].as!(typeof(member)); - } - - - /** - * Deserializes $(D MessagePack) object to members using direct-conversion deserializer. - * - * Params: - * value = the reference to direct-conversion deserializer. - * - * Throws: - * MessagePackException if the size of deserialized value is mismatched. - */ - void fromMsgpack(ref Unpacker unpacker) - { - auto length = unpacker.beginArray(); - if (length != this.tupleof.length) - throw new MessagePackException("The size of deserialized value is mismatched"); - - foreach (i, member; this.tupleof) - unpacker.unpack(this.tupleof[i]); - } - } else { - /** - * Member selecting version of toMsgpack. - */ - void toMsgpack(Packer)(ref Packer packer, bool withFieldName = false) const - { - if (withFieldName) { - packer.beginMap(Members.length); - foreach (member; Members) { - packer.pack(member); - packer.pack(mixin(member)); - } - } else { - packer.beginArray(Members.length); - foreach (member; Members) - packer.pack(mixin(member)); - } - } - - - /** - * Member selecting version of fromMsgpack for Value. - */ - void fromMsgpack(Value value) - { - if (value.type != Value.Type.array) - throw new MessagePackException("Value must be an Array type"); - if (value.via.array.length != Members.length) - throw new MessagePackException("The size of deserialized value is mismatched"); - - foreach (i, member; Members) - mixin(member ~ "= value.via.array[i].as!(typeof(" ~ member ~ "));"); - } - - - /** - * Member selecting version of fromMsgpack for direct-converion deserializer. - */ - void fromMsgpack(ref Unpacker unpacker) - { - auto length = unpacker.beginArray(); - if (length != Members.length) - throw new MessagePackException("The size of deserialized value is mismatched"); - - foreach (member; Members) - unpacker.unpack(mixin(member)); - } - } -} - - -unittest -{ - { // all members - /* - * Comment out because "src/msgpack.d(4048): Error: struct msgpack.__unittest16.S no size yet for forward reference" occurs - */ - static struct S - { - uint num; string str; - mixin MessagePackable; - } - - mixin DefinePacker; - - S orig = S(10, "Hi!"); orig.toMsgpack(packer); - - { // stream - auto unpacker = StreamingUnpacker(packer.stream.data); unpacker.execute(); - - S result; result.fromMsgpack(unpacker.unpacked); - - assert(result.num == 10); - assert(result.str == "Hi!"); - } - { // direct conversion - auto unpacker = Unpacker(packer.stream.data); - - S result; unpacker.unpack(result); - - assert(result.num == 10); - assert(result.str == "Hi!"); - } - } - { // member select - static class C - { - uint num; string str; - - this() {} - this(uint n, string s) { num = n; str = s; } - - mixin MessagePackable!("num"); - } - - mixin DefinePacker; - - C orig = new C(10, "Hi!"); orig.toMsgpack(packer); - - { // stream - auto unpacker = StreamingUnpacker(packer.stream.data); unpacker.execute(); - - C result = new C; result.fromMsgpack(unpacker.unpacked); - - assert(result.num == 10); - } - { // direct conversion - auto unpacker = Unpacker(packer.stream.data); - - C result; unpacker.unpack(result); - - assert(result.num == 10); - } - } -} - - -private: - - -// Common and system dependent operations - - -/* - * MessagePack type-information format - * - * See_Also: - * $(LINK2 http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec, MessagePack Specificaton) - */ -enum Format : ubyte -{ - // unsinged integer - UINT8 = 0xcc, // ubyte - UINT16 = 0xcd, // ushort - UINT32 = 0xce, // uint - UINT64 = 0xcf, // ulong - - // signed integer - INT8 = 0xd0, // byte - INT16 = 0xd1, // short - INT32 = 0xd2, // int - INT64 = 0xd3, // long - - // floating point - FLOAT = 0xca, // float - DOUBLE = 0xcb, // double - - // raw byte - RAW = 0xa0, - RAW16 = 0xda, - RAW32 = 0xdb, - - // bin type - BIN8 = 0xc4, - BIN16 = 0xc5, - BIN32 = 0xc6, - - // str type - STR8 = 0xd9, - //STR16 = 0xda, - //STR32 = 0xdb, - - // array - ARRAY = 0x90, - ARRAY16 = 0xdc, - ARRAY32 = 0xdd, - - // map - MAP = 0x80, - MAP16 = 0xde, - MAP32 = 0xdf, - - // other - NIL = 0xc0, // null - TRUE = 0xc3, - FALSE = 0xc2, - - // real (This format is D only!) - REAL = 0xd4 -} - - -/* - * For float type serialization / deserialization - */ -union _f -{ - float f; - uint i; -} - - -/* - * For double type serialization / deserialization - */ -union _d -{ - double f; - ulong i; -} - - -/* - * For real type serialization / deserialization - * - * 80-bit real is padded to 12 bytes(Linux) and 16 bytes(Mac). - * http://lists.puremagic.com/pipermail/digitalmars-d/2010-June/077394.html - */ -union _r -{ - real f; - - struct - { - ulong fraction; - ushort exponent; // includes sign - } -} - -enum RealSize = 10; // Real size is 80bit - - -/* - * Detects whether $(D_PARAM T) is a built-in byte type. - */ -template isByte(T) -{ - enum isByte = staticIndexOf!(Unqual!T, byte, ubyte) >= 0; -} - - -unittest -{ - static assert(isByte!(byte)); - static assert(isByte!(const(byte))); - static assert(isByte!(ubyte)); - static assert(isByte!(immutable(ubyte))); - static assert(!isByte!(short)); - static assert(!isByte!(char)); - static assert(!isByte!(string)); -} - - -/* - * Gets asterisk string from pointer type - */ -template AsteriskOf(T) -{ - static if (is(T P == U*, U)) - enum AsteriskOf = "*" ~ AsteriskOf!U; - else - enum AsteriskOf = ""; -} - -/** - * Get the number of member to serialize. - */ -template SerializingMemberNumbers(Classes...) -{ - static if (Classes.length == 0) - enum SerializingMemberNumbers = 0; - else - enum SerializingMemberNumbers = Filter!(isPackedField, Classes[0].tupleof).length + SerializingMemberNumbers!(Classes[1..$]); -} - -/** - * Get derived classes with serialization-order - */ -template SerializingClasses(T) -{ - // There is no information in Object type. Currently disable Object serialization. - static if (is(T == Object)) - static assert(false, "Object type serialization doesn't support yet. Please define toMsgpack/fromMsgpack and use cast"); - else - alias TypeTuple!(Reverse!(Erase!(Object, BaseClassesTuple!(T))), T) SerializingClasses; -} - - -/** - * Get a field name of class or struct. - */ -template getFieldName(Type, size_t i) -{ - import std.conv : text; - - static assert((is(Unqual!Type == class) || is(Unqual!Type == struct)), "Type must be class or struct: type = " ~ Type.stringof); - static assert(i < Type.tupleof.length, text(Type.stringof, " has ", Type.tupleof.length, " attributes: given index = ", i)); - - enum getFieldName = __traits(identifier, Type.tupleof[i]); -} - - -version (LittleEndian) -{ - /* - * Converts $(value) to different Endian. - * - * Params: - * value = the LittleEndian value to convert. - * - * Returns: - * the converted value. - */ - @trusted - ushort convertEndianTo(size_t Bit, T)(in T value) if (Bit == 16) - { - return ntohs(cast(ushort)value); - } - - - // ditto - @trusted - uint convertEndianTo(size_t Bit, T)(in T value) if (Bit == 32) - { - return ntohl(cast(uint)value); - } - - - // ditto - @trusted - ulong convertEndianTo(size_t Bit, T)(in T value) if (Bit == 64) - { - // dmd has convert function? - return ((((cast(ulong)value) << 56) & 0xff00000000000000UL) | - (((cast(ulong)value) << 40) & 0x00ff000000000000UL) | - (((cast(ulong)value) << 24) & 0x0000ff0000000000UL) | - (((cast(ulong)value) << 8) & 0x000000ff00000000UL) | - (((cast(ulong)value) >> 8) & 0x00000000ff000000UL) | - (((cast(ulong)value) >> 24) & 0x0000000000ff0000UL) | - (((cast(ulong)value) >> 40) & 0x000000000000ff00UL) | - (((cast(ulong)value) >> 56) & 0x00000000000000ffUL)); - } - - - unittest - { - assert(convertEndianTo!16(0x0123) == 0x2301); - assert(convertEndianTo!32(0x01234567) == 0x67452301); - assert(convertEndianTo!64(0x0123456789abcdef) == 0xefcdab8967452301); - } - - - /* - * Comapatible for BigEndian environment. - */ - ubyte take8from(size_t bit = 8, T)(T value) - { - static if (bit == 8 || bit == 16 || bit == 32 || bit == 64) - return (cast(ubyte*)&value)[0]; - else - static assert(false, bit.stringof ~ " is not support bit width."); - } - - - unittest - { - foreach (Integer; TypeTuple!(ubyte, ushort, uint, ulong)) { - assert(take8from!8 (cast(Integer)0x01) == 0x01); - assert(take8from!16(cast(Integer)0x0123) == 0x23); - assert(take8from!32(cast(Integer)0x01234567) == 0x67); - assert(take8from!64(cast(Integer)0x0123456789abcdef) == 0xef); - } - } -} -else -{ - /* - * Comapatible for LittleEndian environment. - */ - @safe - ushort convertEndianTo(size_t Bit, T)(in T value) if (Bit == 16) - { - return cast(ushort)value; - } - - - // ditto - @safe - uint convertEndianTo(size_t Bit, T)(in T value) if (Bit == 32) - { - return cast(uint)value; - } - - - // ditto - @safe - ulong convertEndianTo(size_t Bit, T)(in T value) if (Bit == 64) - { - return cast(ulong)value; - } - - - unittest - { - assert(convertEndianTo!16(0x0123) == 0x0123); - assert(convertEndianTo!32(0x01234567) == 0x01234567); - assert(convertEndianTo!64(0x0123456789) == 0x0123456789); - } - - - /* - * Takes 8bit from $(D_PARAM value) - * - * Params: - * value = the content to take. - * - * Returns: - * the 8bit value corresponding $(D_PARAM bit) width. - */ - ubyte take8from(size_t bit = 8, T)(T value) - { - static if (bit == 8) - return (cast(ubyte*)&value)[0]; - else static if (bit == 16) - return (cast(ubyte*)&value)[1]; - else static if (bit == 32) - return (cast(ubyte*)&value)[3]; - else static if (bit == 64) - return (cast(ubyte*)&value)[7]; - else - static assert(false, bit.stringof ~ " is not support bit width."); - } - - - unittest - { - foreach (Integer; TypeTuple!(ubyte, ushort, uint, ulong)) { - assert(take8from!8 (cast(Integer)0x01) == 0x01); - assert(take8from!16(cast(Integer)0x0123) == 0x23); - assert(take8from!32(cast(Integer)0x01234567) == 0x67); - assert(take8from!64(cast(Integer)0x0123456789abcdef) == 0xef); - } - } -} - - -/* - * Loads $(D_PARAM T) type value from $(D_PARAM buffer). - * - * Params: - * buffer = the serialized contents. - * - * Returns: - * the Endian-converted value. - */ -T load16To(T)(ubyte[] buffer) -{ - return cast(T)(convertEndianTo!16(*cast(ushort*)buffer.ptr)); -} - - -// ditto -T load32To(T)(ubyte[] buffer) -{ - return cast(T)(convertEndianTo!32(*cast(uint*)buffer.ptr)); -} - - -// ditto -T load64To(T)(ubyte[] buffer) -{ - return cast(T)(convertEndianTo!64(*cast(ulong*)buffer.ptr)); -} diff --git a/BioD/contrib/msgpack-d b/BioD/contrib/msgpack-d new file mode 160000 index 00000000..26ef07e1 --- /dev/null +++ b/BioD/contrib/msgpack-d @@ -0,0 +1 @@ +Subproject commit 26ef07e16023483ad93e3f86374b19d0e541c924 diff --git a/Makefile b/Makefile index 529c4d7e..ed312d58 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ # # Debug version # -# make debug +# make debug VERBOSE=1 # D_COMPILER=ldc2 diff --git a/Makefile.guix b/Makefile.guix index 15a10612..663918ec 100644 --- a/Makefile.guix +++ b/Makefile.guix @@ -9,7 +9,7 @@ D_COMPILER=ldc2 CC=gcc CPP=gcc -BIOD_PATH=./BioD +BIOD_PATH=./BioD:./BioD/contrib/msgpack-d/src # DLIBS = $(LIBRARY_PATH)/libphobos2-ldc.a $(LIBRARY_PATH)/libdruntime-ldc.a # DLIBS_DEBUG = $(LIBRARY_PATH)/libphobos2-ldc-debug.a $(LIBRARY_PATH)/libdruntime-ldc-debug.a @@ -24,7 +24,7 @@ RPATH = -L--rpath=$(LIBRARY_PATH) LIBS = -L-L$(LDC_LIB_PATH) -L-lrt -L-lpthread -L-lm -L-lz -L-llz4 LIBS_STATIC = $(DLIBS) -L-lz -L-llz4 -L-lphobos2-ldc -L-ldruntime-ldc -L-lldc-jit-rt -SRC = $(wildcard main.d utils/*.d thirdparty/*.d) $(wildcard BioD/contrib/undead/*.d BioD/contrib/undead/*/*.d) $(wildcard BioD/bio/*/*.d BioD/bio/*/*/*.d BioD/bio/*/*/*/*.d BioD/bio/*/*/*/*/*.d BioD/bio/*/*/*/*/*/*/*.d) $(wildcard sambamba/*.d sambamba/*/*.d sambamba/*/*/*.d) +SRC = $(wildcard main.d utils/*.d thirdparty/*.d) $(wildcard BioD/contrib/undead/*.d BioD/contrib/undead/*/*.d) $(wildcard BioD/bio/*/*.d BioD/bio/*/*/*.d BioD/bio/*/*/*/*.d BioD/bio/*/*/*/*/*.d BioD/bio/*/*/*/*/*/*/*.d BioD/contrib/msgpack-d/src/msgpack/*.d) $(wildcard sambamba/*.d sambamba/*/*.d sambamba/*/*/*.d) OBJ = $(SRC:.d=.o) OUT = bin/sambamba-$(shell cat VERSION) static: OUT += -static @@ -60,7 +60,7 @@ all: debug # sambamba/subsample.d: bio2/pileup.d utils/ldc_version_info_.d: - python3 ./gen_ldc_version_info.py $(shell which ldmd2) > utils/ldc_version_info_.d + python ./gen_ldc_version_info.py $(shell which ldmd2) > utils/ldc_version_info_.d cat utils/ldc_version_info_.d ldc_version_info: utils/ldc_version_info_.d diff --git a/guix.scm b/guix.scm new file mode 100644 index 00000000..15f10ede --- /dev/null +++ b/guix.scm @@ -0,0 +1,101 @@ +;; To get a development container (inside emacs shell will work) +;; +;; guix shell -C -D -f guix.scm -- bash --init-file <(echo "ln -s /bin/sh /bin/bash") +;; +;; and build +;; +;; find -name CMakeCache.txt|xargs rm -v +;; cd build +;; cmake -DCMAKE_BUILD_TYPE=Debug .. +;; cmake --build . --verbose -- -j 14 && ctest . --verbose +;; +;; For the tests you may need /usr/bin/env. In a container create it with +;; +;; mkdir -p /usr/bin ; ln -s $GUIX_ENVIRONMENT/bin/env /usr/bin/env +;; + +(use-modules + (srfi srfi-1) + (ice-9 popen) + (ice-9 rdelim) + ((guix licenses) #:prefix license:) + (guix packages) + (guix utils) + (guix download) + (guix git-download) + (guix build-system gnu) + (guix gexp) + (gnu packages base) + (gnu packages bioinformatics) ; for samtools in sambamba + (gnu packages compression) + (gnu packages curl) + (gnu packages dlang) + (gnu packages gcc) + (gnu packages perl) + (gnu packages python) + (gnu packages ruby) + (gnu packages tls) + (gnu packages version-control) + ) + +#! + (use-modules + (guix build-system cmake) + (guix utils) + (gnu packages base) + (gnu packages compression) + (gnu packages build-tools) + (gnu packages commencement) ; gcc-toolchain + (gnu packages curl) + (gnu packages datastructures) + (gnu packages gdb) + (gnu packages gcc) + (gnu packages jemalloc) + (gnu packages libffi) + (gnu packages mpi) + (gnu packages python) + (gnu packages python-xyz) + (gnu packages pkg-config) + (gnu packages tls) +) + +!# + +(define %source-dir (dirname (current-filename))) + +(define %git-commit + (read-string (open-pipe "git show HEAD | head -1 | cut -d ' ' -f 2" OPEN_READ))) + +(define-public sambamba-git + (package + (name "sambamba-git") + (version (git-version "1.0.0" "HEAD" %git-commit)) + (source (local-file %source-dir #:recursive? #t)) + (build-system gnu-build-system) + (outputs '("out" ; disable all checks for speed + "debug")) + (inputs + `(("samtools" ,samtools) ; for pileup + ("bcftools" ,bcftools) ; for pileup + ("lz4" ,lz4) + ("zlib" ,zlib) + )) + (native-inputs + `(("ldc" ,ldc) + ("coreutils" ,coreutils) ; for env + ("perl" ,perl) ; Needed for building htslib + ("ruby" ,ruby) ; Needed for building htslib + ("python" ,python-2) ; Needed for building htslib and sambamba + ("gcc" ,gcc) + ("which" ,which))) + (home-page "https://github.com/lomereiter/sambamba") + (synopsis "Fast tool for working with SAM, BAM and CRAM files written in D.") + (description + "Sambamba is a high performance modern robust and fast +tool (and library), written in the D programming language, for working +with SAM, BAM and CRAM files. Current parallelised functionality is +an important subset of samtools functionality, including view, index, +sort, markdup, and depth.") + (license license:gpl2+))) + +sambamba-git diff --git a/meson.build b/meson.build index 5d293f42..5b575728 100644 --- a/meson.build +++ b/meson.build @@ -120,7 +120,6 @@ biod_src = [ 'BioD/bio/std/hts/sam/utils/recordparser.d', 'BioD/bio/std/hts/snpcallers/maq.d', 'BioD/bio/std/hts/snpcallers/simple.d', - 'BioD/bio/std/hts/thirdparty/msgpack.d', 'BioD/bio/std/hts/utils/array.d', 'BioD/bio/std/hts/utils/graph.d', 'BioD/bio/std/hts/utils/samheadermerger.d', diff --git a/sambamba/utils/view/alignmentrangeprocessor.d b/sambamba/utils/view/alignmentrangeprocessor.d index 54e5d93b..ab9f57b0 100644 --- a/sambamba/utils/view/alignmentrangeprocessor.d +++ b/sambamba/utils/view/alignmentrangeprocessor.d @@ -22,7 +22,7 @@ module sambamba.utils.view.alignmentrangeprocessor; import bio.std.hts.bam.reader; import bio.std.hts.bam.read; import bio.std.hts.bam.writer; -import bio.std.hts.thirdparty.msgpack; +import msgpack; import bio.core.utils.outbuffer; import bio.core.utils.range; import sambamba.utils.common.readstorage; diff --git a/sambamba/utils/view/headerserializer.d b/sambamba/utils/view/headerserializer.d index d97b2c86..1d8dae08 100644 --- a/sambamba/utils/view/headerserializer.d +++ b/sambamba/utils/view/headerserializer.d @@ -23,7 +23,7 @@ import bio.std.hts.sam.header; import std.conv; import std.array; import std.stdio; -import bio.std.hts.thirdparty.msgpack; +import msgpack; private { abstract class IHeaderSerializer { @@ -69,7 +69,7 @@ final class HeaderSerializer { case "msgpack": _serializer = new HeaderMsgpackSerializer(f); break; default: - throw new Exception("unknown format for serialization: '" ~ format ~ + throw new Exception("unknown format for serialization: '" ~ format ~ "' (expected 'sam', 'bam', 'json', or 'msgpack')"); } }