moparisthebest
/
minetest
mirror of https://github.com/moparisthebest/minetest
1
0
Fork 0
Code Issues Releases Wiki Activity

Add BufReader and vector-based serialization methods

This commit is contained in:
kwolekr 2015-08-06 00:26:18 -04:00
parent e067ceacb8
commit 1a5b4b38f3
3 changed files with 573 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

263
src/unittest/test_serialization.cpp
View File

@ -40,6 +40,9 @@ class TestSerialization : public TestBase {
void testDeSerializeLongString();
void testStreamRead();
void testStreamWrite();
void testVecPut();
void testStringLengthLimits();
void testBufReader();
std::string teststring2;
std::wstring teststring2_w;
@ -64,6 +67,9 @@ void TestSerialization::runTests(IGameDef *gamedef)
TEST(testSerializeHex);
TEST(testStreamRead);
TEST(testStreamWrite);
TEST(testVecPut);
TEST(testStringLengthLimits);
TEST(testBufReader);
}
////////////////////////////////////////////////////////////////////////////////
@ -369,6 +375,263 @@ void TestSerialization::testStreamWrite()
}
void TestSerialization::testVecPut()
{
std::vector<u8> buf;
putU8(&buf, 0x11);
putU16(&buf, 0x2233);
putU32(&buf, 0x44556677);
putU64(&buf, 0x8899AABBCCDDEEFF);
putS8(&buf, -128);
putS16(&buf, 30000);
putS32(&buf, -6);
putS64(&buf, -43);
putF1000(&buf, 53.53467f);
putF1000(&buf, -300000.32f);
putF1000(&buf, F1000_MIN);
putF1000(&buf, F1000_MAX);
putString(&buf, "foobar!");
putV2S16(&buf, v2s16(500, 500));
putV3S16(&buf, v3s16(4207, 604, -30));
putV2S32(&buf, v2s32(1920, 1080));
putV3S32(&buf, v3s32(-400, 6400054, 290549855));
putV2F1000(&buf, v2f(500.65661f, 350.34567f));
putWideString(&buf, L"\x02~woof~\x5455");
putV3F1000(&buf, v3f(500, 10024.2f, -192.54f));
putARGB8(&buf, video::SColor(255, 128, 50, 128));
putLongString(&buf, "some longer string here");
putU16(&buf, 0xF00D);
UASSERT(buf.size() == sizeof(test_serialized_data));
UASSERT(!memcmp(&buf[0], test_serialized_data, sizeof(test_serialized_data)));
}
void TestSerialization::testStringLengthLimits()
{
std::vector<u8> buf;
std::string too_long(STRING_MAX_LEN + 1, 'A');
std::string way_too_large(LONG_STRING_MAX_LEN + 1, 'B');
std::wstring too_long_wide(WIDE_STRING_MAX_LEN + 1, L'C');
EXCEPTION_CHECK(SerializationError, putString(&buf, too_long));
putLongString(&buf, too_long);
too_long.resize(too_long.size() - 1);
putString(&buf, too_long);
EXCEPTION_CHECK(SerializationError, putWideString(&buf, too_long_wide));
too_long_wide.resize(too_long_wide.size() - 1);
putWideString(&buf, too_long_wide);
}
void TestSerialization::testBufReader()
{
u8 u8_data;
u16 u16_data;
u32 u32_data;
u64 u64_data;
s8 s8_data;
s16 s16_data;
s32 s32_data;
s64 s64_data;
f32 f32_data, f32_data2, f32_data3, f32_data4;
video::SColor scolor_data;
v2s16 v2s16_data;
v3s16 v3s16_data;
v2s32 v2s32_data;
v3s32 v3s32_data;
v2f v2f_data;
v3f v3f_data;
std::string string_data;
std::wstring widestring_data;
std::string longstring_data;
u8 raw_data[10] = {0};
BufReader buf(test_serialized_data, sizeof(test_serialized_data));
// Try reading data like normal
UASSERT(buf.getU8() == 0x11);
UASSERT(buf.getU16() == 0x2233);
UASSERT(buf.getU32() == 0x44556677);
UASSERT(buf.getU64() == 0x8899AABBCCDDEEFF);
UASSERT(buf.getS8() == -128);
UASSERT(buf.getS16() == 30000);
UASSERT(buf.getS32() == -6);
UASSERT(buf.getS64() == -43);
UASSERT(buf.getF1000() == 53.534f);
UASSERT(buf.getF1000() == -300000.32f);
UASSERT(buf.getF1000() == F1000_MIN);
UASSERT(buf.getF1000() == F1000_MAX);
UASSERT(buf.getString() == "foobar!");
UASSERT(buf.getV2S16() == v2s16(500, 500));
UASSERT(buf.getV3S16() == v3s16(4207, 604, -30));
UASSERT(buf.getV2S32() == v2s32(1920, 1080));
UASSERT(buf.getV3S32() == v3s32(-400, 6400054, 290549855));
UASSERT(buf.getV2F1000() == v2f(500.656f, 350.345f));
UASSERT(buf.getWideString() == L"\x02~woof~\x5455");
UASSERT(buf.getV3F1000() == v3f(500, 10024.2f, -192.54f));
UASSERT(buf.getARGB8() == video::SColor(255, 128, 50, 128));
UASSERT(buf.getLongString() == "some longer string here");
// Verify the offset and data is unchanged after a failed read
size_t orig_pos = buf.pos;
u32_data = 0;
UASSERT(buf.getU32NoEx(&u32_data) == false);
UASSERT(buf.pos == orig_pos);
UASSERT(u32_data == 0);
// Now try the same for a failed string read
UASSERT(buf.getStringNoEx(&string_data) == false);
UASSERT(buf.pos == orig_pos);
UASSERT(string_data == "");
// Now try the same for a failed string read
UASSERT(buf.getWideStringNoEx(&widestring_data) == false);
UASSERT(buf.pos == orig_pos);
UASSERT(widestring_data == L"");
UASSERT(buf.getU16() == 0xF00D);
UASSERT(buf.remaining() == 0);
// Check to make sure these each blow exceptions as they're supposed to
EXCEPTION_CHECK(SerializationError, buf.getU8());
EXCEPTION_CHECK(SerializationError, buf.getU16());
EXCEPTION_CHECK(SerializationError, buf.getU32());
EXCEPTION_CHECK(SerializationError, buf.getU64());
EXCEPTION_CHECK(SerializationError, buf.getS8());
EXCEPTION_CHECK(SerializationError, buf.getS16());
EXCEPTION_CHECK(SerializationError, buf.getS32());
EXCEPTION_CHECK(SerializationError, buf.getS64());
EXCEPTION_CHECK(SerializationError, buf.getF1000());
EXCEPTION_CHECK(SerializationError, buf.getARGB8());
EXCEPTION_CHECK(SerializationError, buf.getV2S16());
EXCEPTION_CHECK(SerializationError, buf.getV3S16());
EXCEPTION_CHECK(SerializationError, buf.getV2S32());
EXCEPTION_CHECK(SerializationError, buf.getV3S32());
EXCEPTION_CHECK(SerializationError, buf.getV2F1000());
EXCEPTION_CHECK(SerializationError, buf.getV3F1000());
EXCEPTION_CHECK(SerializationError, buf.getString());
EXCEPTION_CHECK(SerializationError, buf.getWideString());
EXCEPTION_CHECK(SerializationError, buf.getLongString());
EXCEPTION_CHECK(SerializationError,
buf.getRawData(raw_data, sizeof(raw_data)));
// See if we can skip backwards
buf.pos = 5;
UASSERT(buf.getRawDataNoEx(raw_data, 3) == true);
UASSERT(raw_data[0] == 0x66);
UASSERT(raw_data[1] == 0x77);
UASSERT(raw_data[2] == 0x88);
UASSERT(buf.getU32() == 0x99AABBCC);
UASSERT(buf.pos == 12);
// Now let's try it all over again using the NoEx variants
buf.pos = 0;
UASSERT(buf.getU8NoEx(&u8_data));
UASSERT(buf.getU16NoEx(&u16_data));
UASSERT(buf.getU32NoEx(&u32_data));
UASSERT(buf.getU64NoEx(&u64_data));
UASSERT(buf.getS8NoEx(&s8_data));
UASSERT(buf.getS16NoEx(&s16_data));
UASSERT(buf.getS32NoEx(&s32_data));
UASSERT(buf.getS64NoEx(&s64_data));
UASSERT(buf.getF1000NoEx(&f32_data));
UASSERT(buf.getF1000NoEx(&f32_data2));
UASSERT(buf.getF1000NoEx(&f32_data3));
UASSERT(buf.getF1000NoEx(&f32_data4));
UASSERT(buf.getStringNoEx(&string_data));
UASSERT(buf.getV2S16NoEx(&v2s16_data));
UASSERT(buf.getV3S16NoEx(&v3s16_data));
UASSERT(buf.getV2S32NoEx(&v2s32_data));
UASSERT(buf.getV3S32NoEx(&v3s32_data));
UASSERT(buf.getV2F1000NoEx(&v2f_data));
UASSERT(buf.getWideStringNoEx(&widestring_data));
UASSERT(buf.getV3F1000NoEx(&v3f_data));
UASSERT(buf.getARGB8NoEx(&scolor_data));
UASSERT(buf.getLongStringNoEx(&longstring_data));
// and make sure we got the correct data
UASSERT(u8_data == 0x11);
UASSERT(u16_data == 0x2233);
UASSERT(u32_data == 0x44556677);
UASSERT(u64_data == 0x8899AABBCCDDEEFF);
UASSERT(s8_data == -128);
UASSERT(s16_data == 30000);
UASSERT(s32_data == -6);
UASSERT(s64_data == -43);
UASSERT(f32_data == 53.534f);
UASSERT(f32_data2 == -300000.32f);
UASSERT(f32_data3 == F1000_MIN);
UASSERT(f32_data4 == F1000_MAX);
UASSERT(string_data == "foobar!");
UASSERT(v2s16_data == v2s16(500, 500));
UASSERT(v3s16_data == v3s16(4207, 604, -30));
UASSERT(v2s32_data == v2s32(1920, 1080));
UASSERT(v3s32_data == v3s32(-400, 6400054, 290549855));
UASSERT(v2f_data == v2f(500.656f, 350.345f));
UASSERT(widestring_data == L"\x02~woof~\x5455");
UASSERT(v3f_data == v3f(500, 10024.2f, -192.54f));
UASSERT(scolor_data == video::SColor(255, 128, 50, 128));
UASSERT(longstring_data == "some longer string here");
UASSERT(buf.remaining() == 2);
UASSERT(buf.getRawDataNoEx(raw_data, 3) == false);
UASSERT(buf.remaining() == 2);
UASSERT(buf.getRawDataNoEx(raw_data, 2) == true);
UASSERT(raw_data[0] == 0xF0);
UASSERT(raw_data[1] == 0x0D);
UASSERT(buf.remaining() == 0);
// Make sure no more available data causes a failure
UASSERT(!buf.getU8NoEx(&u8_data));
UASSERT(!buf.getU16NoEx(&u16_data));
UASSERT(!buf.getU32NoEx(&u32_data));
UASSERT(!buf.getU64NoEx(&u64_data));
UASSERT(!buf.getS8NoEx(&s8_data));
UASSERT(!buf.getS16NoEx(&s16_data));
UASSERT(!buf.getS32NoEx(&s32_data));
UASSERT(!buf.getS64NoEx(&s64_data));
UASSERT(!buf.getF1000NoEx(&f32_data));
UASSERT(!buf.getARGB8NoEx(&scolor_data));
UASSERT(!buf.getV2S16NoEx(&v2s16_data));
UASSERT(!buf.getV3S16NoEx(&v3s16_data));
UASSERT(!buf.getV2S32NoEx(&v2s32_data));
UASSERT(!buf.getV3S32NoEx(&v3s32_data));
UASSERT(!buf.getV2F1000NoEx(&v2f_data));
UASSERT(!buf.getV3F1000NoEx(&v3f_data));
UASSERT(!buf.getStringNoEx(&string_data));
UASSERT(!buf.getWideStringNoEx(&widestring_data));
UASSERT(!buf.getLongStringNoEx(&longstring_data));
UASSERT(!buf.getRawDataNoEx(raw_data, sizeof(raw_data)));
}
const u8 TestSerialization::test_serialized_data[12 * 13] = {
0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc,
0xdd, 0xee, 0xff, 0x80, 0x75, 0x30, 0xff, 0xff, 0xff, 0xfa, 0xff, 0xff,

71
src/util/serialize.cpp
View File

@ -28,6 +28,77 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <iomanip>
#include <vector>
SerializationError eof_ser_err("Attempted read past end of data");
////
//// BufReader
////
bool BufReader::getStringNoEx(std::string *val)
{
u16 num_chars;
if (!getU16NoEx(&num_chars))
return false;
if (pos + num_chars > size) {
pos -= sizeof(num_chars);
return false;
}
val->assign((const char *)data + pos, num_chars);
pos += num_chars;
return true;
}
bool BufReader::getWideStringNoEx(std::wstring *val)
{
u16 num_chars;
if (!getU16NoEx(&num_chars))
return false;
if (pos + num_chars * 2 > size) {
pos -= sizeof(num_chars);
return false;
}
for (size_t i = 0; i != num_chars; i++) {
val->push_back(readU16(data + pos));
pos += 2;
}
return true;
}
bool BufReader::getLongStringNoEx(std::string *val)
{
u32 num_chars;
if (!getU32NoEx(&num_chars))
return false;
if (pos + num_chars > size) {
pos -= sizeof(num_chars);
return false;
}
val->assign((const char *)data + pos, num_chars);
pos += num_chars;
return true;
}
bool BufReader::getRawDataNoEx(void *val, size_t len)
{
if (pos + len > size)
return false;
memcpy(val, data + pos, len);
pos += len;
return true;
}
////
//// String
////

241
src/util/serialize.h
View File

@ -21,14 +21,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#define UTIL_SERIALIZE_HEADER
#include "../irrlichttypes_bloated.h"
#include "../exceptions.h" // for SerializationError
#include "../debug.h" // for assert
#include "config.h"
#if HAVE_ENDIAN_H
#include <endian.h>
#include <string.h> // for memcpy
#include <endian.h>
#endif
#include <string.h> // for memcpy
#include <iostream>
#include <string>
#include <vector>
#define FIXEDPOINT_FACTOR 1000.0f
@ -405,4 +408,238 @@ bool serializeStructToString(std::string *out,
bool deSerializeStringToStruct(std::string valstr,
std::string format, void *out, size_t olen);
////
//// BufReader
////
extern SerializationError eof_ser_err;
#define MAKE_BUFREADER_GETNOEX_FXN(T, N, S) \
inline bool get ## N ## NoEx(T *val) \
{ \
if (pos + S > size) \
return false; \
*val = read ## N(data + pos); \
pos += S; \
return true; \
}
#define MAKE_BUFREADER_GET_FXN(T, N) \
inline T get ## N() \
{ \
T val; \
if (!get ## N ## NoEx(&val)) \
throw eof_ser_err; \
return val; \
}
class BufReader {
public:
BufReader(const u8 *data_, size_t size_) :
data(data_),
size(size_),
pos(0)
{
}
MAKE_BUFREADER_GETNOEX_FXN(u8, U8, 1);
MAKE_BUFREADER_GETNOEX_FXN(u16, U16, 2);
MAKE_BUFREADER_GETNOEX_FXN(u32, U32, 4);
MAKE_BUFREADER_GETNOEX_FXN(u64, U64, 8);
MAKE_BUFREADER_GETNOEX_FXN(s8, S8, 1);
MAKE_BUFREADER_GETNOEX_FXN(s16, S16, 2);
MAKE_BUFREADER_GETNOEX_FXN(s32, S32, 4);
MAKE_BUFREADER_GETNOEX_FXN(s64, S64, 8);
MAKE_BUFREADER_GETNOEX_FXN(f32, F1000, 4);
MAKE_BUFREADER_GETNOEX_FXN(v2s16, V2S16, 4);
MAKE_BUFREADER_GETNOEX_FXN(v3s16, V3S16, 6);
MAKE_BUFREADER_GETNOEX_FXN(v2s32, V2S32, 8);
MAKE_BUFREADER_GETNOEX_FXN(v3s32, V3S32, 12);
MAKE_BUFREADER_GETNOEX_FXN(v2f, V2F1000, 8);
MAKE_BUFREADER_GETNOEX_FXN(v3f, V3F1000, 12);
MAKE_BUFREADER_GETNOEX_FXN(video::SColor, ARGB8, 4);
bool getStringNoEx(std::string *val);
bool getWideStringNoEx(std::wstring *val);
bool getLongStringNoEx(std::string *val);
bool getRawDataNoEx(void *data, size_t len);
MAKE_BUFREADER_GET_FXN(u8, U8);
MAKE_BUFREADER_GET_FXN(u16, U16);
MAKE_BUFREADER_GET_FXN(u32, U32);
MAKE_BUFREADER_GET_FXN(u64, U64);
MAKE_BUFREADER_GET_FXN(s8, S8);
MAKE_BUFREADER_GET_FXN(s16, S16);
MAKE_BUFREADER_GET_FXN(s32, S32);
MAKE_BUFREADER_GET_FXN(s64, S64);
MAKE_BUFREADER_GET_FXN(f32, F1000);
MAKE_BUFREADER_GET_FXN(v2s16, V2S16);
MAKE_BUFREADER_GET_FXN(v3s16, V3S16);
MAKE_BUFREADER_GET_FXN(v2s32, V2S32);
MAKE_BUFREADER_GET_FXN(v3s32, V3S32);
MAKE_BUFREADER_GET_FXN(v2f, V2F1000);
MAKE_BUFREADER_GET_FXN(v3f, V3F1000);
MAKE_BUFREADER_GET_FXN(video::SColor, ARGB8);
MAKE_BUFREADER_GET_FXN(std::string, String);
MAKE_BUFREADER_GET_FXN(std::wstring, WideString);
MAKE_BUFREADER_GET_FXN(std::string, LongString);
inline void getRawData(void *val, size_t len)
{
if (!getRawDataNoEx(val, len))
throw eof_ser_err;
}
inline size_t remaining()
{
assert(pos <= size);
return size - pos;
}
const u8 *data;
size_t size;
size_t pos;
};
#undef MAKE_BUFREADER_GET_FXN
#undef MAKE_BUFREADER_GETNOEX_FXN
////
//// Vector-based write routines
////
inline void putU8(std::vector<u8> *dest, u8 val)
{
dest->push_back((val >> 0) & 0xFF);
}
inline void putU16(std::vector<u8> *dest, u16 val)
{
dest->push_back((val >> 8) & 0xFF);
dest->push_back((val >> 0) & 0xFF);
}
inline void putU32(std::vector<u8> *dest, u32 val)
{
dest->push_back((val >> 24) & 0xFF);
dest->push_back((val >> 16) & 0xFF);
dest->push_back((val >> 8) & 0xFF);
dest->push_back((val >> 0) & 0xFF);
}
inline void putU64(std::vector<u8> *dest, u64 val)
{
dest->push_back((val >> 56) & 0xFF);
dest->push_back((val >> 48) & 0xFF);
dest->push_back((val >> 40) & 0xFF);
dest->push_back((val >> 32) & 0xFF);
dest->push_back((val >> 24) & 0xFF);
dest->push_back((val >> 16) & 0xFF);
dest->push_back((val >> 8) & 0xFF);
dest->push_back((val >> 0) & 0xFF);
}
inline void putS8(std::vector<u8> *dest, s8 val)
{
putU8(dest, val);
}
inline void putS16(std::vector<u8> *dest, s16 val)
{
putU16(dest, val);
}
inline void putS32(std::vector<u8> *dest, s32 val)
{
putU32(dest, val);
}
inline void putS64(std::vector<u8> *dest, s64 val)
{
putU64(dest, val);
}
inline void putF1000(std::vector<u8> *dest, f32 val)
{
putS32(dest, val * FIXEDPOINT_FACTOR);
}
inline void putV2S16(std::vector<u8> *dest, v2s16 val)
{
putS16(dest, val.X);
putS16(dest, val.Y);
}
inline void putV3S16(std::vector<u8> *dest, v3s16 val)
{
putS16(dest, val.X);
putS16(dest, val.Y);
putS16(dest, val.Z);
}
inline void putV2S32(std::vector<u8> *dest, v2s32 val)
{
putS32(dest, val.X);
putS32(dest, val.Y);
}
inline void putV3S32(std::vector<u8> *dest, v3s32 val)
{
putS32(dest, val.X);
putS32(dest, val.Y);
putS32(dest, val.Z);
}
inline void putV2F1000(std::vector<u8> *dest, v2f val)
{
putF1000(dest, val.X);
putF1000(dest, val.Y);
}
inline void putV3F1000(std::vector<u8> *dest, v3f val)
{
putF1000(dest, val.X);
putF1000(dest, val.Y);
putF1000(dest, val.Z);
}
inline void putARGB8(std::vector<u8> *dest, video::SColor val)
{
putU32(dest, val.color);
}
inline void putString(std::vector<u8> *dest, const std::string &val)
{
if (val.size() > STRING_MAX_LEN)
throw SerializationError("String too long");
putU16(dest, val.size());
dest->insert(dest->end(), val.begin(), val.end());
}
inline void putWideString(std::vector<u8> *dest, const std::wstring &val)
{
if (val.size() > WIDE_STRING_MAX_LEN)
throw SerializationError("String too long");
putU16(dest, val.size());
for (size_t i = 0; i != val.size(); i++)
putU16(dest, val[i]);
}
inline void putLongString(std::vector<u8> *dest, const std::string &val)
{
if (val.size() > LONG_STRING_MAX_LEN)
throw SerializationError("String too long");
putU32(dest, val.size());
dest->insert(dest->end(), val.begin(), val.end());
}
inline void putRawData(std::vector<u8> *dest, const void *src, size_t len)
{
dest->insert(dest->end(), (u8 *)src, (u8 *)src + len);
}
#endif