From 4473a530b45d313929769d51e67244f886601970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Puppi?= Date: Mon, 18 Jun 2012 00:48:10 -0300 Subject: [PATCH 01/30] Fixed faulty implementation of table_is_an_array(). --- .gitignore | 2 ++ lua_cmsgpack.c | 27 +++++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 302bf46..9f7b2a5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ *.o *.so + +build/ diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index f7325e7..e65c1a4 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -372,29 +372,32 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { * of keys from numerical keys from 1 up to N, with N being the total number * of elements, without any hole in the middle. */ static int table_is_an_array(lua_State *L) { - long count = 0, idx = 0; + int count = 0, max = 0; lua_Number n; + /* Stack top on function entry */ + int stacktop; + + stacktop = lua_gettop(L); + lua_pushnil(L); while(lua_next(L,-2)) { /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ - if (lua_type(L,-1) != LUA_TNUMBER) goto not_array; - n = lua_tonumber(L,-1); - idx = n; - if (idx != n || idx < 1) goto not_array; + if (!lua_isnumber(L,-1) || (n = lua_tonumber(L, -1)) <= 0) { + lua_settop(L, stacktop); + return 0; + } + max = (n > max ? n : max); count++; } /* We have the total number of elements in "count". Also we have - * the max index encountered in "idx". We can't reach this code + * the max index encountered in "max". We can't reach this code * if there are indexes <= 0. If you also note that there can not be - * repeated keys into a table, you have that if idx==count you are sure + * repeated keys into a table, you have that if max==count you are sure * that there are all the keys form 1 to count (both included). */ - return idx == count; - -not_array: - lua_pop(L,1); - return 0; + lua_settop(L, stacktop); + return max == count; } /* If the length operator returns non-zero, that is, there is at least From fd2f98c69e59ec73f2e2823039d75f821eb30805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Puppi?= Date: Mon, 18 Jun 2012 01:29:28 -0300 Subject: [PATCH 02/30] Compensate for Lua telling us +-inf is an integer Fixed the issue where +-inf of NaNs would be encoded as integers. Added a header file which isolates inclusions and conditionally defines macros for inspecting float-type to integral-type conversions in a portable fashion. --- lua_cmsgpack.c | 26 ++++++++++++++++++++++---- rockspec/lua-cmsgpack-scm-1.rockspec | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index e65c1a4..e7c92e1 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -13,6 +13,23 @@ #define LUACMSGPACK_MAX_NESTING 16 /* Max tables nesting. */ +/* Allows a preprocessor directive to override MAX_NESTING */ +#ifndef LUACMSGPACK_MAX_NESTING + #define LUACMSGPACK_MAX_NESTING 16 +#endif + +#if (_XOPEN_SOURCE >= 600 || _ISOC99_SOURCE || _POSIX_C_SOURCE >= 200112L) + #define IS_FINITE(x) isfinite(x) +#else + #define IS_FINITE(x) ((x) == (x) && (x) + 1 > (x)) +#endif + +/* Check if float or double can be an integer without loss of precision */ +#define IS_INT_TYPE_EQUIVALENT(x, T) (IS_FINITE(x) && (T)(x) == (x)) + +#define IS_INT64_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int64_t) +#define IS_INT_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int) + /* ============================================================================== * MessagePack implementation and bindings for Lua 5.1/5.2. * Copyright(C) 2012 Salvatore Sanfilippo @@ -318,10 +335,10 @@ static void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { static void mp_encode_lua_number(lua_State *L, mp_buf *buf) { lua_Number n = lua_tonumber(L,-1); - if (floor(n) != n) { - mp_encode_double(buf,(double)n); - } else { + if (IS_INT64_EQUIVALENT(n)) { mp_encode_int(buf,(int64_t)n); + } else { + mp_encode_double(buf,(double)n); } } @@ -384,7 +401,8 @@ static int table_is_an_array(lua_State *L) { while(lua_next(L,-2)) { /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ - if (!lua_isnumber(L,-1) || (n = lua_tonumber(L, -1)) <= 0) { + if (!lua_isnumber(L,-1) || (n = lua_tonumber(L, -1)) <= 0 || + !IS_INT_EQUIVALENT(n)) { lua_settop(L, stacktop); return 0; } diff --git a/rockspec/lua-cmsgpack-scm-1.rockspec b/rockspec/lua-cmsgpack-scm-1.rockspec index e95fd42..fe2a034 100644 --- a/rockspec/lua-cmsgpack-scm-1.rockspec +++ b/rockspec/lua-cmsgpack-scm-1.rockspec @@ -18,7 +18,7 @@ build = { modules = { cmsgpack = { sources = { - "lua_cmsgpack.c", + "lua_cmsgpack.c" } } } From dccf0fe3ce7ef07c54f7a2f2133fd14df8818fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Puppi?= Date: Mon, 18 Jun 2012 01:52:36 -0300 Subject: [PATCH 03/30] Added tests for +-inf to test.lua --- test.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test.lua b/test.lua index 56cabf8..aa528b6 100644 --- a/test.lua +++ b/test.lua @@ -120,6 +120,8 @@ test_circular("fix array (1)",{1,2,3,"foo"}) test_circular("fix array (2)",{}) test_circular("fix array (3)",{1,{},{}}) test_circular("fix map",{a=5,b=10,c="string"}) +test_circular("positive infinity", math.huge) +test_circular("negative infinity", -math.huge) -- The following test vectors are taken from the Javascript lib at: -- https://github.com/cuzic/MessagePack-JS/blob/master/test/test_pack.html From 48f338c171230fef9f6aaf2a77aaeeb9312fd8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Puppi?= Date: Mon, 18 Jun 2012 18:17:40 -0300 Subject: [PATCH 04/30] Add stream support Now pack can receive any number of arguments returning a stream of the serialized data (on 0 objects, returns the empty string). unpack deserialized any stream of objects, returning the result of the whole stream deserialization (this, the number of return values of unpack is equal to the number of objects unpacked). If given the empty string, returns nothing. I also removed all warnings and made the code C++ compatible. [untested at this point] --- lua_cmsgpack.c | 74 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index e7c92e1..f4a33f3 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -59,7 +59,9 @@ * simplicity of the Lua build system we prefer to check for endianess at runtime. * The performance difference should be acceptable. */ static void memrevifle(void *ptr, size_t len) { - unsigned char *p = ptr, *e = p+len-1, aux; + unsigned char *p = (unsigned char *)ptr, + *e = (unsigned char *)p+len-1, + aux; int test = 1; unsigned char *testp = (unsigned char*) &test; @@ -86,7 +88,7 @@ typedef struct mp_buf { } mp_buf; static mp_buf *mp_buf_new(void) { - mp_buf *buf = malloc(sizeof(*buf)); + mp_buf *buf = (mp_buf*)malloc(sizeof(*buf)); buf->b = NULL; buf->len = buf->free = 0; @@ -97,7 +99,7 @@ void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { if (buf->free < len) { size_t newlen = buf->len+len; - buf->b = realloc(buf->b,newlen*2); + buf->b = (unsigned char*)realloc(buf->b,newlen*2); buf->free = newlen; } memcpy(buf->b+buf->len,s,len); @@ -130,7 +132,7 @@ typedef struct mp_cur { } mp_cur; static mp_cur *mp_cur_new(const unsigned char *s, size_t len) { - mp_cur *cursor = malloc(sizeof(*cursor)); + mp_cur *cursor = (mp_cur*)malloc(sizeof(*cursor)); cursor->p = s; cursor->left = len; @@ -430,6 +432,7 @@ static void mp_encode_lua_table(lua_State *L, mp_buf *buf, int level) { static void mp_encode_lua_null(lua_State *L, mp_buf *buf) { unsigned char b[1]; + (void)L; b[0] = 0xc0; mp_buf_append(buf,b,1); @@ -451,12 +454,30 @@ static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { lua_pop(L,1); } +/* + * Packs all arguments as a stream. + * Returns the empty string if no argument was given. + */ static int mp_pack(lua_State *L) { - mp_buf *buf = mp_buf_new(); + int nargs = lua_gettop(L); + int i; - mp_encode_lua_type(L,buf,0); - lua_pushlstring(L,(char*)buf->b,buf->len); - mp_buf_free(buf); + if (nargs == 0) { + lua_pushliteral(L, ""); + return 1; + } + + /* Create nargs packed buffers */ + for (i = 0; i < nargs; i++) { + mp_buf *buf = mp_buf_new(); + + mp_encode_lua_type(L,buf,0); + lua_pushlstring(L,(char*)buf->b,buf->len); + mp_buf_free(buf); + } + + /* Concatenate all nargs buffers together */ + lua_concat(L, nargs); return 1; } @@ -683,6 +704,7 @@ static int mp_unpack(lua_State *L) { size_t len; const unsigned char *s; mp_cur *c; + int cnt; /* Number of objects unpacked */ if (!lua_isstring(L,-1)) { lua_pushstring(L,"MessagePack decoding needs a string as input."); @@ -691,24 +713,26 @@ static int mp_unpack(lua_State *L) { s = (const unsigned char*) lua_tolstring(L,-1,&len); c = mp_cur_new(s,len); - mp_decode_to_lua_type(L,c); - - if (c->err == MP_CUR_ERROR_EOF) { - mp_cur_free(c); - lua_pushstring(L,"Missing bytes in input."); - lua_error(L); - } else if (c->err == MP_CUR_ERROR_BADFMT) { - mp_cur_free(c); - lua_pushstring(L,"Bad data format in input."); - lua_error(L); - } else if (c->left != 0) { - mp_cur_free(c); - lua_pushstring(L,"Extra bytes in input."); - lua_error(L); - } else { - mp_cur_free(c); + + /* We loop over the decode because this could be a stream */ + for(cnt = 0; c->left > 0; cnt++) { + mp_decode_to_lua_type(L,c); + + if (c->err == MP_CUR_ERROR_EOF) { + mp_cur_free(c); + c = NULL; + lua_pushstring(L,"Missing bytes in input."); + lua_error(L); + } else if (c->err == MP_CUR_ERROR_BADFMT) { + mp_cur_free(c); + c = NULL; + lua_pushstring(L,"Bad data format in input."); + lua_error(L); + } } - return 1; + + mp_cur_free(c); + return cnt; } /* ---------------------------------------------------------------------------- */ From 41e4e129739638dfb31d1f947a2cd4c12ec75d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Puppi?= Date: Mon, 18 Jun 2012 19:06:24 -0300 Subject: [PATCH 05/30] Correct 1-indexing and encode popping --- lua_cmsgpack.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index f4a33f3..2de9c0f 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -466,12 +466,19 @@ static int mp_pack(lua_State *L) { lua_pushliteral(L, ""); return 1; } - + /* Create nargs packed buffers */ - for (i = 0; i < nargs; i++) { - mp_buf *buf = mp_buf_new(); - + for(i = 1; i <= nargs; i++) { + mp_buf *buf; + + /* Copy argument i to top of stack for _encode processing; + * the encode function pops it from the stack when complete. */ + lua_pushvalue(L, i); + + buf = mp_buf_new(); mp_encode_lua_type(L,buf,0); + + lua_settop(L, nargs + i - 1); lua_pushlstring(L,(char*)buf->b,buf->len); mp_buf_free(buf); } From ed6abff9c3d511c57406ac671e6bfa8cf15866f6 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 5 Apr 2014 14:59:40 -0400 Subject: [PATCH 06/30] Add cmsgpack.safe version @moteus suggested this change in https://github.com/moteus/lua-cmsgpack/commit/7f64c4f93989364899c777f888ce3b86da50546c That patch is a bit of a mess (it's a merge of somebody else's work squashed with about four other changes too), so I re-implemented the safe functionality in a slightly cleaner way (using Lua C closures with upvalues). Now we have a cmsgpack.safe module that won't throw errors, but rather return them as clean (nil, err) return values. Minor API change: calling pack() with no arguments now throws an error instead of returning an empty string (because nothing was encoded, we shouldn't return anything resembling usable data). Also: now we prefer luaL_argerror for cleaner errors caused by invalid arguments and also luaL_error for easier msg-and-return-error functionality. Double also: now we prefer to call return on error functions so it's clear to us the Lua error functions do long jumps and never return control to the original function from then onward. --- lua_cmsgpack.c | 114 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 33 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 2de9c0f..cf9e6fa 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -7,6 +7,8 @@ #include "lua.h" #include "lauxlib.h" +#define LUACMSGPACK_NAME "cmsgpack" +#define LUACMSGPACK_SAFE_NAME "cmsgpack_safe" #define LUACMSGPACK_VERSION "lua-cmsgpack 0.3.1" #define LUACMSGPACK_COPYRIGHT "Copyright (C) 2012, Salvatore Sanfilippo" #define LUACMSGPACK_DESCRIPTION "MessagePack C implementation for Lua" @@ -455,17 +457,15 @@ static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { } /* - * Packs all arguments as a stream. - * Returns the empty string if no argument was given. + * Packs all arguments as a stream for multiple upacking later. + * Returns error if no arguments provided. */ static int mp_pack(lua_State *L) { int nargs = lua_gettop(L); int i; - if (nargs == 0) { - lua_pushliteral(L, ""); - return 1; - } + if (nargs == 0) + return luaL_argerror(L, 0, "MessagePack pack needs input."); /* Create nargs packed buffers */ for(i = 1; i <= nargs; i++) { @@ -713,28 +713,22 @@ static int mp_unpack(lua_State *L) { mp_cur *c; int cnt; /* Number of objects unpacked */ - if (!lua_isstring(L,-1)) { - lua_pushstring(L,"MessagePack decoding needs a string as input."); - lua_error(L); - } + if ((s = (unsigned char*)luaL_checklstring(L,1,&len)) == NULL) + return luaL_argerror(L,1,"MessagePack decoding requires string input."); - s = (const unsigned char*) lua_tolstring(L,-1,&len); c = mp_cur_new(s,len); - /* We loop over the decode because this could be a stream */ + /* We loop over the decode because this could be a stream + * of multiple top-level values serialized together */ for(cnt = 0; c->left > 0; cnt++) { mp_decode_to_lua_type(L,c); if (c->err == MP_CUR_ERROR_EOF) { mp_cur_free(c); - c = NULL; - lua_pushstring(L,"Missing bytes in input."); - lua_error(L); + return luaL_error(L,"Missing bytes in input."); } else if (c->err == MP_CUR_ERROR_BADFMT) { mp_cur_free(c); - c = NULL; - lua_pushstring(L,"Bad data format in input."); - lua_error(L); + return luaL_error(L,"Bad data format in input."); } } @@ -742,25 +736,45 @@ static int mp_unpack(lua_State *L) { return cnt; } +static int mp_safe(lua_State *L) { + int argc, err, total_results; + + argc = lua_gettop(L); + + /* This adds our function to the bottom of the stack + * (the "call this function" position) */ + lua_pushvalue(L, lua_upvalueindex(1)); + lua_insert(L, 1); + + err = lua_pcall(L, argc, LUA_MULTRET, 0); + total_results = lua_gettop(L); + + if (!err) { + return total_results; + } else { + lua_pushnil(L); + lua_insert(L,-2); + return 2; + } +} + /* ---------------------------------------------------------------------------- */ +int luaopen_create(lua_State *L) { + /* Manually construct our module table instead of + * relying on _register or _newlib */ + lua_newtable(L); -#if LUA_VERSION_NUM < 502 -static const struct luaL_reg thislib[] = { -#else -static const struct luaL_Reg thislib[] = { -#endif - {"pack", mp_pack}, - {"unpack", mp_unpack}, - {NULL, NULL} -}; + /* Add unpack function */ + lua_pushcfunction(L, mp_unpack); + lua_setfield(L, -2, "unpack"); -LUALIB_API int luaopen_cmsgpack (lua_State *L) { -#if LUA_VERSION_NUM < 502 - luaL_register(L, "cmsgpack", thislib); -#else - luaL_newlib(L, thislib); -#endif + /* Add pack function */ + lua_pushcfunction(L, mp_pack); + lua_setfield(L, -2, "pack"); + /* Add metadata */ + lua_pushliteral(L, LUACMSGPACK_NAME); + lua_setfield(L, -2, "_NAME"); lua_pushliteral(L, LUACMSGPACK_VERSION); lua_setfield(L, -2, "_VERSION"); lua_pushliteral(L, LUACMSGPACK_COPYRIGHT); @@ -770,6 +784,40 @@ LUALIB_API int luaopen_cmsgpack (lua_State *L) { return 1; } +LUALIB_API int luaopen_cmsgpack(lua_State *L) { + luaopen_create(L); + +#if LUA_VERSION_NUM < 502 + /* Register name globally for 5.1 */ + lua_pushvalue(L, -1); + lua_setglobal(L, LUACMSGPACK_NAME); +#endif + + return 1; +} + +LUALIB_API int luaopen_cmsgpack_safe(lua_State *L) { + luaopen_cmsgpack(L); + + /* Add safe wrapper to pack */ + lua_getfield(L, -1, "pack"); + lua_pushcclosure(L, mp_safe, 1); + lua_setfield(L, -2, "pack"); + + /* Add safe wrapper to unpack */ + lua_getfield(L, -1, "unpack"); + lua_pushcclosure(L, mp_safe, 1); + lua_setfield(L, -2, "unpack"); + +#if LUA_VERSION_NUM < 502 + /* Register name globally for 5.1 */ + lua_pushvalue(L, -1); + lua_setglobal(L, LUACMSGPACK_SAFE_NAME); +#endif + + return 1; +} + /****************************************************************************** * Copyright (C) 2012 Salvatore Sanfilippo. All rights reserved. * From 5308f7cbdb9b2ba9736014d0d770c88611127a53 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 5 Apr 2014 15:07:28 -0400 Subject: [PATCH 07/30] Remove noop stack setter This was a complex way of saying "make the top of stack the current top of the stack" and caused no side effects. --- lua_cmsgpack.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index cf9e6fa..1e986d1 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -478,7 +478,6 @@ static int mp_pack(lua_State *L) { buf = mp_buf_new(); mp_encode_lua_type(L,buf,0); - lua_settop(L, nargs + i - 1); lua_pushlstring(L,(char*)buf->b,buf->len); mp_buf_free(buf); } From df39126ce57e5c4670e49c7dc5fd829eb7e583d5 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 5 Apr 2014 15:10:09 -0400 Subject: [PATCH 08/30] Update tests and add build infrastructure We're using CMake here because building a Makefile takes longer and won't automatically find your system-wide Lua for including against. You can run the tests with: mkdir build cd build cmake .. lua ../test.lua As of this commit all tests pass: TEST PASSED: 207 TEST FAILED: 0 TEST SKIPPED: 0 This commit adds many new tests and test checking methods for existing functionality and the new streaming/multi-pack/multi-return functionality. Thanks to @moteus for testing suggestions which I've included. Sadly, I couldn't include the changes directly because they were too intermixed with the rest of a 250 line commit[1]. [1]: https://github.com/moteus/lua-cmsgpack/commit/7f64c4f93989364899c777f888ce3b86da50546c --- CMakeLists.txt | 37 +++++++++++ test.lua | 169 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..461e9a9 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,37 @@ +# If Lua is installed in a non-standard location, please set the LUA_DIR +# environment variable to point to prefix for the install. Eg: +# Unix: export LUA_DIR=/home/user/pkg +# Windows: set LUA_DIR=c:\lua51 + +project(lua-cmsgpack C) +cmake_minimum_required(VERSION 2.6) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING + "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() + +find_package(Lua51 REQUIRED) +include_directories(${LUA_INCLUDE_DIR}) + +if(APPLE) + set(CMAKE_SHARED_MODULE_CREATE_C_FLAGS + "${CMAKE_SHARED_MODULE_CREATE_C_FLAGS} -undefined dynamic_lookup") +endif() + +if(WIN32) + # Win32 modules need to be linked to the Lua library. + set(_MODULE_LINK ${LUA_LIBRARY} ${_MODULE_LINK}) + set(_lua_module_dir "${_lua_lib_dir}") +else() + set(_lua_module_dir "${_lua_lib_dir}/lua/5.1") +endif() + +set(CMAKE_C_FLAGS "-O2 -g -ggdb -Wall -pedantic -std=c99") +add_library(cmsgpack MODULE lua_cmsgpack.c) +set_target_properties(cmsgpack PROPERTIES PREFIX "") +target_link_libraries(cmsgpack ${_MODULE_LINK}) +install(TARGETS cmsgpack DESTINATION "${_lua_module_dir}") + +# vi:ai et sw=4 ts=4: diff --git a/test.lua b/test.lua index aa528b6..e5463fa 100644 --- a/test.lua +++ b/test.lua @@ -3,9 +3,12 @@ -- See the copyright notice at the end of lua_cmsgpack.c for more information. local cmsgpack = require "cmsgpack" +local ok, cmsgpack_safe = pcall(require, 'cmsgpack.safe') +if not ok then cmsgpack_safe = nil end passed = 0 failed = 0 +skipped = 0 function hex(s) local i @@ -40,11 +43,57 @@ function unhex(h) return s end -function compare_objects(a,b) +function test_error(name, fn) + io.write("Testing generate error '",name,"' ...") + local ok, ret, err = pcall(fn) + -- 'ok' is an error because we are testing for expicit *failure* + if ok then + print("ERROR: result ", ret, err) + failed = failed+1 + else + print("ok") + passed = passed+1 + end +end + +local function test_multiple(name, ...) + io.write("Multiple test '",name,"' ...") + if not compare_objects({...},{cmsgpack.unpack(cmsgpack.pack(...))}) then + print("ERROR:", {...}, cmsgpack.unpack(cmsgpack.pack(...))) + failed = failed+1 + else + print("ok") + passed = passed+1 + end +end + +function test_noerror(name, fn) + io.write("Testing safe calling '",name,"' ...") + if not cmsgpack_safe then + print("skip: no `cmsgpack.safe` module") + skipped = skipped + 1 + return + end + local ok, ret, err = pcall(fn) + if not ok then + print("ERROR: result ", ret, err) + failed = failed+1 + else + print("ok") + passed = passed+1 + end +end + +function compare_objects(a,b,depth) if (type(a) == "table") then local count = 0 + if not depth then + depth = 1 + elseif depth == 10 then + return true -- assume if match down 10 levels, the rest is okay too + end for k,v in pairs(a) do - if not compare_objects(b[k],v) then return false end + if not compare_objects(b[k],v, depth + 1) then return false end count = count + 1 end -- All the 'a' keys are equal to their 'b' equivalents. @@ -67,6 +116,45 @@ function test_circular(name,obj) end end +function test_stream(mod, name, ...) + io.write("Stream test '", name, "' ...") + if not mod then + print("skip: no `cmsgpack.safe` module") + skipped = skipped + 1 + return + end + local argc = select('#', ...) + for i=1, argc do + test_circular(name, select(i, ...)) + end + local ret = {mod.unpack(mod.pack(unpack({...})))} + for i=1, argc do + local origin = select(i, ...) + if (type(origin) == "table") then + for k,v in pairs(origin) do + local fail = not compare_objects(v, ret[i][k]) + if fail then + print("ERRORa:", k, v, " not match ", ret[i][k]) + failed = failed + 1 + elseif not fail then + print("ok; matched stream table member") + passed = passed + 1 + end + end + else + local fail = not compare_objects(origin, ret[i]) + if fail then + print("ERRORc:", origin, " not match ", ret[i]) + failed = failed + 1 + elseif not fail then + print("ok; matched individual stream member") + passed = passed + 1 + end + end + end + +end + function test_pack(name,obj,raw) io.write("Testing encoder '",name,"' ...") if hex(cmsgpack.pack(obj)) ~= raw then @@ -94,6 +182,54 @@ function test_pack_and_unpack(name,obj,raw) test_unpack(name,raw,obj) end +local function test_global() + io.write("Testing global variable ...") + + if _VERSION == "Lua 5.1" then + if not _G.cmsgpack then + print("ERROR: Lua 5.1 should set global") + failed = failed+1 + else + print("ok") + passed = passed+1 + end + else + if _G.cmsgpack then + print("ERROR: Lua 5.2 should not set global") + failed = failed+1 + else + print("ok") + passed = passed+1 + end + end +end + +local function test_array() + io.write("Testing array detection ...") + + local a = {a1 = 1, a2 = 1, a3 = 1, a4 = 1, a5 = 1, a6 = 1, a7 = 1, a8 = 1, a9 = 1} + a[1] = 10 a[2] = 20 a[3] = 30 + a.a1,a.a2,a.a3,a.a4,a.a5,a.a6,a.a7,a.a8, a.a9 = nil + + local test_obj = {10,20,30} + assert(compare_objects(test_obj, a)) + + local etalon = cmsgpack.pack(test_obj) + local encode = cmsgpack.pack(a) + + if etalon ~= encode then + print("ERROR:") + print("", "expected: ", hex(etalon)) + print("", " got: ", hex(encode)) + failed = failed+1 + else + print("ok") + passed = passed+1 + end +end + +test_global() +test_array() test_circular("positive fixnum",17); test_circular("negative fixnum",-1); test_circular("true boolean",true); @@ -151,8 +287,37 @@ b = {x=a} a['x'] = b pack = cmsgpack.pack(a) test_pack("regression for issue #4",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0") +test_circular("regression for issue #4",a) + +-- Tests from github.com/moteus +test_circular("map with number keys", {[1] = {1,2,3}}) +test_circular("map with float keys", {[1.5] = {1,2,3}}) +test_error("unpack nil", function() cmsgpack.unpack(nil) end) +test_error("unpack table", function() cmsgpack.unpack({}) end) +test_error("unpack udata", function() cmsgpack.unpack(io.stdout) end) +test_noerror("unpack nil", function() cmsgpack_safe.unpack(nil) end) +test_noerror("unpack nil", function() cmsgpack_safe.unpack(nil) end) +test_noerror("unpack table", function() cmsgpack_safe.unpack({}) end) +test_noerror("unpack udata", function() cmsgpack_safe.unpack(io.stdout) end) +test_multiple("two ints", 1, 2) +test_multiple("holes", 1, nil, 2, nil, 4) + +-- Streaming/Multi-Input Tests +test_stream(cmsgpack, "simple", {a=1}, {b=2}, {c=3}, 4, 5, 6, 7) +test_stream(cmsgpack_safe, "safe simple", {a=1}, {b=2}, {c=3}, 4, 5, 6, 7) +test_stream(cmsgpack, "oddities", {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, {0}, {a=64}, math.huge, -math.huge) +test_stream(cmsgpack_safe, "safe oddities", {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, {0}, {a=64}, math.huge, -math.huge) +test_stream(cmsgpack, "strange things", nil, {}, {nil}, a, b, b, b, a, a, b, {c = a, d = b}) +test_stream(cmsgpack_safe, "strange things", nil, {}, {nil}, a, b, b, b, a, a, b, {c = a, d = b}) +test_error("pack nothing", function() cmsgpack.pack() end) +test_noerror("pack nothing safe", function() cmsgpack_safe.pack() end) -- Final report print() print("TEST PASSED:",passed) print("TEST FAILED:",failed) +print("TEST SKIPPED:",skipped) + +if failed > 0 then + os.exit(1) +end From 155a6bacf5ece6cfe7900986722e2929971bf922 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 5 Apr 2014 15:16:10 -0400 Subject: [PATCH 09/30] Add better type checking when Lua 5.3 is available Lua 5.3 adds an actual Integer type, which by default will be a 64-bit long long. Currently, Lua {5.1,5.2} only has a Number type which by default is a 64-bit double. Note: Lua can be custom compiled to re-define Integer to be 32 bits (as well as Number to be just a float), but that should be rare and the interal cmsgpack conversion routes store integers in the most compact representation discernable. This also uses the existing lua_tointeger function instead of casting a lua_Number to an int64_t directly. --- lua_cmsgpack.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 1e986d1..588c180 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -336,11 +336,20 @@ static void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { mp_buf_append(buf,&b,1); } +/* Lua 5.3 has a built in 64-bit integer type */ +static void mp_encode_lua_integer(lua_State *L, mp_buf *buf) { + lua_Integer i = lua_tointeger(L,-1); + mp_encode_int(buf, (int64_t)i); +} + +/* Lua 5.2 and lower only has 64-bit doubles, so we need to + * detect if the double may be representable as an int + * for Lua < 5.3 */ static void mp_encode_lua_number(lua_State *L, mp_buf *buf) { lua_Number n = lua_tonumber(L,-1); if (IS_INT64_EQUIVALENT(n)) { - mp_encode_int(buf,(int64_t)n); + mp_encode_lua_integer(L, buf); } else { mp_encode_double(buf,(double)n); } @@ -394,7 +403,11 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { * of elements, without any hole in the middle. */ static int table_is_an_array(lua_State *L) { int count = 0, max = 0; +#if LUA_VERSION_NUM < 503 lua_Number n; +#else + lua_Integer n; +#endif /* Stack top on function entry */ int stacktop; @@ -405,8 +418,13 @@ static int table_is_an_array(lua_State *L) { while(lua_next(L,-2)) { /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ + /* The <= 0 check is valid here because we're comparing indexes. */ +#if LUA_VERSION_NUM < 503 if (!lua_isnumber(L,-1) || (n = lua_tonumber(L, -1)) <= 0 || !IS_INT_EQUIVALENT(n)) { +#else + if (!lua_isinteger(L,-1) || (n = lua_tointeger(L, -1)) <= 0) { +#endif lua_settop(L, stacktop); return 0; } @@ -449,7 +467,17 @@ static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { switch(t) { case LUA_TSTRING: mp_encode_lua_string(L,buf); break; case LUA_TBOOLEAN: mp_encode_lua_bool(L,buf); break; - case LUA_TNUMBER: mp_encode_lua_number(L,buf); break; + case LUA_TNUMBER: + #if LUA_VERSION_NUM < 503 + mp_encode_lua_number(L,buf); break; + #else + if (lua_isinteger(L, -1)) { + mp_encode_lua_integer(L, buf); + } else { + mp_encode_lua_number(L, buf); + } + break; + #endif case LUA_TTABLE: mp_encode_lua_table(L,buf,level); break; default: mp_encode_lua_null(L,buf); break; } From cdb4e481c4a4e6b3a5eb2896a34f0121a7d4dce4 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 5 Apr 2014 15:46:34 -0400 Subject: [PATCH 10/30] Cleanup: limit to 80 chars wide Also remove empty end-of-line spaces from a few places. --- lua_cmsgpack.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 588c180..b7af3d6 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -32,7 +32,7 @@ #define IS_INT64_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int64_t) #define IS_INT_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int) -/* ============================================================================== +/* ============================================================================= * MessagePack implementation and bindings for Lua 5.1/5.2. * Copyright(C) 2012 Salvatore Sanfilippo * @@ -49,16 +49,16 @@ * 20-Feb-2012 (ver 0.2.1): Minor bug fixing. * 20-Feb-2012 (ver 0.3.0): Module renamed lua-cmsgpack (was lua-msgpack). * 04-Apr-2014 (ver 0.3.1): Lua 5.2 support and minor bug fix. - * ============================================================================ */ + * ========================================================================== */ -/* --------------------------- Endian conversion -------------------------------- - * We use it only for floats and doubles, all the other conversions are performed +/* -------------------------- Endian conversion -------------------------------- + * We use it only for floats and doubles, all the other conversions performed * in an endian independent fashion. So the only thing we need is a function - * that swaps a binary string if the arch is little endian (and left it untouched + * that swaps a binary string if arch is little endian (and left it untouched * otherwise). */ /* Reverse memory bytes if arch is little endian. Given the conceptual - * simplicity of the Lua build system we prefer to check for endianess at runtime. + * simplicity of the Lua build system we prefer check for endianess at runtime. * The performance difference should be acceptable. */ static void memrevifle(void *ptr, size_t len) { unsigned char *p = (unsigned char *)ptr, @@ -78,7 +78,7 @@ static void memrevifle(void *ptr, size_t len) { } } -/* ----------------------------- String buffer ---------------------------------- +/* ---------------------------- String buffer ---------------------------------- * This is a simple implementation of string buffers. The only opereation * supported is creating empty buffers and appending bytes to it. * The string buffer uses 2x preallocation on every realloc for O(N) append @@ -91,7 +91,7 @@ typedef struct mp_buf { static mp_buf *mp_buf_new(void) { mp_buf *buf = (mp_buf*)malloc(sizeof(*buf)); - + buf->b = NULL; buf->len = buf->free = 0; return buf; @@ -114,7 +114,7 @@ void mp_buf_free(mp_buf *buf) { free(buf); } -/* ------------------------------ String cursor ---------------------------------- +/* ---------------------------- String cursor ---------------------------------- * This simple data structure is used for parsing. Basically you create a cursor * using a string pointer and a length, then it is possible to access the * current string position with cursor->p, check the remaining length @@ -124,7 +124,7 @@ void mp_buf_free(mp_buf *buf) { * be used to report errors. */ #define MP_CUR_ERROR_NONE 0 -#define MP_CUR_ERROR_EOF 1 /* Not enough data to complete the opereation. */ +#define MP_CUR_ERROR_EOF 1 /* Not enough data to complete opereation. */ #define MP_CUR_ERROR_BADFMT 2 /* Bad data format */ typedef struct mp_cur { @@ -158,7 +158,7 @@ static void mp_cur_free(mp_cur *cursor) { } \ } while(0) -/* --------------------------- Low level MP encoding -------------------------- */ +/* ------------------------- Low level MP encoding -------------------------- */ static void mp_encode_bytes(mp_buf *buf, const unsigned char *s, size_t len) { unsigned char hdr[5]; @@ -321,7 +321,7 @@ static void mp_encode_map(mp_buf *buf, int64_t n) { mp_buf_append(buf,b,enclen); } -/* ----------------------------- Lua types encoding --------------------------- */ +/* --------------------------- Lua types encoding --------------------------- */ static void mp_encode_lua_string(lua_State *L, mp_buf *buf) { size_t len; @@ -509,13 +509,13 @@ static int mp_pack(lua_State *L) { lua_pushlstring(L,(char*)buf->b,buf->len); mp_buf_free(buf); } - + /* Concatenate all nargs buffers together */ lua_concat(L, nargs); return 1; } -/* --------------------------------- Decoding --------------------------------- */ +/* ------------------------------- Decoding --------------------------------- */ void mp_decode_to_lua_type(lua_State *L, mp_cur *c); @@ -749,7 +749,7 @@ static int mp_unpack(lua_State *L) { * of multiple top-level values serialized together */ for(cnt = 0; c->left > 0; cnt++) { mp_decode_to_lua_type(L,c); - + if (c->err == MP_CUR_ERROR_EOF) { mp_cur_free(c); return luaL_error(L,"Missing bytes in input."); @@ -758,7 +758,7 @@ static int mp_unpack(lua_State *L) { return luaL_error(L,"Bad data format in input."); } } - + mp_cur_free(c); return cnt; } @@ -785,7 +785,7 @@ static int mp_safe(lua_State *L) { } } -/* ---------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ int luaopen_create(lua_State *L) { /* Manually construct our module table instead of * relying on _register or _newlib */ @@ -807,7 +807,7 @@ int luaopen_create(lua_State *L) { lua_pushliteral(L, LUACMSGPACK_COPYRIGHT); lua_setfield(L, -2, "_COPYRIGHT"); lua_pushliteral(L, LUACMSGPACK_DESCRIPTION); - lua_setfield(L, -2, "_DESCRIPTION"); + lua_setfield(L, -2, "_DESCRIPTION"); return 1; } From 507f353d00c59333773b3627b6e0dedd3b6dd79e Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 5 Apr 2014 23:26:02 -0400 Subject: [PATCH 11/30] Add fine-grained control of unpacking objects Inspired by #10, I added two new functions allowing you to limit multiple return from unpack. If you don't trust your msgpack input or want to manage flow control of returned object generation yourself, you can now limit the objects returned from unpack (but not with unpack() itself because it already returns an arbitrary number of objects). You can now unpack_one(msgpack) to get (next_offset, obj) returned; then call unpack_one(msgpack, next_offset) to get another offset and one more object> You can now also unpack_limit(msgpack, N) to get (next_offset, obj1, obj2, ..., objN) returned; then call unpack_limit(msgpack, K, next_offset) to get another offset and K more objects. This also refactors the copy/paste command addition in module bring-up. Tests added for new commands and all tests pass: TEST PASSED: 225 TEST FAILED: 0 TEST SKIPPED: 0 Closes #10 --- lua_cmsgpack.c | 96 ++++++++++++++++++++++++++++++++++++++------------ test.lua | 94 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 166 insertions(+), 24 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index b7af3d6..52027bd 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -734,20 +734,30 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { } } -static int mp_unpack(lua_State *L) { +static int mp_unpack_full(lua_State *L, int limit, int offset) { size_t len; const unsigned char *s; mp_cur *c; int cnt; /* Number of objects unpacked */ + int decode_all = (!limit && !offset); + + s = (unsigned char*)luaL_checklstring(L,1,&len); /* if no match, exits */ + + if (offset < 0 || limit < 0) /* requesting negative off or lim is invalid */ + return luaL_error(L, + "Invalid request to unpack with offset of %d and limit of %d.", + offset, len); + else if (offset > len) + return luaL_error(L, + "Start offset %d greater than input length %d.", offset, len); - if ((s = (unsigned char*)luaL_checklstring(L,1,&len)) == NULL) - return luaL_argerror(L,1,"MessagePack decoding requires string input."); + if (decode_all) limit = INT_MAX; - c = mp_cur_new(s,len); + c = mp_cur_new(s+offset,len-offset); /* We loop over the decode because this could be a stream * of multiple top-level values serialized together */ - for(cnt = 0; c->left > 0; cnt++) { + for(cnt = 0; c->left > 0 && cnt < limit; cnt++) { mp_decode_to_lua_type(L,c); if (c->err == MP_CUR_ERROR_EOF) { @@ -759,10 +769,47 @@ static int mp_unpack(lua_State *L) { } } + if (!decode_all) { + /* c->left is the remaining size of the input buffer. + * subtract the entire buffer size from the unprocessed size + * to get our next start offset */ + int offset = len - c->left; + /* Return offset -1 when we have have processed the entire buffer. */ + lua_pushinteger(L, c->left == 0 ? -1 : offset); + /* Results are returned with the arg elements still + * in place. Lua takes care of only returning + * elements above the args for us. + * In this case, we have one arg on the stack + * for this function, so we insert our first return + * value at position 2. */ + lua_insert(L, 2); + cnt += 1; /* increase return count by one to make room for offset */ + } + mp_cur_free(c); return cnt; } +static int mp_unpack(lua_State *L) { + return mp_unpack_full(L, 0, 0); +} + +static int mp_unpack_one(lua_State *L) { + int offset = luaL_optint(L, 2, 0); + /* Variable pop because offset may not exist */ + lua_pop(L, lua_gettop(L)-1); + return mp_unpack_full(L, 1, offset); +} + +static int mp_unpack_limit(lua_State *L) { + int limit = luaL_checkint(L, 2); + int offset = luaL_optint(L, 3, 0); + /* Variable pop because offset may not exist */ + lua_pop(L, lua_gettop(L)-1); + + return mp_unpack_full(L, limit, offset); +} + static int mp_safe(lua_State *L) { int argc, err, total_results; @@ -786,18 +833,24 @@ static int mp_safe(lua_State *L) { } /* -------------------------------------------------------------------------- */ -int luaopen_create(lua_State *L) { +static const struct luaL_Reg cmds[] = { + {"pack", mp_pack}, + {"unpack", mp_unpack}, + {"unpack_one", mp_unpack_one}, + {"unpack_limit", mp_unpack_limit}, + {0} +}; + +static int luaopen_create(lua_State *L) { + int i; /* Manually construct our module table instead of * relying on _register or _newlib */ lua_newtable(L); - /* Add unpack function */ - lua_pushcfunction(L, mp_unpack); - lua_setfield(L, -2, "unpack"); - - /* Add pack function */ - lua_pushcfunction(L, mp_pack); - lua_setfield(L, -2, "pack"); + for (i = 0; i < (sizeof(cmds)/sizeof(*cmds) - 1); i++) { + lua_pushcfunction(L, cmds[i].func); + lua_setfield(L, -2, cmds[i].name); + } /* Add metadata */ lua_pushliteral(L, LUACMSGPACK_NAME); @@ -824,17 +877,16 @@ LUALIB_API int luaopen_cmsgpack(lua_State *L) { } LUALIB_API int luaopen_cmsgpack_safe(lua_State *L) { - luaopen_cmsgpack(L); + int i; - /* Add safe wrapper to pack */ - lua_getfield(L, -1, "pack"); - lua_pushcclosure(L, mp_safe, 1); - lua_setfield(L, -2, "pack"); + luaopen_cmsgpack(L); - /* Add safe wrapper to unpack */ - lua_getfield(L, -1, "unpack"); - lua_pushcclosure(L, mp_safe, 1); - lua_setfield(L, -2, "unpack"); + /* Wrap all functions in the safe handler */ + for (i = 0; i < (sizeof(cmds)/sizeof(*cmds) - 1); i++) { + lua_getfield(L, -1, cmds[i].name); + lua_pushcclosure(L, mp_safe, 1); + lua_setfield(L, -2, cmds[i].name); + } #if LUA_VERSION_NUM < 502 /* Register name globally for 5.1 */ diff --git a/test.lua b/test.lua index e5463fa..65d6e2f 100644 --- a/test.lua +++ b/test.lua @@ -7,7 +7,7 @@ local ok, cmsgpack_safe = pcall(require, 'cmsgpack.safe') if not ok then cmsgpack_safe = nil end passed = 0 -failed = 0 +failed = 0 skipped = 0 function hex(s) @@ -117,7 +117,7 @@ function test_circular(name,obj) end function test_stream(mod, name, ...) - io.write("Stream test '", name, "' ...") + io.write("Stream test '", name, "' ...\n") if not mod then print("skip: no `cmsgpack.safe` module") skipped = skipped + 1 @@ -155,6 +155,61 @@ function test_stream(mod, name, ...) end +function test_partial_unpack(name, count, ...) + io.write("Testing partial unpack '",name,"' ...\n") + local first = select(1, ...) + local pack, unpacked, args, offset, cargs, ok, err + if (type(first) == "table") then + pack = first.p + args = first.remaining + offset = first.o + cargs = {pack, count, offset} + else + pack = cmsgpack.pack(unpack({...})) + args = {...} + cargs = {pack, count} + end + if offset and offset < 0 then + ok, unpacked, err = pcall(function()return {cmsgpack.unpack_limit(unpack(cargs))} end) + if not ok then + print("ok; received error as expected") --, unpacked) + passed = passed + 1 + return + end + else + unpacked = {cmsgpack.unpack_limit(unpack(cargs))} + -- print ("GOT RETURNED:", unpack(unpacked)) + end + + if count == 0 and #unpacked == 1 then + print("ok; received zero decodes as expected") + passed = passed + 1 + return + end + + if not (((#unpacked)-1) == count) then + print(string.format("ERROR: received %d instead of %d objects:", (#unpacked)-1, count), + unpack(select(1, unpacked))) + failed = failed + 1 + return + end + + for i=2, #unpacked do + local origin = args[i-1] + --print("Comparing ", origin, unpacked[i]) + if not compare_objects(origin, unpacked[i]) then + print("ERROR:", origin, " not match ", unpacked[i]) + failed = failed + 1 + else + print("ok; matched unpacked value to input") + passed = passed + 1 + end + end + + -- return the packed value and our continue offset + return pack, unpacked[1] +end + function test_pack(name,obj,raw) io.write("Testing encoder '",name,"' ...") if hex(cmsgpack.pack(obj)) ~= raw then @@ -166,6 +221,24 @@ function test_pack(name,obj,raw) end end +function test_unpack_one(name, packed, check, offset) + io.write("Testing one unpack '",name,"' ...") + local unpacked = {cmsgpack.unpack_one(unpack({packed, offset}))} + + if #unpacked > 2 then + print("ERROR: unpacked more than one object:", unpack(unpacked)) + failed = failed + 1 + elseif not compare_objects(unpacked[2], check) then + print("ERROR: unpacked unexpected result:", unpack(unpacked)) + failed = failed + 1 + else + print("ok") --; unpacked", unpacked[2]) + passed = passed + 1 + end + + return unpacked[1] +end + function test_unpack(name,raw,obj) io.write("Testing decoder '",name,"' ...") if not compare_objects(cmsgpack.unpack(unhex(raw)),obj) then @@ -312,6 +385,23 @@ test_stream(cmsgpack_safe, "strange things", nil, {}, {nil}, a, b, b, b, a, a, b test_error("pack nothing", function() cmsgpack.pack() end) test_noerror("pack nothing safe", function() cmsgpack_safe.pack() end) +-- Test limited streaming +packed, offset = test_partial_unpack("unpack 1a out of 7", 1, "a", "b", "c", "d", "e", "f", "g") +packed, offset = test_partial_unpack("unpack 1b of remaining 7", 1, {p=packed,o=offset,remaining={"b"}}) +packed, offset = test_partial_unpack("unpack 1c of remaining 7", 1, {p=packed,o=offset,remaining={"c"}}) +packed, offset = test_partial_unpack("unpack 1d of remaining 7", 1, {p=packed,o=offset,remaining={"d"}}) +packed, offset = test_partial_unpack("unpack 1e of remaining 7", 1, {p=packed,o=offset,remaining={"e"}}) +packed, offset = test_partial_unpack("unpack 1f of remaining 7", 1, {p=packed,o=offset,remaining={"f"}}) +packed, offset = test_partial_unpack("unpack 1g of remaining 7", 1, {p=packed,o=offset,remaining={"g"}}) +packed, offset = test_partial_unpack("unpack 1nil of remaining 7", 0, {p=packed,o=offset}) + +packed, offset = test_partial_unpack("unpack 3 out of 7", 3, "a", "b", "c", "d", "e", "f", "g") +test_partial_unpack("unpack remaining 4", 4, {p=packed,o=offset,remaining={"d", "e", "f", "g"}}) + +test_unpack_one("simple", packed, "a") +offset = test_unpack_one("simple", cmsgpack.pack({f = 3, j = 2}, "m", "e", 7), {f = 3, j = 2}) +test_unpack_one("simple", cmsgpack.pack({f = 3, j = 2}, "m", "e", 7), "m", offset) + -- Final report print() print("TEST PASSED:",passed) From 6deb4496d0c0485d55dbb3296035251a94d4e01c Mon Sep 17 00:00:00 2001 From: Francois Perrad Date: Fri, 2 Nov 2012 10:12:15 +0100 Subject: [PATCH 12/30] Use stack allocation for mp_cur We don't really need malloc here since we use it and collect it all in the same spot without ever needing to resize anything. --- lua_cmsgpack.c | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 52027bd..085ed0d 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -133,17 +133,10 @@ typedef struct mp_cur { int err; } mp_cur; -static mp_cur *mp_cur_new(const unsigned char *s, size_t len) { - mp_cur *cursor = (mp_cur*)malloc(sizeof(*cursor)); - +static void mp_cur_init(mp_cur *cursor, const unsigned char *s, size_t len) { cursor->p = s; cursor->left = len; cursor->err = MP_CUR_ERROR_NONE; - return cursor; -} - -static void mp_cur_free(mp_cur *cursor) { - free(cursor); } #define mp_cur_consume(_c,_len) do { _c->p += _len; _c->left -= _len; } while(0) @@ -736,12 +729,12 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { static int mp_unpack_full(lua_State *L, int limit, int offset) { size_t len; - const unsigned char *s; - mp_cur *c; + const char *s; + mp_cur c; int cnt; /* Number of objects unpacked */ int decode_all = (!limit && !offset); - s = (unsigned char*)luaL_checklstring(L,1,&len); /* if no match, exits */ + s = luaL_checklstring(L,1,&len); /* if no match, exits */ if (offset < 0 || limit < 0) /* requesting negative off or lim is invalid */ return luaL_error(L, @@ -753,18 +746,16 @@ static int mp_unpack_full(lua_State *L, int limit, int offset) { if (decode_all) limit = INT_MAX; - c = mp_cur_new(s+offset,len-offset); + mp_cur_init(&c,(const unsigned char *)s+offset,len-offset); /* We loop over the decode because this could be a stream * of multiple top-level values serialized together */ - for(cnt = 0; c->left > 0 && cnt < limit; cnt++) { - mp_decode_to_lua_type(L,c); + for(cnt = 0; c.left > 0 && cnt < limit; cnt++) { + mp_decode_to_lua_type(L,&c); - if (c->err == MP_CUR_ERROR_EOF) { - mp_cur_free(c); + if (c.err == MP_CUR_ERROR_EOF) { return luaL_error(L,"Missing bytes in input."); - } else if (c->err == MP_CUR_ERROR_BADFMT) { - mp_cur_free(c); + } else if (c.err == MP_CUR_ERROR_BADFMT) { return luaL_error(L,"Bad data format in input."); } } @@ -773,9 +764,9 @@ static int mp_unpack_full(lua_State *L, int limit, int offset) { /* c->left is the remaining size of the input buffer. * subtract the entire buffer size from the unprocessed size * to get our next start offset */ - int offset = len - c->left; + int offset = len - c.left; /* Return offset -1 when we have have processed the entire buffer. */ - lua_pushinteger(L, c->left == 0 ? -1 : offset); + lua_pushinteger(L, c.left == 0 ? -1 : offset); /* Results are returned with the arg elements still * in place. Lua takes care of only returning * elements above the args for us. @@ -786,7 +777,6 @@ static int mp_unpack_full(lua_State *L, int limit, int offset) { cnt += 1; /* increase return count by one to make room for offset */ } - mp_cur_free(c); return cnt; } From 29c78eaf58dc34d93e8a3422b83d0bec1eec4bae Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 6 Apr 2014 15:09:00 -0400 Subject: [PATCH 13/30] Add large pack test We need a test to pack something larger than the default allocation buffer. --- test.lua | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/test.lua b/test.lua index 65d6e2f..49643f6 100644 --- a/test.lua +++ b/test.lua @@ -384,6 +384,121 @@ test_stream(cmsgpack, "strange things", nil, {}, {nil}, a, b, b, b, a, a, b, {c test_stream(cmsgpack_safe, "strange things", nil, {}, {nil}, a, b, b, b, a, a, b, {c = a, d = b}) test_error("pack nothing", function() cmsgpack.pack() end) test_noerror("pack nothing safe", function() cmsgpack_safe.pack() end) +test_circular("large object test", + {A=9483, a=9483, aa=9483, aal=9483, aalii=9483, aam=9483, Aani=9483, + aardvark=9483, aardwolf=9483, Aaron=9483, Aaronic=9483, Aaronical=9483, + Aaronite=9483, Aaronitic=9483, Aaru=9483, Ab=9483, aba=9483, Ababdeh=9483, + Ababua=9483, abac=9483, abaca=9483, abacate=9483, abacay=9483, + abacinate=9483, abacination=9483, abaciscus=9483, abacist=9483, + aback=9483, abactinal=9483, abactinally=9483, abaction=9483, abactor=9483, + abaculus=9483, abacus=9483, Abadite=9483, abaff=9483, abaft=9483, + abaisance=9483, abaiser=9483, abaissed=9483, abalienate=9483, + abalienation=9483, abalone=9483, Abama=9483, abampere=9483, abandon=9483, + abandonable=9483, abandoned=9483, abandonedly=9483, abandonee=9483, + abandoner=9483, abandonment=9483, Abanic=9483, Abantes=9483, + abaptiston=9483, Abarambo=9483, Abaris=9483, abarthrosis=9483, + abarticular=9483, abarticulation=9483, abas=9483, abase=9483, abased=9483, + abasedly=9483, abasedness=9483, abasement=9483, abaser=9483, Abasgi=9483, + abash=9483, abashed=9483, abashedly=9483, abashedness=9483, + abashless=9483, abashlessly=9483, abashment=9483, abasia=9483, + abasic=9483, abask=9483, Abassin=9483, abastardize=9483, abatable=9483, + abate=9483, abatement=9483, abater=9483, abatis=9483, abatised=9483, + abaton=9483, abator=9483, abattoir=9483, Abatua=9483, abature=9483, + abave=9483, abaxial=9483, abaxile=9483, abaze=9483, abb=9483, Abba=9483, + abbacomes=9483, abbacy=9483, Abbadide=9483, abbas=9483, abbasi=9483, + abbassi=9483, Abbasside=9483, abbatial=9483, abbatical=9483, abbess=9483, + abbey=9483, abbeystede=9483, Abbie=9483, abbot=9483, abbotcy=9483, + abbotnullius=9483, abbotship=9483, abbreviate=9483, abbreviately=9483, + abbreviation=9483, abbreviator=9483, abbreviatory=9483, abbreviature=9483, + Abby=9483, abcoulomb=9483, abdal=9483, abdat=9483, Abderian=9483, + Abderite=9483, abdest=9483, abdicable=9483, abdicant=9483, abdicate=9483, + abdication=9483, abdicative=9483, abdicator=9483, Abdiel=9483, + abditive=9483, abditory=9483, abdomen=9483, abdominal=9483, + Abdominales=9483, abdominalian=9483, abdominally=9483, + abdominoanterior=9483, abdominocardiac=9483, abdominocentesis=9483, + abdominocystic=9483, abdominogenital=9483, abdominohysterectomy=9483, + abdominohysterotomy=9483, abdominoposterior=9483, abdominoscope=9483, + abdominoscopy=9483, abdominothoracic=9483, abdominous=9483, + abdominovaginal=9483, abdominovesical=9483, abduce=9483, abducens=9483, + abducent=9483, abduct=9483, abduction=9483, abductor=9483, Abe=9483, + abeam=9483, abear=9483, abearance=9483, abecedarian=9483, + abecedarium=9483, abecedary=9483, abed=9483, abeigh=9483, Abel=9483, + abele=9483, Abelia=9483, Abelian=9483, Abelicea=9483, Abelite=9483, + abelite=9483, Abelmoschus=9483, abelmosk=9483, Abelonian=9483, + abeltree=9483, Abencerrages=9483, abenteric=9483, abepithymia=9483, + Aberdeen=9483, aberdevine=9483, Aberdonian=9483, Aberia=9483, + aberrance=9483, aberrancy=9483, aberrant=9483, aberrate=9483, + aberration=9483, aberrational=9483, aberrator=9483, aberrometer=9483, + aberroscope=9483, aberuncator=9483, abet=9483, abetment=9483, + abettal=9483, abettor=9483, abevacuation=9483, abey=9483, abeyance=9483, + abeyancy=9483, abeyant=9483, abfarad=9483, abhenry=9483, abhiseka=9483, + abhominable=9483, abhor=9483, abhorrence=9483, abhorrency=9483, + abhorrent=9483, abhorrently=9483, abhorrer=9483, abhorrible=9483, + abhorring=9483, Abhorson=9483, abidal=9483, abidance=9483, abide=9483, + abider=9483, abidi=9483, abiding=9483, abidingly=9483, abidingness=9483, + Abie=9483, Abies=9483, abietate=9483, abietene=9483, abietic=9483, + abietin=9483, Abietineae=9483, abietineous=9483, abietinic=9483, + Abiezer=9483, Abigail=9483, abigail=9483, abigailship=9483, abigeat=9483, + abigeus=9483, abilao=9483, ability=9483, abilla=9483, abilo=9483, + abintestate=9483, abiogenesis=9483, abiogenesist=9483, abiogenetic=9483, + abiogenetical=9483, abiogenetically=9483, abiogenist=9483, + abiogenous=9483, abiogeny=9483, abiological=9483, abiologically=9483, + abiology=9483, abiosis=9483, abiotic=9483, abiotrophic=9483, + abiotrophy=9483, Abipon=9483, abir=9483, abirritant=9483, abirritate=9483, + abirritation=9483, abirritative=9483, abiston=9483, Abitibi=9483, + abiuret=9483, abject=9483, abjectedness=9483, abjection=9483, + abjective=9483, abjectly=9483, abjectness=9483, abjoint=9483, + abjudge=9483, abjudicate=9483, abjudication=9483, abjunction=9483, + abjunctive=9483, abjuration=9483, abjuratory=9483, abjure=9483, + abjurement=9483, abjurer=9483, abkar=9483, abkari=9483, Abkhas=9483, + Abkhasian=9483, ablach=9483, ablactate=9483, ablactation=9483, + ablare=9483, ablastemic=9483, ablastous=9483, ablate=9483, ablation=9483, + ablatitious=9483, ablatival=9483, ablative=9483, ablator=9483, + ablaut=9483, ablaze=9483, able=9483, ableeze=9483, ablegate=9483, + ableness=9483, ablepharia=9483, ablepharon=9483, ablepharous=9483, + Ablepharus=9483, ablepsia=9483, ableptical=9483, ableptically=9483, + abler=9483, ablest=9483, ablewhackets=9483, ablins=9483, abloom=9483, + ablow=9483, ablude=9483, abluent=9483, ablush=9483, ablution=9483, + ablutionary=9483, abluvion=9483, ably=9483, abmho=9483, Abnaki=9483, + abnegate=9483, abnegation=9483, abnegative=9483, abnegator=9483, + Abner=9483, abnerval=9483, abnet=9483, abneural=9483, abnormal=9483, + abnormalism=9483, abnormalist=9483, abnormality=9483, abnormalize=9483, + abnormally=9483, abnormalness=9483, abnormity=9483, abnormous=9483, + abnumerable=9483, Abo=9483, aboard=9483, Abobra=9483, abode=9483, + abodement=9483, abody=9483, abohm=9483, aboil=9483, abolish=9483, + abolisher=9483, abolishment=9483, abolition=9483, abolitionary=9483, + abolitionism=9483, abolitionist=9483, abolitionize=9483, abolla=9483, + aboma=9483, abomasum=9483, abomasus=9483, abominable=9483, + abominableness=9483, abominably=9483, abominate=9483, abomination=9483, + abominator=9483, abomine=9483, Abongo=9483, aboon=9483, aborad=9483, + aboral=9483, aborally=9483, abord=9483, aboriginal=9483, + aboriginality=9483, aboriginally=9483, aboriginary=9483, aborigine=9483, + abort=9483, aborted=9483, aborticide=9483, abortient=9483, + abortifacient=9483, abortin=9483, abortion=9483, abortional=9483, + abortionist=9483, abortive=9483, abortively=9483, abortiveness=9483, + abortus=9483, abouchement=9483, abound=9483, abounder=9483, + abounding=9483, aboundingly=9483, about=9483, abouts=9483, above=9483, + aboveboard=9483, abovedeck=9483, aboveground=9483, aboveproof=9483, + abovestairs=9483, abox=9483, abracadabra=9483, abrachia=9483, + abradant=9483, abrade=9483, abrader=9483, Abraham=9483, Abrahamic=9483, + Abrahamidae=9483, Abrahamite=9483, Abrahamitic=9483, abraid=9483, + Abram=9483, Abramis=9483, abranchial=9483, abranchialism=9483, + abranchian=9483, Abranchiata=9483, abranchiate=9483, abranchious=9483, + abrasax=9483, abrase=9483, abrash=9483, abrasiometer=9483, abrasion=9483, + abrasive=9483, abrastol=9483, abraum=9483, abraxas=9483, abreact=9483, + abreaction=9483, abreast=9483, abrenounce=9483, abret=9483, abrico=9483, + abridge=9483, abridgeable=9483, abridged=9483, abridgedly=9483, + abridger=9483, abridgment=9483, abrim=9483, abrin=9483, abristle=9483, + abroach=9483, abroad=9483, Abrocoma=9483, abrocome=9483, abrogable=9483, + abrogate=9483, abrogation=9483, abrogative=9483, abrogator=9483, + Abroma=9483, Abronia=9483, abrook=9483, abrotanum=9483, abrotine=9483, + abrupt=9483, abruptedly=9483, abruption=9483, abruptly=9483, + abruptness=9483, Abrus=9483, Absalom=9483, absampere=9483, Absaroka=9483, + absarokite=9483, abscess=9483, abscessed=9483, abscession=9483, + abscessroot=9483, abscind=9483, abscise=9483, abscision=9483, + absciss=9483, abscissa=9483, abscissae=9483, abscisse=9483, + abscission=9483, absconce=9483, abscond=9483, absconded=9483, + abscondedly=9483, abscondence=9483}) -- Test limited streaming packed, offset = test_partial_unpack("unpack 1a out of 7", 1, "a", "b", "c", "d", "e", "f", "g") From e93d977e9143a45701f658df308dd6c5fca13798 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 6 Apr 2014 15:23:00 -0400 Subject: [PATCH 14/30] Unpack integers as integers instead of numbers This fixes Lua 5.3 compatability and works fine on Lua 5.1 too. Lua 5.3 has an actual 64-bit Integer type, so we need to use _pushinteger instead of _pushnumber so our unpacked integers don't convert to floating point values. For older versions, lua_pushinteger casts input to the default numeric type and behavior is unchanged. Lua 5.3 also has an unsigned integer type, so I added a compatibility macro to allow us to properly use lua_pushunsigned on 5.3+ with a fallback to the regular lua_pushinteger on other versions. --- lua_cmsgpack.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 085ed0d..7ab9ffc 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -32,6 +32,10 @@ #define IS_INT64_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int64_t) #define IS_INT_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int) +#if LUA_VERSION_NUM < 503 + #define lua_pushunsigned(L, n) lua_pushinteger(L, n) +#endif + /* ============================================================================= * MessagePack implementation and bindings for Lua 5.1/5.2. * Copyright(C) 2012 Salvatore Sanfilippo @@ -542,31 +546,31 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { switch(c->p[0]) { case 0xcc: /* uint 8 */ mp_cur_need(c,2); - lua_pushnumber(L,c->p[1]); + lua_pushunsigned(L,c->p[1]); mp_cur_consume(c,2); break; case 0xd0: /* int 8 */ mp_cur_need(c,2); - lua_pushnumber(L,(char)c->p[1]); + lua_pushinteger(L,(char)c->p[1]); mp_cur_consume(c,2); break; case 0xcd: /* uint 16 */ mp_cur_need(c,3); - lua_pushnumber(L, + lua_pushunsigned(L, (c->p[1] << 8) | c->p[2]); mp_cur_consume(c,3); break; case 0xd1: /* int 16 */ mp_cur_need(c,3); - lua_pushnumber(L,(int16_t) + lua_pushinteger(L,(int16_t) (c->p[1] << 8) | c->p[2]); mp_cur_consume(c,3); break; case 0xce: /* uint 32 */ mp_cur_need(c,5); - lua_pushnumber(L, + lua_pushunsigned(L, ((uint32_t)c->p[1] << 24) | ((uint32_t)c->p[2] << 16) | ((uint32_t)c->p[3] << 8) | @@ -575,7 +579,7 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xd2: /* int 32 */ mp_cur_need(c,5); - lua_pushnumber(L, + lua_pushinteger(L, ((int32_t)c->p[1] << 24) | ((int32_t)c->p[2] << 16) | ((int32_t)c->p[3] << 8) | @@ -584,7 +588,7 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xcf: /* uint 64 */ mp_cur_need(c,9); - lua_pushnumber(L, + lua_pushunsigned(L, ((uint64_t)c->p[1] << 56) | ((uint64_t)c->p[2] << 48) | ((uint64_t)c->p[3] << 40) | @@ -597,7 +601,7 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xd3: /* int 64 */ mp_cur_need(c,9); - lua_pushnumber(L, + lua_pushinteger(L, ((int64_t)c->p[1] << 56) | ((int64_t)c->p[2] << 48) | ((int64_t)c->p[3] << 40) | @@ -703,10 +707,10 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; default: /* types that can't be idenitified by first byte value. */ if ((c->p[0] & 0x80) == 0) { /* positive fixnum */ - lua_pushnumber(L,c->p[0]); + lua_pushunsigned(L,c->p[0]); mp_cur_consume(c,1); } else if ((c->p[0] & 0xe0) == 0xe0) { /* negative fixnum */ - lua_pushnumber(L,(signed char)c->p[0]); + lua_pushinteger(L,(signed char)c->p[0]); mp_cur_consume(c,1); } else if ((c->p[0] & 0xe0) == 0xa0) { /* fix raw */ size_t l = c->p[0] & 0x1f; From acb2f79b894766c7cd2d9e016a8be7bb85596cb8 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 6 Apr 2014 16:54:01 -0400 Subject: [PATCH 15/30] Add better testing for verifying serializations Lua 5.3 doesn't iterate tables in any specific order, so the resulting msgpack may vary when using associative tables with more than one element. I've only seen two variations on the the regression test output so far, so tracking those are still simple. If they get wildly out of sync and start generating new msgpack on every run, we may want to drop the exact output verification check. The recursive regression test now has a circular test too, so even if the output doesn't match, we can verify the unpacked result matches the input result (to a fixed depth). As of this commit, all tests pass on Lua 5.1 and Lua 5.3: TEST PASSED: 226 TEST FAILED: 0 TEST SKIPPED: 0 --- test.lua | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test.lua b/test.lua index 49643f6..160f4dc 100644 --- a/test.lua +++ b/test.lua @@ -210,9 +210,13 @@ function test_partial_unpack(name, count, ...) return pack, unpacked[1] end -function test_pack(name,obj,raw) +function test_pack(name,obj,raw,optraw) io.write("Testing encoder '",name,"' ...") - if hex(cmsgpack.pack(obj)) ~= raw then + local result = hex(cmsgpack.pack(obj)) + if optraw and (result == optraw) then + print("ok") + passed = passed + 1 + elseif result ~= raw then print("ERROR:", obj, hex(cmsgpack.pack(obj)), raw) failed = failed+1 else @@ -359,8 +363,11 @@ a = {x=nil,y=5} b = {x=a} a['x'] = b pack = cmsgpack.pack(a) -test_pack("regression for issue #4",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0") -test_circular("regression for issue #4",a) +-- Note: the generated result isn't stable because the order of traversal for +-- a table isn't defined. So far we've only noticed two serializations of a +-- (and the second serialization only happens on Lua 5.3 sometimes) +test_pack("regression for issue #4 output matching",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0", "82a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a178c0a17905a17905a17905a17905a17905a17905a17905a17905") +test_circular("regression for issue #4 circular",a) -- Tests from github.com/moteus test_circular("map with number keys", {[1] = {1,2,3}}) From 4ca832bf153154bac85328791c8888a1777b88ee Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 6 Apr 2014 21:27:45 -0400 Subject: [PATCH 16/30] Malloc one buffer per pack() instead of per arg We can reuse the same malloc'd buffer for encoding multiple arguments. Just tell the buffer all current contents is now free space and reset the current offset back to zero. We now only malloc once per pack() call instead of argc times per call, giving us better performance with large argument counts. All 226 tests pass. --- lua_cmsgpack.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 7ab9ffc..bf8119f 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -488,24 +488,28 @@ static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { static int mp_pack(lua_State *L) { int nargs = lua_gettop(L); int i; + mp_buf *buf; if (nargs == 0) return luaL_argerror(L, 0, "MessagePack pack needs input."); - /* Create nargs packed buffers */ + buf = mp_buf_new(); for(i = 1; i <= nargs; i++) { - mp_buf *buf; - /* Copy argument i to top of stack for _encode processing; * the encode function pops it from the stack when complete. */ lua_pushvalue(L, i); - buf = mp_buf_new(); mp_encode_lua_type(L,buf,0); lua_pushlstring(L,(char*)buf->b,buf->len); - mp_buf_free(buf); + + /* Reuse the buffer for the next operation by + * setting its free count to the total buffer size + * and the current position to zero. */ + buf->free += buf->len; + buf->len = 0; } + mp_buf_free(buf); /* Concatenate all nargs buffers together */ lua_concat(L, nargs); From 067a2581cd482d5ecab38171ea25a5041006d940 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 6 Apr 2014 22:14:08 -0400 Subject: [PATCH 17/30] Use Lua memory allocator instead of malloc cmsgpack is a much better Lua citizen now because we use the memory allocator assigned to our lua_State instead of grabbing memory from the system ourselves. Typically the memory allocator is just the system's realloc, but some use cases involve providing custom allocation routines (for accounting or performance or limiting memory). This closes #20 too because those commits were just trying to remove the previous direct allocation behavior. All tests pass under Lua 5.1 and Lua 5.3-work2. --- lua_cmsgpack.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index bf8119f..e652760 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -89,23 +89,37 @@ static void memrevifle(void *ptr, size_t len) { * behavior. */ typedef struct mp_buf { + lua_State *L; unsigned char *b; size_t len, free; } mp_buf; -static mp_buf *mp_buf_new(void) { - mp_buf *buf = (mp_buf*)malloc(sizeof(*buf)); +static void *mp_realloc(lua_State *L, void *target, size_t osize,size_t nsize) { + void *(*local_realloc) (void *, void *, size_t osize, size_t nsize) = NULL; + void *ud; + local_realloc = lua_getallocf(L, &ud); + + return local_realloc(ud, target, osize, nsize); +} + +static mp_buf *mp_buf_new(lua_State *L) { + mp_buf *buf = NULL; + + /* Old size = 0; new size = sizeof(*buf) */ + buf = (mp_buf*)mp_realloc(L, NULL, 0, sizeof(*buf)); + + buf->L = L; buf->b = NULL; buf->len = buf->free = 0; return buf; } -void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { +static void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { if (buf->free < len) { size_t newlen = buf->len+len; - buf->b = (unsigned char*)realloc(buf->b,newlen*2); + buf->b = (unsigned char*)mp_realloc(buf->L, buf->b, buf->len, newlen*2); buf->free = newlen; } memcpy(buf->b+buf->len,s,len); @@ -114,8 +128,8 @@ void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { } void mp_buf_free(mp_buf *buf) { - free(buf->b); - free(buf); + mp_realloc(buf->L, buf->b, buf->len, 0); /* realloc to 0 = free */ + mp_realloc(buf->L, buf, sizeof(*buf), 0); } /* ---------------------------- String cursor ---------------------------------- @@ -493,7 +507,7 @@ static int mp_pack(lua_State *L) { if (nargs == 0) return luaL_argerror(L, 0, "MessagePack pack needs input."); - buf = mp_buf_new(); + buf = mp_buf_new(L); for(i = 1; i <= nargs; i++) { /* Copy argument i to top of stack for _encode processing; * the encode function pops it from the stack when complete. */ From 72e64c2b6d92f9dd4eb7de135644ee65f80545de Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 7 Apr 2014 02:41:50 -0400 Subject: [PATCH 18/30] Fix potential segfault If more than ~20 objects were being returned at once, Lua would segfault because the default stack size is 20 and nobody was resizing the stack. Now we can return up to ~8,000 objects at once before erroring out the function properly instead of segfaulting. Also added test for segfault. All tests currently pass. --- lua_cmsgpack.c | 10 ++++++++++ test.lua | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index e652760..dbf99b1 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -561,6 +561,16 @@ void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) { * a Lua type, that is left as the only result on the stack. */ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { mp_cur_need(c,1); + + /* If we return more than 18 elements, we must resize the stack to + * fit all our return values. But, there is no way to + * determine how many objects a msgpack will unpack to up front, so + * we request a +1 larger stack on each iteration (noop if stack is + * big enough, and when stack does require resize it doubles in size) */ + luaL_checkstack(L, 1, + "too many return values at once; " + "use unpack_one or unpack_limit instead."); + switch(c->p[0]) { case 0xcc: /* uint 8 */ mp_cur_need(c,2); diff --git a/test.lua b/test.lua index 160f4dc..84f6cac 100644 --- a/test.lua +++ b/test.lua @@ -369,6 +369,10 @@ pack = cmsgpack.pack(a) test_pack("regression for issue #4 output matching",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0", "82a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a178c0a17905a17905a17905a17905a17905a17905a17905a17905") test_circular("regression for issue #4 circular",a) +-- test unpacking malformed input without crashing. This actually returns one integer value (the ASCII code) +-- for each character in the string. We don't care about the return value, just that we don't segfault. +cmsgpack.unpack("82a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a17881a17") + -- Tests from github.com/moteus test_circular("map with number keys", {[1] = {1,2,3}}) test_circular("map with float keys", {[1.5] = {1,2,3}}) From 8692d79ef2d08c5eb06ed7723284fc63b990ddd3 Mon Sep 17 00:00:00 2001 From: Alexey Melnichuk Date: Fri, 11 Apr 2014 15:54:13 +0500 Subject: [PATCH 19/30] Add tests to detect arrays with string keys --- test.lua | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test.lua b/test.lua index 84f6cac..3912084 100644 --- a/test.lua +++ b/test.lua @@ -303,6 +303,19 @@ local function test_array() print("ok") passed = passed+1 end + + io.write("Testing array detection ...") + + a = {["1"] = 20, [2] = 30, [3] = 40} + encode = cmsgpack.pack(a) + if etalon == encode then + print("ERROR:") + print("", " incorrect: ", hex(etalon)) + failed = failed+1 + else + print("ok") + passed = passed+1 + end end test_global() @@ -375,6 +388,8 @@ cmsgpack.unpack("82a17881a17882a17881a17882a17881a17882a17881a17882a17881a17882a -- Tests from github.com/moteus test_circular("map with number keys", {[1] = {1,2,3}}) +test_circular("map with string keys", {["1"] = {1,2,3}}) +test_circular("map with string keys", {["1"] = 20, [2] = 30, ["3"] = 40}) test_circular("map with float keys", {[1.5] = {1,2,3}}) test_error("unpack nil", function() cmsgpack.unpack(nil) end) test_error("unpack table", function() cmsgpack.unpack({}) end) From 351d38e5fe745e519d2d97cb1007220abb3792c7 Mon Sep 17 00:00:00 2001 From: Alexey Melnichuk Date: Fri, 11 Apr 2014 15:59:36 +0500 Subject: [PATCH 20/30] Fix array detection when keys are strings --- lua_cmsgpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index dbf99b1..acdae4e 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -431,7 +431,7 @@ static int table_is_an_array(lua_State *L) { lua_pop(L,1); /* Stack: ... key */ /* The <= 0 check is valid here because we're comparing indexes. */ #if LUA_VERSION_NUM < 503 - if (!lua_isnumber(L,-1) || (n = lua_tonumber(L, -1)) <= 0 || + if ((LUA_TNUMBER != lua_type(L,-1)) || (n = lua_tonumber(L, -1)) <= 0 || !IS_INT_EQUIVALENT(n)) { #else if (!lua_isinteger(L,-1) || (n = lua_tointeger(L, -1)) <= 0) { From 2820da33691b19624287e2637540377092de4036 Mon Sep 17 00:00:00 2001 From: Dmitry Chestnykh Date: Wed, 9 Apr 2014 15:53:40 +0200 Subject: [PATCH 21/30] Fix comment typos Closes #31 --- lua_cmsgpack.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index acdae4e..2b3038b 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -71,7 +71,7 @@ static void memrevifle(void *ptr, size_t len) { int test = 1; unsigned char *testp = (unsigned char*) &test; - if (testp[0] == 0) return; /* Big endian, nothign to do. */ + if (testp[0] == 0) return; /* Big endian, nothing to do. */ len /= 2; while(len--) { aux = *p; @@ -83,7 +83,7 @@ static void memrevifle(void *ptr, size_t len) { } /* ---------------------------- String buffer ---------------------------------- - * This is a simple implementation of string buffers. The only opereation + * This is a simple implementation of string buffers. The only operation * supported is creating empty buffers and appending bytes to it. * The string buffer uses 2x preallocation on every realloc for O(N) append * behavior. */ @@ -142,7 +142,7 @@ void mp_buf_free(mp_buf *buf) { * be used to report errors. */ #define MP_CUR_ERROR_NONE 0 -#define MP_CUR_ERROR_EOF 1 /* Not enough data to complete opereation. */ +#define MP_CUR_ERROR_EOF 1 /* Not enough data to complete operation. */ #define MP_CUR_ERROR_BADFMT 2 /* Bad data format */ typedef struct mp_cur { @@ -159,7 +159,7 @@ static void mp_cur_init(mp_cur *cursor, const unsigned char *s, size_t len) { #define mp_cur_consume(_c,_len) do { _c->p += _len; _c->left -= _len; } while(0) -/* When there is not enough room we set an error in the cursor and return, this +/* When there is not enough room we set an error in the cursor and return. This * is very common across the code so we have a macro to make the code look * a bit simpler. */ #define mp_cur_need(_c,_len) do { \ @@ -391,7 +391,7 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { /* First step: count keys into table. No other way to do it with the * Lua API, we need to iterate a first time. Note that an alternative * would be to do a single run, and then hack the buffer to insert the - * map opcodes for message pack. Too hachish for this lib. */ + * map opcodes for message pack. Too hackish for this lib. */ lua_pushnil(L); while(lua_next(L,-2)) { lua_pop(L,1); /* remove value, keep key for next iteration. */ @@ -472,7 +472,7 @@ static void mp_encode_lua_null(lua_State *L, mp_buf *buf) { static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { int t = lua_type(L,-1); - /* Limit the encoding of nested tables to a specfiied maximum depth, so that + /* Limit the encoding of nested tables to a specified maximum depth, so that * we survive when called against circular references in tables. */ if (t == LUA_TTABLE && level == LUACMSGPACK_MAX_NESTING) t = LUA_TNIL; switch(t) { From ef6b6180b09ea719cdc38afa61775066d3a57c7f Mon Sep 17 00:00:00 2001 From: Alexey Melnichuk Date: Mon, 14 Apr 2014 15:23:44 +0500 Subject: [PATCH 22/30] Fix Lua 5.2 compatability by using table.unpack 'unpack' is only on 5.1, in 5.2 it got moved to 'table.unpack' This line gives us the proper 'unpack' based on what's in our global namespace. --- test.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test.lua b/test.lua index 3912084..000e9d4 100644 --- a/test.lua +++ b/test.lua @@ -6,6 +6,8 @@ local cmsgpack = require "cmsgpack" local ok, cmsgpack_safe = pcall(require, 'cmsgpack.safe') if not ok then cmsgpack_safe = nil end +local unpack = unpack or table.unpack + passed = 0 failed = 0 skipped = 0 From 450526380144d99be62eb580b460fc95e7cd5af5 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 7 Apr 2014 00:54:17 -0400 Subject: [PATCH 23/30] Bump to version 0.4.0 Changes: - Improve table vs. array detection - Improve packing +-inf - Add multiple pack/unpack support - Add cmsgpack.safe module variant - Add local build infrastructure for easier testing - Add user-controlled unpack support limiting returned objects - Add Lua 5.3 compatibility - Remove an unnecessary malloc - Use Lua memory allocator instead of malloc for buffer creation Issues involved: - closes #16 - allow multi pack/unpack by default - closes #10 - unpack one/limit API added - closes #13 and closes #20 - use Lua allocator - closes #15 - (included in #16) - ignores #22 because it's confusing - closes #23 - fixed elsewhere - closes #26 - extracted some useful parts from a difficult commit - closes #28 - we started tagging versions again recently - closes #27 - that failure case works for me now - closes #31 - fix comment typos I merged commits with original author information where possible, but each commit required manual cleanup of one or more of: formatting fixes (no tabs, please), commit message fixes (more details please), extracting contents from a single 300 line commit with 5 different logical changes merged together, and general correctness checking after merging with newer code. As of this commit, all tests pass on Lua 5.1.5 and Lua 5.3-work2. --- README.md | 41 +++++++++++++++++++++++--- lua_cmsgpack.c | 3 +- rockspec/lua-cmsgpack-0.4.0-0.rockspec | 25 ++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 rockspec/lua-cmsgpack-0.4.0-0.rockspec diff --git a/README.md b/README.md index 5404856..7789f44 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ README for lua-cmsgpack.c === Lua-cmsgpack is a [MessagePack](http://msgpack.org) implementation and bindings for -Lua 5.1/5.2 in a self contained C file without external dependencies. +Lua 5.1/5.2/5.3 in a self contained C file without external dependencies. This library is open source software licensed under the BSD two-clause license. @@ -33,10 +33,33 @@ interpreter: USAGE --- -The exported API is very simple, consisting in two functions: +The exported API is very simple, consisting of four functions: - msgpack = cmsgpack.pack(lua_object) - lua_object = cmsgpack.unpack(msgpack) +Basic API: + + msgpack = cmsgpack.pack(lua_object1, lua_object2, ..., lua_objectN) + lua_object1, lua_object2, ..., lua_objectN = cmsgpack.unpack(msgpack) + +Detailed API giving you more control over unpacking multiple values: + + resume_offset, lua_object1 = cmsgpack.unpack_one(msgpack) + resume_offset1, lua_object2 = cmsgpack.unpack_one(msgpack, resume_offset) + ... + -1, lua_objectN = cmsgpack.unpack_one(msgpack, resume_offset_previous) + + resume_offset, lua_object1, lua_object2 = cmsgpack.unpack_limit(msgpack, 2) + resume_offset2, lua_object3 = cmsgpack.unpack_limit(msgpack, 1, resume_offset1) + +Functions: + + - `pack(arg1, arg2, ..., argn)` - pack any number of lua objects into one msgpack stream. returns: msgpack + - `unpack(msgpack)` - unpack all objects in msgpack to individual return values. returns: object1, object2, ..., objectN + - `unpack_one(msgpack); unpack_one(msgpack, offset)` - unpacks the first object after offset. returns: offset, object + - `unpack_limit(msgpack, limit); unpack_limit(msgpack, limit, offset)` - unpacks the first `limit` objects and returns: offset, object1, objet2, ..., objectN (up to limit, but may return fewer than limit if not that many objects remain to be unpacked) + +When you reach the end of your input stream with `unpack_one` or `unpack_limit`, an offset of `-1` is returned. + +You may `require "msgpack"` or you may `require "msgpack.safe"`. The safe version returns errors as (nil, errstring). However because of the nature of Lua numerical and table type a few behavior of the library must be well understood to avoid problems: @@ -50,6 +73,16 @@ maps. * When a Lua number is converted to float or double, the former is preferred if there is no loss of precision compared to the double representation. * When a MessagePack big integer (64 bit) is converted to a Lua number it is possible that the resulting number will not represent the original number but just an approximation. This is unavoidable because the Lua numerical type is usually a double precision floating point type. +TESTING +--- + +Build and test: + + mkdir build; cd build + cmake .. + make + lua ../test.lua + NESTED TABLES --- Nested tables are handled correctly up to `LUACMSGPACK_MAX_NESTING` levels of diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 2b3038b..6b9e9d1 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -9,7 +9,7 @@ #define LUACMSGPACK_NAME "cmsgpack" #define LUACMSGPACK_SAFE_NAME "cmsgpack_safe" -#define LUACMSGPACK_VERSION "lua-cmsgpack 0.3.1" +#define LUACMSGPACK_VERSION "lua-cmsgpack 0.4.0" #define LUACMSGPACK_COPYRIGHT "Copyright (C) 2012, Salvatore Sanfilippo" #define LUACMSGPACK_DESCRIPTION "MessagePack C implementation for Lua" @@ -53,6 +53,7 @@ * 20-Feb-2012 (ver 0.2.1): Minor bug fixing. * 20-Feb-2012 (ver 0.3.0): Module renamed lua-cmsgpack (was lua-msgpack). * 04-Apr-2014 (ver 0.3.1): Lua 5.2 support and minor bug fix. + * 07-Apr-2014 (ver 0.4.0): Multiple pack/unpack, lua allocator, efficiency. * ========================================================================== */ /* -------------------------- Endian conversion -------------------------------- diff --git a/rockspec/lua-cmsgpack-0.4.0-0.rockspec b/rockspec/lua-cmsgpack-0.4.0-0.rockspec new file mode 100644 index 0000000..30552a1 --- /dev/null +++ b/rockspec/lua-cmsgpack-0.4.0-0.rockspec @@ -0,0 +1,25 @@ +package = "lua-cmsgpack" +version = "0.4.0-0" +source = { + url = "git://github.com/antirez/lua-cmsgpack.git", + tag = "0.4.0" +} +description = { + summary = "MessagePack C implementation and bindings for Lua 5.1/5.2/5.3", + homepage = "http://github.com/antirez/lua-cmsgpack", + license = "Two-clause BSD", + maintainer = "Salvatore Sanfilippo " +} +dependencies = { + "lua >= 5.1" +} +build = { + type = "builtin", + modules = { + cmsgpack = { + sources = { + "lua_cmsgpack.c" + } + } + } +} From 17f2606a77643b3707394aedefe8c33c93c8c37c Mon Sep 17 00:00:00 2001 From: Alexey Melnichuk Date: Fri, 23 May 2014 15:25:57 +0500 Subject: [PATCH 24/30] Add Travis CI --- .travis.yml | 29 +++++++++++++++++++++++++ .travis/setup_lua.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 .travis.yml create mode 100644 .travis/setup_lua.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0188fcd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +language: erlang + +env: + global: + - PLATFORM=linux + - LUAROCKS_VER=2.1.0 + matrix: + - LUA=lua5.1 LUA_SFX= + - LUA=lua5.2 LUA_SFX= + - LUA=luajit LUA_SFX=jit + - LUA=lua5.3 LUA_SFX= + +before_install: + - bash .travis/setup_lua.sh + - sudo pip install cpp-coveralls + +install: + - sudo luarocks make rockspec/lua-cmsgpack-scm-1.rockspec CFLAGS="-O2 -fPIC -ftest-coverage -fprofile-arcs" LIBFLAG="-shared --coverage" + +script: + - lua$LUA_SFX test.lua + +after_success: + - coveralls + +notifications: + email: + on_success: change + on_failure: always diff --git a/.travis/setup_lua.sh b/.travis/setup_lua.sh new file mode 100644 index 0000000..8514adf --- /dev/null +++ b/.travis/setup_lua.sh @@ -0,0 +1,50 @@ +# A script for setting up environment for travis-ci testing. +# Sets up Lua and Luarocks. +# LUA must be "lua5.1", "lua5.2" or "luajit". +# PLATFORM must be "linux" or "macosx". + +if [ "$LUA" == "luajit" ]; then + curl http://luajit.org/download/LuaJIT-2.0.2.tar.gz | tar xz + cd LuaJIT-2.0.2 + make && sudo make install + cd $TRAVIS_BUILD_DIR; +else + if [ "$LUA" == "lua5.1" ]; then + curl http://www.lua.org/ftp/lua-5.1.5.tar.gz | tar xz + cd lua-5.1.5; + elif [ "$LUA" == "lua5.2" ]; then + curl http://www.lua.org/ftp/lua-5.2.3.tar.gz | tar xz + cd lua-5.2.3; + elif [ "$LUA" == "lua5.3" ]; then + curl http://www.lua.org/work/lua-5.3.0-work2.tar.gz | tar xz + cd lua-5.3.0-work2; + fi + sudo make $PLATFORM install + cd $TRAVIS_BUILD_DIR; +fi + +LUAROCKS_BASE=luarocks-$LUAROCKS_VER +curl http://luarocks.org/releases/$LUAROCKS_BASE.tar.gz | tar xz +cd $LUAROCKS_BASE; + +if [ "$LUA" == "luajit" ]; then + ./configure --lua-suffix=jit --with-lua-include=/usr/local/include/luajit-2.0; +else + ./configure; +fi + +make && sudo make install + +cd $TRAVIS_BUILD_DIR + +rm -rf $LUAROCKS_BASE + +if [ "$LUA" == "luajit" ]; then + rm -rf LuaJIT-2.0.2; +elif [ "$LUA" == "lua5.1" ]; then + rm -rf lua-5.1.5; +elif [ "$LUA" == "lua5.2" ]; then + rm -rf lua-5.2.3; +elif [ "$LUA" == "lua5.3" ]; then + rm -rf lua-5.3.0-work2; +fi From 31b260ee44cb28cc437b91e1b647a6c57c28c74c Mon Sep 17 00:00:00 2001 From: Alexey Melnichuk Date: Fri, 23 May 2014 15:29:40 +0500 Subject: [PATCH 25/30] Allow LUACMSGPACK_MAX_NESTING to be set by CPP --- lua_cmsgpack.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 6b9e9d1..0966d3b 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -13,11 +13,9 @@ #define LUACMSGPACK_COPYRIGHT "Copyright (C) 2012, Salvatore Sanfilippo" #define LUACMSGPACK_DESCRIPTION "MessagePack C implementation for Lua" -#define LUACMSGPACK_MAX_NESTING 16 /* Max tables nesting. */ - /* Allows a preprocessor directive to override MAX_NESTING */ #ifndef LUACMSGPACK_MAX_NESTING - #define LUACMSGPACK_MAX_NESTING 16 + #define LUACMSGPACK_MAX_NESTING 16 /* Max tables nesting. */ #endif #if (_XOPEN_SOURCE >= 600 || _ISOC99_SOURCE || _POSIX_C_SOURCE >= 200112L) From 6078bbd5e04ca40ccea26abc713b3e368b3c961d Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 26 May 2014 19:54:05 -0400 Subject: [PATCH 26/30] Add tests for more edge cases The 0xFFFFFFFF test can potentially fail on 32 bit systems if our internal conversions are casting to wrong Lua types. lua_Integer != lua_Number --- test.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test.lua b/test.lua index 000e9d4..f7eed34 100644 --- a/test.lua +++ b/test.lua @@ -6,6 +6,10 @@ local cmsgpack = require "cmsgpack" local ok, cmsgpack_safe = pcall(require, 'cmsgpack.safe') if not ok then cmsgpack_safe = nil end +print("------------------------------------") +print("Lua version: " .. (_G.jit and _G.jit.version or _G._VERSION)) +print("------------------------------------") + local unpack = unpack or table.unpack passed = 0 @@ -350,6 +354,10 @@ test_circular("fix array (3)",{1,{},{}}) test_circular("fix map",{a=5,b=10,c="string"}) test_circular("positive infinity", math.huge) test_circular("negative infinity", -math.huge) +test_circular("high bits", 0xFFFFFFFF) +test_circular("higher bits", 0xFFFFFFFFFFFFFFFF) +test_circular("high bits", -0x7FFFFFFF) +test_circular("higher bits", -0x7FFFFFFFFFFFFFFF) -- The following test vectors are taken from the Javascript lib at: -- https://github.com/cuzic/MessagePack-JS/blob/master/test/test_pack.html From 9c85420d64a01970e4e44e578b24f5a426ed67f9 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 24 Nov 2014 11:51:46 -0500 Subject: [PATCH 27/30] Allow forcing 32 bit builds on 64 bit platforms --- CMakeLists.txt | 8 ++++++++ README.md | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 461e9a9..6092565 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,9 +28,17 @@ else() set(_lua_module_dir "${_lua_lib_dir}/lua/5.1") endif() +option(Build32Bit "Build 32-bit Library" OFF) + set(CMAKE_C_FLAGS "-O2 -g -ggdb -Wall -pedantic -std=c99") add_library(cmsgpack MODULE lua_cmsgpack.c) set_target_properties(cmsgpack PROPERTIES PREFIX "") + +if(Build32Bit) + set_target_properties(cmsgpack + PROPERTIES COMPILE_FLAGS "-m32" LINK_FLAGS "-m32") +endif() + target_link_libraries(cmsgpack ${_MODULE_LINK}) install(TARGETS cmsgpack DESTINATION "${_lua_module_dir}") diff --git a/README.md b/README.md index 7789f44..417337a 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,13 @@ Build and test: make lua ../test.lua +You can build a 32-bit module on a 64-bit platform with: + + mkdir build; cd build + cmake -DBuild32Bit=ON .. + make + lua ../test.lua + NESTED TABLES --- Nested tables are handled correctly up to `LUACMSGPACK_MAX_NESTING` levels of From 587922d2957e1c5786128dd0da41bee05990494a Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 24 Nov 2014 11:52:32 -0500 Subject: [PATCH 28/30] Update CI to test against 5.3.0 beta 5.3.0-work2 has been obsoleted by the newer 5.3.0-beta This testing infrastructure is taken from: https://github.com/moteus/lua-vararg/tree/246dc3126dd66935a020a0a26c165ceb035268e5/.travis --- .travis.yml | 4 +-- .travis/platform.sh | 15 ++++++++ .travis/setup_lua.sh | 83 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 .travis/platform.sh diff --git a/.travis.yml b/.travis.yml index 0188fcd..7a336c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ language: erlang env: global: - PLATFORM=linux - - LUAROCKS_VER=2.1.0 + - LUAROCKS_VER=2.2.0 matrix: - LUA=lua5.1 LUA_SFX= - LUA=lua5.2 LUA_SFX= @@ -11,7 +11,7 @@ env: - LUA=lua5.3 LUA_SFX= before_install: - - bash .travis/setup_lua.sh + - bash -x .travis/setup_lua.sh - sudo pip install cpp-coveralls install: diff --git a/.travis/platform.sh b/.travis/platform.sh new file mode 100644 index 0000000..4a3af0d --- /dev/null +++ b/.travis/platform.sh @@ -0,0 +1,15 @@ +if [ -z "$PLATFORM" ]; then + PLATFORM=$TRAVIS_OS_NAME; +fi + +if [ "$PLATFORM" == "osx" ]; then + PLATFORM="macosx"; +fi + +if [ -z "$PLATFORM" ]; then + if [ "$(uname)" == "Linux" ]; then + PLATFORM="linux"; + else + PLATFORM="macosx"; + fi; +fi diff --git a/.travis/setup_lua.sh b/.travis/setup_lua.sh index 8514adf..373e24d 100644 --- a/.travis/setup_lua.sh +++ b/.travis/setup_lua.sh @@ -1,13 +1,54 @@ +#! /bin/bash + # A script for setting up environment for travis-ci testing. # Sets up Lua and Luarocks. # LUA must be "lua5.1", "lua5.2" or "luajit". -# PLATFORM must be "linux" or "macosx". +# luajit2.0 - master v2.0 +# luajit2.1 - master v2.1 + +LUAJIT_BASE="LuaJIT-2.0.3" + +source .travis/platform.sh + +LUAJIT="no" + +if [ "$PLATFORM" == "macosx" ]; then + if [ "$LUA" == "luajit" ]; then + LUAJIT="yes"; + fi + if [ "$LUA" == "luajit2.0" ]; then + LUAJIT="yes"; + fi + if [ "$LUA" == "luajit2.1" ]; then + LUAJIT="yes"; + fi; +elif [ "$(expr substr $LUA 1 6)" == "luajit" ]; then + LUAJIT="yes"; +fi + +if [ "$LUAJIT" == "yes" ]; then + + if [ "$LUA" == "luajit" ]; then + curl http://luajit.org/download/$LUAJIT_BASE.tar.gz | tar xz; + else + git clone http://luajit.org/git/luajit-2.0.git $LUAJIT_BASE; + fi + + cd $LUAJIT_BASE + + if [ "$LUA" == "luajit2.1" ]; then + git checkout v2.1; + fi -if [ "$LUA" == "luajit" ]; then - curl http://luajit.org/download/LuaJIT-2.0.2.tar.gz | tar xz - cd LuaJIT-2.0.2 make && sudo make install - cd $TRAVIS_BUILD_DIR; + + if [ "$LUA" == "luajit2.1" ]; then + sudo ln -s /usr/local/bin/luajit-2.1.0-alpha /usr/local/bin/luajit + sudo ln -s /usr/local/bin/luajit /usr/local/bin/lua; + else + sudo ln -s /usr/local/bin/luajit /usr/local/bin/lua; + fi; + else if [ "$LUA" == "lua5.1" ]; then curl http://www.lua.org/ftp/lua-5.1.5.tar.gz | tar xz @@ -16,35 +57,45 @@ else curl http://www.lua.org/ftp/lua-5.2.3.tar.gz | tar xz cd lua-5.2.3; elif [ "$LUA" == "lua5.3" ]; then - curl http://www.lua.org/work/lua-5.3.0-work2.tar.gz | tar xz - cd lua-5.3.0-work2; + curl http://www.lua.org/work/lua-5.3.0-beta.tar.gz | tar xz + cd lua-5.3.0-beta; fi - sudo make $PLATFORM install - cd $TRAVIS_BUILD_DIR; + sudo make $PLATFORM install; fi -LUAROCKS_BASE=luarocks-$LUAROCKS_VER -curl http://luarocks.org/releases/$LUAROCKS_BASE.tar.gz | tar xz -cd $LUAROCKS_BASE; +cd $TRAVIS_BUILD_DIR; + +LUAROCKS_BASE=luarocks-$LUAROCKS + +# curl http://luarocks.org/releases/$LUAROCKS_BASE.tar.gz | tar xz + +git clone https://github.com/keplerproject/luarocks.git $LUAROCKS_BASE +cd $LUAROCKS_BASE + +git checkout v$LUAROCKS if [ "$LUA" == "luajit" ]; then ./configure --lua-suffix=jit --with-lua-include=/usr/local/include/luajit-2.0; +elif [ "$LUA" == "luajit2.0" ]; then + ./configure --lua-suffix=jit --with-lua-include=/usr/local/include/luajit-2.0; +elif [ "$LUA" == "luajit2.1" ]; then + ./configure --lua-suffix=jit --with-lua-include=/usr/local/include/luajit-2.1; else ./configure; fi -make && sudo make install +make build && sudo make install cd $TRAVIS_BUILD_DIR rm -rf $LUAROCKS_BASE -if [ "$LUA" == "luajit" ]; then - rm -rf LuaJIT-2.0.2; +if [ "$LUAJIT" == "yes" ]; then + rm -rf $LUAJIT_BASE; elif [ "$LUA" == "lua5.1" ]; then rm -rf lua-5.1.5; elif [ "$LUA" == "lua5.2" ]; then rm -rf lua-5.2.3; elif [ "$LUA" == "lua5.3" ]; then - rm -rf lua-5.3.0-work2; + rm -rf lua-5.3.0-beta; fi From 42f7a281702737daa2bdc166528018ccf95cfed8 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 24 Nov 2014 11:53:27 -0500 Subject: [PATCH 29/30] Fix 64 bit integer processing on 32 bit platforms This is the solution to mattsta/lua-cmsgpack#4 but rewritten to use compile-time pointer size determination. --- lua_cmsgpack.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 0966d3b..6aa04e2 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -30,8 +30,19 @@ #define IS_INT64_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int64_t) #define IS_INT_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int) +/* If size of pointer is equal to a 4 byte integer, we're on 32 bits. */ +#if UINTPTR_MAX == UINT_MAX + #define BITS_32 1 +#else + #define BITS_32 0 +#endif + #if LUA_VERSION_NUM < 503 - #define lua_pushunsigned(L, n) lua_pushinteger(L, n) + #if BITS_32 + #define lua_pushunsigned(L, n) lua_pushnumber(L, n) + #else + #define lua_pushunsigned(L, n) lua_pushinteger(L, n) + #endif #endif /* ============================================================================= @@ -348,7 +359,11 @@ static void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { /* Lua 5.3 has a built in 64-bit integer type */ static void mp_encode_lua_integer(lua_State *L, mp_buf *buf) { +#if (LUA_VERSION_NUM < 503) && BITS_32 + lua_Number i = lua_tonumber(L,-1); +#else lua_Integer i = lua_tointeger(L,-1); +#endif mp_encode_int(buf, (int64_t)i); } @@ -431,10 +446,11 @@ static int table_is_an_array(lua_State *L) { /* The <= 0 check is valid here because we're comparing indexes. */ #if LUA_VERSION_NUM < 503 if ((LUA_TNUMBER != lua_type(L,-1)) || (n = lua_tonumber(L, -1)) <= 0 || - !IS_INT_EQUIVALENT(n)) { + !IS_INT_EQUIVALENT(n)) #else - if (!lua_isinteger(L,-1) || (n = lua_tointeger(L, -1)) <= 0) { + if (!lua_isinteger(L,-1) || (n = lua_tointeger(L, -1)) <= 0) #endif + { lua_settop(L, stacktop); return 0; } @@ -628,7 +644,11 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xd3: /* int 64 */ mp_cur_need(c,9); +#if LUA_VERSION_NUM < 503 + lua_pushnumber(L, +#else lua_pushinteger(L, +#endif ((int64_t)c->p[1] << 56) | ((int64_t)c->p[2] << 48) | ((int64_t)c->p[3] << 40) | From afbc0099016278f0fd7289737685e6ddfd5ba90f Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 24 Nov 2014 22:41:27 -0500 Subject: [PATCH 30/30] 5.3 compat: short functions to long functions checkint -> checkinteger optint -> optinteger Also removes `< 503` check for creating pushunsigned. Older lua versions had redundant functions, but one with a shorter name than the other. Newer Lua versions only retain the long version. one is an alias of the other, but one was removed in 5.3+, so it's better to use the common function across all versions. --- lua_cmsgpack.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 6aa04e2..786daab 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -37,12 +37,10 @@ #define BITS_32 0 #endif -#if LUA_VERSION_NUM < 503 - #if BITS_32 - #define lua_pushunsigned(L, n) lua_pushnumber(L, n) - #else - #define lua_pushunsigned(L, n) lua_pushinteger(L, n) - #endif +#if BITS_32 + #define lua_pushunsigned(L, n) lua_pushnumber(L, n) +#else + #define lua_pushunsigned(L, n) lua_pushinteger(L, n) #endif /* ============================================================================= @@ -836,15 +834,15 @@ static int mp_unpack(lua_State *L) { } static int mp_unpack_one(lua_State *L) { - int offset = luaL_optint(L, 2, 0); + int offset = luaL_optinteger(L, 2, 0); /* Variable pop because offset may not exist */ lua_pop(L, lua_gettop(L)-1); return mp_unpack_full(L, 1, offset); } static int mp_unpack_limit(lua_State *L) { - int limit = luaL_checkint(L, 2); - int offset = luaL_optint(L, 3, 0); + int limit = luaL_checkinteger(L, 2); + int offset = luaL_optinteger(L, 3, 0); /* Variable pop because offset may not exist */ lua_pop(L, lua_gettop(L)-1);