diff --git a/README.md b/README.md index 13bc42e..72382a0 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ The options for decode are: * `return_maps` - Tell Jiffy to return objects using the maps data type on VMs that support it. This raises an error on VMs that don't support maps. +* `with_trailer` - Tell Jiffy to return the trailing unparsed data (if any) along with + the parsed term instead of failing with {error,{_,invalid_traling_data}}. When + the trailer is available, the return value is {with_trailer, EJson, Trailer}, + where Trailer is a sub-binary of the input, for efficiency. `jiffy:encode/1,2` ------------------ diff --git a/c_src/decoder.c b/c_src/decoder.c index ad3e8d3..066fcc1 100644 --- a/c_src/decoder.c +++ b/c_src/decoder.c @@ -53,6 +53,7 @@ typedef struct { int is_partial; int return_maps; int use_nil; + int with_trailer; char* p; unsigned char* u; @@ -81,6 +82,7 @@ dec_new(ErlNifEnv* env) d->is_partial = 0; d->return_maps = 0; d->use_nil = 0; + d->with_trailer = 0; d->p = NULL; d->u = NULL; @@ -712,6 +714,8 @@ decode_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) #endif } else if(enif_compare(val, d->atoms->atom_use_nil) == 0) { d->use_nil = 1; + } else if(enif_compare(val, d->atoms->atom_with_trailer) == 0) { + d->with_trailer = 1; } else { return enif_make_badarg(env); } @@ -726,6 +730,7 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) Decoder* d; jiffy_st* st = (jiffy_st*) enif_priv_data(env); + ERL_NIF_TERM bin_term = argv[0]; ErlNifBinary bin; ERL_NIF_TERM objs; @@ -736,7 +741,7 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) if(argc != 5) { return enif_make_badarg(env); - } else if(!enif_inspect_binary(env, argv[0], &bin)) { + } else if(!enif_inspect_binary(env, bin_term, &bin)) { return enif_make_badarg(env); } else if(!enif_get_resource(env, argv[1], st->res_dec, (void**) &d)) { return enif_make_badarg(env); @@ -754,8 +759,14 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) start = d->i; while(d->i < bin.size) { //fprintf(stderr, "state: %d\r\n", dec_curr(d)); - if(should_yield(d->i - start, d->bytes_per_iter)) { - consume_timeslice(env, d->i - start, d->bytes_per_iter); + if(should_yield(d->i - start, d->bytes_per_iter) + /* A system could handle roughly 100kb per millisecond on a single core. + * So the total amount of work per millisecond is 100kb. + * We report the percentage of the time every (bytes_per_iter) bytes + * in hope that the system will ask as to yield. We don't yield until + * asked by the system according to our feedback (of questionable accuracy). + */ + && consume_timeslice(env, d->i - start, 100000)) { return enif_make_tuple5( env, st->atom_iter, @@ -1028,8 +1039,16 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) d->i++; break; default: - ret = dec_error(d, "invalid_trailing_data"); - goto done; + if(d->with_trailer) { + ERL_NIF_TERM trailer = enif_make_sub_binary(env, + bin_term, d->i, bin.size - d->i); + val = enif_make_tuple3(env, d->atoms->atom_with_trailer, + val, trailer); + goto soft_done; + } else { + ret = dec_error(d, "invalid_trailing_data"); + goto done; + } } break; @@ -1039,6 +1058,8 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } } +soft_done: + if(dec_curr(d) != st_done) { ret = dec_error(d, "truncated_json"); } else if(d->is_partial) { diff --git a/c_src/jiffy.c b/c_src/jiffy.c index 31aa854..544fc5c 100644 --- a/c_src/jiffy.c +++ b/c_src/jiffy.c @@ -28,6 +28,7 @@ load(ErlNifEnv* env, void** priv, ERL_NIF_TERM info) st->atom_return_maps = make_atom(env, "return_maps"); st->atom_nil = make_atom(env, "nil"); st->atom_use_nil = make_atom(env, "use_nil"); + st->atom_with_trailer = make_atom(env, "with_trailer"); // Markers used in encoding st->ref_object = make_atom(env, "$object_ref$"); diff --git a/c_src/jiffy.h b/c_src/jiffy.h index 6d869a5..d1d607c 100644 --- a/c_src/jiffy.h +++ b/c_src/jiffy.h @@ -8,10 +8,22 @@ #define DEFAULT_BYTES_PER_ITER 2048 +#ifndef UNUSED +#define UNUSED __attribute__((unused)) +#endif + #define MAP_TYPE_PRESENT \ ((ERL_NIF_MAJOR_VERSION == 2 && ERL_NIF_MINOR_VERSION >= 6) \ || (ERL_NIF_MAJOR_VERSION > 2)) +static int UNUSED should_yield(size_t used, size_t limit) { + if(limit == 0 || used < limit) { + return 0; + } + + return 1; +} + typedef struct { ERL_NIF_TERM atom_ok; ERL_NIF_TERM atom_error; @@ -30,6 +42,7 @@ typedef struct { ERL_NIF_TERM atom_return_maps; ERL_NIF_TERM atom_nil; ERL_NIF_TERM atom_use_nil; + ERL_NIF_TERM atom_with_trailer; ERL_NIF_TERM ref_object; ERL_NIF_TERM ref_array; @@ -44,7 +57,6 @@ ERL_NIF_TERM make_error(jiffy_st* st, ErlNifEnv* env, const char* error); ERL_NIF_TERM make_obj_error(jiffy_st* st, ErlNifEnv* env, const char* error, ERL_NIF_TERM obj); int get_bytes_per_iter(ErlNifEnv* env, ERL_NIF_TERM val, size_t* bpi); -int should_yield(size_t used, size_t limit); int consume_timeslice(ErlNifEnv* env, size_t used, size_t limit); ERL_NIF_TERM decode_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); diff --git a/c_src/util.c b/c_src/util.c index c30a6d7..f9f3bfc 100644 --- a/c_src/util.c +++ b/c_src/util.c @@ -62,16 +62,6 @@ get_bytes_per_iter(ErlNifEnv* env, ERL_NIF_TERM val, size_t* bpi) return 1; } -int -should_yield(size_t used, size_t limit) -{ - if(limit == 0 || used < limit) { - return 0; - } - - return 1; -} - int consume_timeslice(ErlNifEnv* env, size_t used, size_t limit) { diff --git a/test/jiffy_15_trailer_tests.erl b/test/jiffy_15_trailer_tests.erl new file mode 100644 index 0000000..1190011 --- /dev/null +++ b/test/jiffy_15_trailer_tests.erl @@ -0,0 +1,16 @@ +% This file is part of Jiffy released under the MIT license. +% See the LICENSE file for more information. + +-module(jiffy_15_trailer_tests). + +-include_lib("eunit/include/eunit.hrl"). + +trailer_test_() -> + Opts = [with_trailer], + {"trailer", [ + ?_assertEqual(true, jiffy:decode(<<"true">>, Opts)), + ?_assertMatch({with_trailer, true, <<";">>}, jiffy:decode(<<"true;">>, Opts)), + ?_assertMatch({with_trailer, true, <<"[]">>}, jiffy:decode(<<"true[]">>, Opts)), + ?_assertMatch({with_trailer, [], <<"{}">>}, jiffy:decode(<<"[]{}">>, Opts)), + ?_assertMatch({with_trailer, 1, <<"2 3">>}, jiffy:decode(<<"1 2 3">>, Opts)) + ]}.