Merge 9e10787013 into 446e284bd5

README.md

@@ -42,6 +42,10 @@ The options for decode are:
 * `return_maps` - Tell Jiffy to return objects using the maps data type
   on VMs that support it. This raises an error on VMs that don't support
   maps.
+* `with_trailer` - Tell Jiffy to return the trailing unparsed data (if any) along with
+  the parsed term instead of failing with {error,{_,invalid_traling_data}}. When
+  the trailer is available, the return value is {with_trailer, EJson, Trailer},
+  where Trailer is a sub-binary of the input, for efficiency.
 
 `jiffy:encode/1,2`
 ------------------

c_src/decoder.c

@@ -53,6 +53,7 @@ typedef struct {
     int             is_partial;
     int             return_maps;
     int             use_nil;
+    int             with_trailer;
 
     char*           p;
     unsigned char*  u;
@@ -81,6 +82,7 @@ dec_new(ErlNifEnv* env)
     d->is_partial = 0;
     d->return_maps = 0;
     d->use_nil = 0;
+    d->with_trailer = 0;
 
     d->p = NULL;
     d->u = NULL;
@@ -712,6 +714,8 @@ decode_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 #endif
         } else if(enif_compare(val, d->atoms->atom_use_nil) == 0) {
             d->use_nil = 1;
+        } else if(enif_compare(val, d->atoms->atom_with_trailer) == 0) {
+            d->with_trailer = 1;
         } else {
             return enif_make_badarg(env);
         }
@@ -726,6 +730,7 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     Decoder* d;
     jiffy_st* st = (jiffy_st*) enif_priv_data(env);
 
+    ERL_NIF_TERM bin_term = argv[0];
     ErlNifBinary bin;
 
     ERL_NIF_TERM objs;
@@ -736,7 +741,7 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 
     if(argc != 5) {
         return enif_make_badarg(env);
-    } else if(!enif_inspect_binary(env, argv[0], &bin)) {
+    } else if(!enif_inspect_binary(env, bin_term, &bin)) {
         return enif_make_badarg(env);
     } else if(!enif_get_resource(env, argv[1], st->res_dec, (void**) &d)) {
         return enif_make_badarg(env);
@@ -754,8 +759,14 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     start = d->i;
     while(d->i < bin.size) {
         //fprintf(stderr, "state: %d\r\n", dec_curr(d));
-        if(should_yield(d->i - start, d->bytes_per_iter)) {
-            consume_timeslice(env, d->i - start, d->bytes_per_iter);
+        if(should_yield(d->i - start, d->bytes_per_iter)
+            /* A system could handle roughly 100kb per millisecond on a single core.
+             * So the total amount of work per millisecond is 100kb.
+             * We report the percentage of the time every (bytes_per_iter) bytes
+             * in hope that the system will ask as to yield. We don't yield until
+             * asked by the system according to our feedback (of questionable accuracy).
+             */
+            && consume_timeslice(env, d->i - start, 100000)) {
             return enif_make_tuple5(
                     env,
                     st->atom_iter,
@@ -1028,8 +1039,16 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
                         d->i++;
                         break;
                     default:
-                        ret = dec_error(d, "invalid_trailing_data");
-                        goto done;
+                        if(d->with_trailer) {
+                            ERL_NIF_TERM trailer = enif_make_sub_binary(env,
+                                bin_term, d->i, bin.size - d->i);
+                            val = enif_make_tuple3(env, d->atoms->atom_with_trailer,
+                                    val, trailer);
+                            goto soft_done;
+                        } else {
+                            ret = dec_error(d, "invalid_trailing_data");
+                            goto done;
+                        }
                 }
                 break;
 
@@ -1039,6 +1058,8 @@ decode_iter(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
         }
     }
 
+soft_done:
+
     if(dec_curr(d) != st_done) {
         ret = dec_error(d, "truncated_json");
     } else if(d->is_partial) {

c_src/jiffy.c

@@ -28,6 +28,7 @@ load(ErlNifEnv* env, void** priv, ERL_NIF_TERM info)
     st->atom_return_maps = make_atom(env, "return_maps");
     st->atom_nil = make_atom(env, "nil");
     st->atom_use_nil = make_atom(env, "use_nil");
+    st->atom_with_trailer = make_atom(env, "with_trailer");
 
     // Markers used in encoding
     st->ref_object = make_atom(env, "$object_ref$");

c_src/jiffy.h

@@ -8,10 +8,22 @@
 
 #define DEFAULT_BYTES_PER_ITER 2048
 
+#ifndef UNUSED
+#define UNUSED  __attribute__((unused))
+#endif
+
 #define MAP_TYPE_PRESENT \
     ((ERL_NIF_MAJOR_VERSION == 2 && ERL_NIF_MINOR_VERSION >= 6) \
     || (ERL_NIF_MAJOR_VERSION > 2))
 
+static int UNUSED should_yield(size_t used, size_t limit) {
+    if(limit == 0 || used < limit) {
+        return 0;
+    }
+
+    return 1;
+}
+
 typedef struct {
     ERL_NIF_TERM    atom_ok;
     ERL_NIF_TERM    atom_error;
@@ -30,6 +42,7 @@ typedef struct {
     ERL_NIF_TERM    atom_return_maps;
     ERL_NIF_TERM    atom_nil;
     ERL_NIF_TERM    atom_use_nil;
+    ERL_NIF_TERM    atom_with_trailer;
 
     ERL_NIF_TERM    ref_object;
     ERL_NIF_TERM    ref_array;
@@ -44,7 +57,6 @@ ERL_NIF_TERM make_error(jiffy_st* st, ErlNifEnv* env, const char* error);
 ERL_NIF_TERM make_obj_error(jiffy_st* st, ErlNifEnv* env, const char* error,
         ERL_NIF_TERM obj);
 int get_bytes_per_iter(ErlNifEnv* env, ERL_NIF_TERM val, size_t* bpi);
-int should_yield(size_t used, size_t limit);
 int consume_timeslice(ErlNifEnv* env, size_t used, size_t limit);
 
 ERL_NIF_TERM decode_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);

c_src/util.c

@@ -62,16 +62,6 @@ get_bytes_per_iter(ErlNifEnv* env, ERL_NIF_TERM val, size_t* bpi)
     return 1;
 }
 
-int
-should_yield(size_t used, size_t limit)
-{
-    if(limit == 0 || used < limit) {
-        return 0;
-    }
-
-    return 1;
-}
-
 int
 consume_timeslice(ErlNifEnv* env, size_t used, size_t limit)
 {

test/jiffy_15_trailer_tests.erl

@@ -0,0 +1,16 @@
+% This file is part of Jiffy released under the MIT license.
+% See the LICENSE file for more information.
+
+-module(jiffy_15_trailer_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+trailer_test_() ->
+   Opts = [with_trailer],
+   {"trailer", [
+        ?_assertEqual(true, jiffy:decode(<<"true">>, Opts)),
+        ?_assertMatch({with_trailer, true, <<";">>}, jiffy:decode(<<"true;">>, Opts)),
+        ?_assertMatch({with_trailer, true, <<"[]">>}, jiffy:decode(<<"true[]">>, Opts)),
+        ?_assertMatch({with_trailer, [], <<"{}">>}, jiffy:decode(<<"[]{}">>, Opts)),
+        ?_assertMatch({with_trailer, 1, <<"2 3">>}, jiffy:decode(<<"1 2 3">>, Opts))
+    ]}.