Make all steps ofterm_to_binary work in chunks and yield

Rewrite and extend of Happi's initial work

Extra_root to process structure to enable GC of state - Changed the
process structure to point to a separate struct, the struct also
contains a destructor function to allow for proper cleanup.

Rewrote encode_size_struct and enc_term to have internal versions with
reduction counters which will result in interrupt for later restart
when the counter reaches zero - removed the EWA_STACK from Happis
version and directly save the ESTACK's and WSTACK's in the above
mentioned struct (or array thereof) that are pointed out from the
process structure. The destructor will take care of the deallocation
in case of process death.

Added ESTACK and WSTACK macros to save and restore stack and to change
allocator, which makes the previously mentioned stack-save work.

Rewrote enc_term to not store pointers on the stack, and use one WSTACK
for commands etc and another ESTACK for Eterms - Slightly different than
Happis version to make halfword code simpler.

Rewrote encode_size_struct2 so that it does not store pointers on the
stack, also switched to ESTACK instead of WSTACK, this also handles
halfword correctly.

Added interfaces for chunkwise compression, that are
used from term_to_binary/2 when the compressed option is given.

Author: bufflig

erts/emulator/beam/atom.names

@@ -197,7 +197,6 @@ atom elib_malloc
 atom emulator
 atom enable_trace
 atom enabled
-atom enc_term_cont
 atom endian
 atom env
 atom eof
@@ -531,7 +530,7 @@ atom system_version
 atom system_architecture
 atom SYSTEM='SYSTEM'
 atom table
-atom term_to_binary_of_size
+atom term_to_binary_trap
 atom this
 atom thread_pool_size
 atom threads

erts/emulator/beam/dist.c

@@ -1740,10 +1740,10 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
     /* Encode internal version of dist header */
     obuf->extp = erts_encode_ext_dist_header_setup(obuf->ext_endp, acmp);
     /* Encode control message */
-    erts_encode_dist_ext(c_p, ctl, &obuf->ext_endp, flags, acmp);
+    erts_encode_dist_ext(ctl, &obuf->ext_endp, flags, acmp);
     if (is_value(msg)) {
 	/* Encode message */
-       erts_encode_dist_ext(c_p, msg, &obuf->ext_endp, flags, acmp);
+	erts_encode_dist_ext(msg, &obuf->ext_endp, flags, acmp);
     }
 
     ASSERT(obuf->extp < obuf->ext_endp);

erts/emulator/beam/erl_alloc.types

@@ -150,6 +150,7 @@ type	LINK_LH		STANDARD	PROCESSES	link_lh
 type	SUSPEND_MON	STANDARD	PROCESSES	suspend_monitor
 type	PEND_SUSPEND	SHORT_LIVED	PROCESSES	pending_suspend
 type	PROC_LIST	SHORT_LIVED	PROCESSES	proc_list
+type	EXTRA_ROOT	SHORT_LIVED	PROCESSES	extra_root
 type	FUN_ENTRY	LONG_LIVED	CODE		fun_entry
 type	ATOM_TXT	LONG_LIVED	ATOM		atom_text
 type 	BEAM_REGISTER	EHEAP		PROCESSES	beam_register

erts/emulator/beam/erl_gc.c

@@ -1964,15 +1964,20 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
         ++n;
     }
 
-    // Check if a suspended bif has live working data.
-    // How do we know n is small enough to fit in roots[32?]?
+    /*
+     * A trapping BIF can add to rootset by setting the extra_root
+     * in the process_structure.
+     */
     if (p->extra_root != NULL) {
-      printf("GC with extra_root 0x%xl\n", p->extra_root);
-        roots[n].v = p->extra_root;
-        roots[n].sz = p->extra_root_sz;
-        ++n;
+#ifdef HARDDEBUG
+	erts_fprintf(stderr,"GC with extra root 0x%xl\n", p->extra_root->objv);
+#endif
+	roots[n].v = p->extra_root->objv;
+	roots[n].sz = p->extra_root->sz;
+	++n;
     }
 
+
     ASSERT((is_nil(p->seq_trace_token) ||
 	    is_tuple(follow_moved(p->seq_trace_token)) ||
 	    is_atom(p->seq_trace_token)));

erts/emulator/beam/erl_process.c

@@ -7512,10 +7512,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->htop = p->heap;
     p->heap_sz = sz;
     p->catches = 0;
-
     p->extra_root = NULL;
-    p->extra_root_sz = 0;
-    p->extra_root_allocator = 0;
 
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
@@ -8948,6 +8945,12 @@ erts_continue_exit_process(Process *p)
     if (pbt)
         erts_free(ERTS_ALC_T_BPD, (void *) pbt);
 
+    if (p->extra_root != NULL) {
+	(p->extra_root->cleanup)(p->extra_root); /* Should deallocate 
+						    whole structure */
+	p->extra_root = NULL;
+    }
+
     delete_process(p);
 
 #ifdef ERTS_SMP

erts/emulator/beam/erl_process.h

@@ -699,6 +699,14 @@ struct ErtsPendingSuspend_ {
 
 #endif
 
+
+typedef struct ErlExtraRootSet_ ErlExtraRootSet;
+struct ErlExtraRootSet_ {
+    Eterm *objv;
+    Uint sz;
+    void (*cleanup)(ErlExtraRootSet *);
+};
+
 /* Defines to ease the change of memory architecture */
 #  define HEAP_START(p)     (p)->heap
 #  define HEAP_TOP(p)       (p)->htop
@@ -792,10 +800,7 @@ struct process {
 
     ErlMessageQueue msg;	/* Message queue */
 
-    Eterm *extra_root;          /* Extra root set, used e.g. by yielding bifs. */
-    Uint extra_root_sz;         /* Size of extra root set. */
-    ErtsAlcType_t extra_root_allocator; /* Type of memory allocator used,
-                                used for freeing extra_root if process dies. */
+    ErlExtraRootSet *extra_root;   /* Used by trapping BIF's */
 
     union {
 	ErtsBifTimer *bif_timers;	/* Bif timers aiming at this process */
@@ -1980,6 +1985,7 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
     }
 }
 
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* #ifdef ERTS_SMP */

erts/emulator/beam/erl_zlib.c

@@ -44,6 +44,48 @@ void erl_zlib_zfree_callback (voidpf opaque, voidpf ptr)
     erts_free(ERTS_ALC_T_ZLIB, ptr);
 }
 
+/*
+ * Initialize a z_stream with a source, to later *chunk* data into a destination 
+ * Returns Z_OK or Error.
+ */
+int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, 
+				   uLong sourceLen, int level) 
+{
+    streamp->next_in = (Bytef*)source;
+    streamp->avail_in = (uInt)sourceLen;
+    streamp->total_out = streamp->avail_out = 0;
+    streamp->next_out = NULL;
+    erl_zlib_alloc_init(streamp);
+    return deflateInit(streamp, level);
+}
+/* 
+ * Deflate a chunk, The destination length is the limit.
+ * Returns Z_OK if more to process, Z_STREAM_END if we are done.
+ */
+int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen) 
+{
+    int err;
+    uLongf last_tot = streamp->total_out;
+
+    streamp->next_out = dest;
+    streamp->avail_out = (uInt)*destLen;
+    
+    if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR;
+    
+    err = deflate(streamp, Z_FINISH);
+    *destLen = streamp->total_out - last_tot;
+    return err; 
+}
+
+
+/*
+ * When we are done, free up the deflate structure
+ * Retyurns Z_OK or Error
+ */
+int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp)
+{
+    return deflateEnd(streamp);
+}
 
 int ZEXPORT erl_zlib_compress2 (Bytef* dest, uLongf* destLen,
 				const Bytef* source, uLong sourceLen,

erts/emulator/beam/erl_zlib.h

@@ -31,6 +31,14 @@
     (s)->zfree = erl_zlib_zfree_callback;   \
   } while (0)
 
+/*
+ * Chunked interface, used by term_to_binary among others.
+ */
+int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, 
+				   uLong sourceLen, int level);
+int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen);
+int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp);
+
 /* Use instead of compress
 */
 #define erl_zlib_compress(dest,destLen,source,sourceLen) \