Merge branch 'pan/happi/yield_in_term_to_binary' into maint

* pan/happi/yield_in_term_to_binary:
  Add testcase to stress extra_root
  term_to_binary: Remove debug code and set production trap levels
  Teach erl_gc:offset_rootset about extra_root
  Teach external.c to handle reallocs before compression
  Make all steps ofterm_to_binary work in chunks and yield
  Make term_to_binary yield (trap).

OTP-11163

erts/emulator/beam/atom.names

@@ -534,6 +534,7 @@ atom system_version
 atom system_architecture
 atom SYSTEM='SYSTEM'
 atom table
+atom term_to_binary_trap
 atom this
 atom thread_pool_size
 atom threads

erts/emulator/beam/erl_alloc.types

@@ -150,6 +150,7 @@ type	LINK_LH		STANDARD	PROCESSES	link_lh
 type	SUSPEND_MON	STANDARD	PROCESSES	suspend_monitor
 type	PEND_SUSPEND	SHORT_LIVED	PROCESSES	pending_suspend
 type	PROC_LIST	SHORT_LIVED	PROCESSES	proc_list
+type	EXTRA_ROOT	SHORT_LIVED	PROCESSES	extra_root
 type	FUN_ENTRY	LONG_LIVED	CODE		fun_entry
 type	ATOM_TXT	LONG_LIVED	ATOM		atom_text
 type 	BEAM_REGISTER	EHEAP		PROCESSES	beam_register

erts/emulator/beam/erl_gc.c

@@ -1964,6 +1964,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
         ++n;
     }
 
+    /*
+     * A trapping BIF can add to rootset by setting the extra_root
+     * in the process_structure.
+     */
+    if (p->extra_root != NULL) {
+	roots[n].v = p->extra_root->objv;
+	roots[n].sz = p->extra_root->sz;
+	++n;
+    }
+
+
     ASSERT((is_nil(p->seq_trace_token) ||
 	    is_tuple(follow_moved(p->seq_trace_token)) ||
 	    is_atom(p->seq_trace_token)));
@@ -2541,6 +2552,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 		    p->dictionary->used, 
 		    offs, area, area_size);
     }
+    if (p->extra_root != NULL) {
+	offset_heap_ptr(p->extra_root->objv, 
+			p->extra_root->sz, 
+			offs, area, area_size);
+    }
+
     offset_heap_ptr(&p->fvalue, 1, offs, area, area_size);
     offset_heap_ptr(&p->ftrace, 1, offs, area, area_size);
     offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size);

erts/emulator/beam/erl_init.c

@@ -340,6 +340,7 @@ erl_init(int ncpu,
     erts_init_bif_binary();
     erts_init_bif_re();
     erts_init_unicode(); /* after RE to get access to PCRE unicode */
+    erts_init_external();
     erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2);
     erts_late_init_process();
 #if HAVE_ERTS_MSEG

erts/emulator/beam/erl_process.c

@@ -7513,6 +7513,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->htop = p->heap;
     p->heap_sz = sz;
     p->catches = 0;
+    p->extra_root = NULL;
 
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
@@ -8945,6 +8946,12 @@ erts_continue_exit_process(Process *p)
     if (pbt)
         erts_free(ERTS_ALC_T_BPD, (void *) pbt);
 
+    if (p->extra_root != NULL) {
+	(p->extra_root->cleanup)(p->extra_root); /* Should deallocate 
+						    whole structure */
+	p->extra_root = NULL;
+    }
+
     delete_process(p);
 
 #ifdef ERTS_SMP

erts/emulator/beam/erl_process.h

@@ -699,6 +699,14 @@ struct ErtsPendingSuspend_ {
 
 #endif
 
+
+typedef struct ErlExtraRootSet_ ErlExtraRootSet;
+struct ErlExtraRootSet_ {
+    Eterm *objv;
+    Uint sz;
+    void (*cleanup)(ErlExtraRootSet *);
+};
+
 /* Defines to ease the change of memory architecture */
 #  define HEAP_START(p)     (p)->heap
 #  define HEAP_TOP(p)       (p)->htop
@@ -792,6 +800,8 @@ struct process {
 
     ErlMessageQueue msg;	/* Message queue */
 
+    ErlExtraRootSet *extra_root;   /* Used by trapping BIF's */
+
     union {
 	ErtsBifTimer *bif_timers;	/* Bif timers aiming at this process */
 	void *terminate;
@@ -1976,6 +1986,7 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
     }
 }
 
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* #ifdef ERTS_SMP */

erts/emulator/beam/erl_zlib.c

@@ -44,6 +44,48 @@ void erl_zlib_zfree_callback (voidpf opaque, voidpf ptr)
     erts_free(ERTS_ALC_T_ZLIB, ptr);
 }
 
+/*
+ * Initialize a z_stream with a source, to later *chunk* data into a destination 
+ * Returns Z_OK or Error.
+ */
+int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, 
+				   uLong sourceLen, int level) 
+{
+    streamp->next_in = (Bytef*)source;
+    streamp->avail_in = (uInt)sourceLen;
+    streamp->total_out = streamp->avail_out = 0;
+    streamp->next_out = NULL;
+    erl_zlib_alloc_init(streamp);
+    return deflateInit(streamp, level);
+}
+/* 
+ * Deflate a chunk, The destination length is the limit.
+ * Returns Z_OK if more to process, Z_STREAM_END if we are done.
+ */
+int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen) 
+{
+    int err;
+    uLongf last_tot = streamp->total_out;
+
+    streamp->next_out = dest;
+    streamp->avail_out = (uInt)*destLen;
+
+    if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR;
+
+    err = deflate(streamp, Z_FINISH);
+    *destLen = streamp->total_out - last_tot;
+    return err; 
+}
+
+
+/*
+ * When we are done, free up the deflate structure
+ * Retyurns Z_OK or Error
+ */
+int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp)
+{
+    return deflateEnd(streamp);
+}
 
 int ZEXPORT erl_zlib_compress2 (Bytef* dest, uLongf* destLen,
 				const Bytef* source, uLong sourceLen,

erts/emulator/beam/erl_zlib.h

@@ -31,6 +31,14 @@
     (s)->zfree = erl_zlib_zfree_callback;   \
   } while (0)
 
+/*
+ * Chunked interface, used by term_to_binary among others.
+ */
+int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, 
+				   uLong sourceLen, int level);
+int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen);
+int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp);
+
 /* Use instead of compress
 */
 #define erl_zlib_compress(dest,destLen,source,sourceLen) \