Merge branch 'rickard/sched-spec-data-fix/os-monotonic-fix/25/OTP-190…

…43/OTP-19044' into rickard/sched-spec-data-fix/os-monotonic-fix/26/OTP-19043/OTP-19044

* rickard/sched-spec-data-fix/os-monotonic-fix/25/OTP-19043/OTP-19044:
  [erts] configure option for ensuring monotonicity of OS monotonic time
  [erts] Improve checking of monotonic time
  [erts] Replace potential harmful uses of erts_proc_sched_data()

HOWTO/INSTALL.md

@@ -410,6 +410,14 @@ Some of the available `configure` options are:
     time, and OS monotonic time with higher or lower resolution than chosen by
     default. Note that both alternatives may have a negative impact on the performance
     and scalability compared to the default clock sources chosen.
+*   `--enable-ensure-os-monotonic-time` - Enable functionality ensuring the
+    monotonicity of monotonic timestamps delivered by the OS. When a
+    non-monotonic timestamp is detected, it will be replaced by the last
+    delivered monotonic timestamp before being used by Erlang's time
+    functionality. Note that you do *not* want to enable this unless the OS
+    monotonic time source on the system fails to produce monotonic timestamps.
+    This since ensuring the monotonicity of OS monotonic timestamps will hurt
+    scalability and performance of the system.
 *   `--disable-saved-compile-time` - Disable saving of compile date and time
     in the emulator binary.
 *   `--enable-ei-dynamic-lib` - Make erl\_interface build a shared library in addition

erts/config.h.in

@@ -54,6 +54,10 @@
 /* Define > 0 if big-endian < 0 if little-endian, or 0 if unknown */
 #undef ERTS_ENDIANNESS
 
+/* Define if ensurance of the monotonicity of OS monotonic timestamps should
+   be enabled */
+#undef ERTS_ENSURE_OS_MONOTONIC_TIME
+
 /* Define if OS monotonic clock is corrected */
 #undef ERTS_HAVE_CORRECTED_OS_MONOTONIC_TIME
 

erts/configure

@@ -844,6 +844,7 @@ enable_vm_probes
 with_assumed_cache_line_size
 enable_systemd
 with_microstate_accounting
+enable_ensure_os_monotonic_time
 enable_static_nifs
 enable_static_drivers
 with_ets_write_concurrency_locks
@@ -1557,6 +1558,17 @@ Optional Features:
                           possible if --with-dynamic-trace is enabled, and
                           then default)
   --enable-systemd        enable systemd support in epmd
+  --enable-ensure-os-monotonic-time
+                          enable functionality ensuring the monotonicity of
+                          monotonic timestamps delivered by the OS. When a
+                          non-monotonic timestamp is detected, it will be
+                          replaced by the last delivered monotonic timestamp
+                          before being used by Erlang's time functionality.
+                          Note that you do *not* want to enable this unless
+                          the OS monotonic time source on the system fails to
+                          produce monotonic timestamps. This since ensuring
+                          the monotonicity of OS monotonic timestamps will
+                          hurt scalability and performance of the system.
   --enable-static-nifs    link nifs statically. If yes then all nifs in all
                           Erlang/OTP applications will be statically linked
                           into the main binary. It is also possible to give a
@@ -5194,6 +5206,22 @@ printf "%s\n" "#define ERTS_ENABLE_MSACC 2" >>confdefs.h
     *) ;;
 esac
 
+# Check whether --enable-ensure-os-monotonic-time was given.
+if test ${enable_ensure_os_monotonic_time+y}
+then :
+  enableval=$enable_ensure_os_monotonic_time;
+fi
+
+
+if test "$enable_ensure_os_monotonic_time" = "yes"
+then :
+
+
+printf "%s\n" "#define ERTS_ENSURE_OS_MONOTONIC_TIME 1" >>confdefs.h
+
+
+fi
+
 OTP_RELEASE=
 if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then
 	OTP_EXTRA_FLAGS=-DOTP_RELEASE

erts/configure.ac

@@ -301,6 +301,15 @@ case "$with_microstate_accounting" in
     *) ;;
 esac
 
+AC_ARG_ENABLE(ensure-os-monotonic-time,
+AS_HELP_STRING([--enable-ensure-os-monotonic-time],
+               [enable functionality ensuring the monotonicity of monotonic timestamps delivered by the OS. When a non-monotonic timestamp is detected, it will be replaced by the last delivered monotonic timestamp before being used by Erlang's time functionality. Note that you do *not* want to enable this unless the OS monotonic time source on the system fails to produce monotonic timestamps. This since ensuring the monotonicity of OS monotonic timestamps will hurt scalability and performance of the system.]))
+
+AS_IF([test "$enable_ensure_os_monotonic_time" = "yes"], [
+   AC_DEFINE(ERTS_ENSURE_OS_MONOTONIC_TIME, [1],
+     [Define if ensurance of the monotonicity of OS monotonic timestamps should be enabled])
+  ])
+
 dnl Magic test for clearcase.
 OTP_RELEASE=
 if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then

erts/emulator/beam/beam_bp.c

@@ -74,13 +74,13 @@ erts_mtx_t erts_dirty_bp_ix_mtx;
 static ERTS_INLINE ErtsMonotonicTime
 get_mtime(Process *c_p)
 {
-    return erts_get_monotonic_time(erts_proc_sched_data(c_p));
+    return erts_get_monotonic_time(NULL);
 }
 
 static ERTS_INLINE Uint32
 acquire_bp_sched_ix(Process *c_p)
 {
-    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ASSERT(esdp);
     if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	erts_mtx_lock(&erts_dirty_bp_ix_mtx);

erts/emulator/beam/beam_debug.c

@@ -1093,7 +1093,7 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, ErtsCodePtr I)
 	ErtsSchedulerData *esdp;
 	if (arg2 != am_type) 
 	    goto badarg;
-	esdp = erts_proc_sched_data(c_p);
+	esdp = erts_get_scheduler_data();
 	if (!esdp)
             goto scheduler_type_error;
 
@@ -1247,7 +1247,7 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, ErtsCodePtr I)
 	Eterm *hp, *hp2;
 	Uint sz;
 	int i;
-	ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
         int dirty_io = esdp->type == ERTS_SCHED_DIRTY_IO;
 
 	if (ERTS_PROC_IS_EXITING(real_c_p))
@@ -1327,7 +1327,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag)
 static int
 ms_wait(Process *c_p, Eterm etimeout, int busy)
 {
-    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsMonotonicTime time, timeout_time;
     Sint64 ms;
 

erts/emulator/beam/erl_lock_check.c

@@ -35,6 +35,7 @@
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 
+#include "sys.h"
 #include "erl_lock_check.h"
 #include "erl_term.h"
 #include "erl_threads.h"
@@ -166,7 +167,10 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"perf", 				NULL			},
     {	"jit_debug_descriptor",			NULL			},
     {	"erts_mmap",				NULL			},
-    {	"proc_sig_queue_buffer",		"address"		}
+    {	"proc_sig_queue_buffer",		"address"		},
+#ifdef ERTS_ENSURE_OS_MONOTONIC_TIME
+    {   "ensure_os_monotonic_time",             NULL                    }
+#endif
 };
 
 #define ERTS_LOCK_ORDER_SIZE \

erts/emulator/beam/erl_nif.c

@@ -452,7 +452,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp,
     env.proc->ftrace = NIL;
     env.proc->i = c_p->i;
 
-    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
 
     erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
 						   | ERTS_PSFLG_DIRTY_IO_PROC));
@@ -3270,7 +3270,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     execution_state(env, &proc, NULL);
 
     ASSERT(argc == 1);
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
     ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc);
     ASSERT(ep);
     nfunc_restore(proc, ep, argv[0]);
@@ -3291,7 +3291,7 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     execution_state(env, &proc, NULL);
 
     ASSERT(argc == 1);
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
     ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc);
     ASSERT(ep);
     exception = argv[0]; /* argv overwritten by restore below... */

erts/emulator/beam/erl_process.c

@@ -5957,6 +5957,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     erts_init_atom_cache_map(&esdp->atom_cache_map);
 
     esdp->last_monotonic_time = 0;
+#ifdef ERTS_CHECK_MONOTONIC_TIME
+    esdp->last_os_monotonic_time = ERTS_SINT64_MIN;
+#endif
     esdp->check_time_reds = 0;
 
     esdp->thr_id = (Uint32) num;
@@ -14988,7 +14991,7 @@ erts_dbg_check_halloc_lock(Process *p)
 	return 1;
     if (p->common.id == ERTS_INVALID_PID)
 	return 1;
-    esdp = erts_proc_sched_data(p);
+    esdp = erts_get_scheduler_data();
     if (esdp && p == esdp->match_pseudo_process)
 	return 1;
     /* erts_thr_progress_is_blocking() is not enough as dirty NIFs may run */

erts/emulator/beam/erl_process.h

@@ -705,6 +705,9 @@ struct ErtsSchedulerData_ {
     ErtsAtomCacheMap atom_cache_map;
 
     ErtsMonotonicTime last_monotonic_time;
+#ifdef ERTS_CHECK_MONOTONIC_TIME
+    ErtsMonotonicTime last_os_monotonic_time;
+#endif
     int check_time_reds;
 
     Uint32 thr_id;
@@ -2120,9 +2123,7 @@ Uint erts_process_memory(Process *c_p, int include_sigs_in_transit);
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
 #  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
 do {										\
-    ErtsSchedulerData *esdp__ = ((P)						\
-				 ? erts_proc_sched_data((Process *) (P))	\
-				 : erts_get_scheduler_data());			\
+    ErtsSchedulerData *esdp__ = erts_get_scheduler_data();			\
     if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
 	esdp__->verify_unused_temp_alloc(					\
 	    esdp__->verify_unused_temp_alloc_data);				\

erts/emulator/beam/erl_time.h

@@ -98,6 +98,12 @@ void erts_p_slpq(void);
 
 /* time_sup */
 
+/*
+ * If ERTS_CHECK_MONOTONIC_TIME is defined we will check that Erlang
+ * and OS monotonic time are monotonic on a per thread basis...
+ */
+#define ERTS_CHECK_MONOTONIC_TIME 1
+
 #if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
 #  ifndef HAVE_ERTS_NOW_CPU
 #    define HAVE_ERTS_NOW_CPU