[mem] move memory free to go routine. add shorten-key-table flag

src/cmd/cmd_query.go

@@ -67,6 +67,7 @@ func addQueryFlags() {
 
 	flag.BoolVar(&sybil.FLAGS.RECYCLE_MEM, "recycle-mem", true, "recycle memory slabs (versus using Go's GC)")
 	flag.BoolVar(&sybil.FLAGS.FAST_RECYCLE, "fast-recycle", false, "faster memory recycling")
+	flag.BoolVar(&sybil.FLAGS.SHORTEN_KEY_TABLE, "shorten-key-table", false, "faster queries by shortening the key table")
 
 	flag.BoolVar(&sybil.FLAGS.CACHED_QUERIES, "cache-queries", false, "Cache query results per block")
 
@@ -171,7 +172,7 @@ func runQueryCmdLine() {
 	sybil.Debug("WILL INSPECT", count, "RECORDS")
 
 	// {{ TRIM KEY TABLE
-	shorten_key_table := true
+	shorten_key_table := sybil.FLAGS.SHORTEN_KEY_TABLE
 	if sybil.FLAGS.PRINT_INFO {
 		shorten_key_table = false
 	}

src/lib/config.go

@@ -73,9 +73,10 @@ type FlagDefs struct {
 	PROFILE     bool
 	PROFILE_MEM bool
 
-	RECYCLE_MEM    bool
-	FAST_RECYCLE   bool
-	CACHED_QUERIES bool
+	RECYCLE_MEM       bool
+	FAST_RECYCLE      bool
+	CACHED_QUERIES    bool
+	SHORTEN_KEY_TABLE bool
 
 	WEIGHT_COL string
 

src/lib/table_query.go

@@ -13,6 +13,7 @@ import (
 )
 
 var MAX_MEM = uint64(1024)
+var FREE_MEM_AFTER = uint64(1024)
 
 func (t *Table) LoadAndQueryRecords(loadSpec *LoadSpec, querySpec *QuerySpec) int {
 	waystart := time.Now()
@@ -278,17 +279,29 @@ func (t *Table) LoadAndQueryRecords(loadSpec *LoadSpec, querySpec *QuerySpec) in
 					}
 					// }}}
 
+					// {{{ Freeing memory back to the OS
+					// We defer the free so that not all threads are halted while we
+					// free. We also schedule the next collection at alloced_mem +
+					// allowed overhead
 					runtime.ReadMemStats(&memstats)
 					alloced := memstats.Alloc / 1024 / 1024
 					if alloced > max_alloc {
 						max_alloc = alloced
 					}
 
-					os_free_start := time.Now()
 					if alloced > MAX_MEM {
-						debug.FreeOSMemory()
+						wg.Add(1)
+						go func() {
+							os_free_start := time.Now()
+							debug.FreeOSMemory()
+							runtime.ReadMemStats(&memstats)
+							after_free := memstats.Alloc / 1024 / 1024
+							MAX_MEM = after_free + FREE_MEM_AFTER
+							os_free_time += time.Now().Sub(os_free_start)
+							wg.Done()
+						}()
 					}
-					os_free_time += time.Now().Sub(os_free_start)
+					// }}} end free memory
 				}
 
 				if FLAGS.DEBUG {

src/lib/testdata/TestDecodeGoldenFiles/flag_defs.golden.json.b64

@@ -1 +1 @@
-eyJPUCI6ImF2ZyIsIlBSSU5UIjp0cnVlLCJFWFBPUlQiOmZhbHNlLCJMSVNUX1RBQkxFUyI6ZmFsc2UsIkRFQ09ERV9GTEFHUyI6ZmFsc2UsIkVOQ09ERV9GTEFHUyI6ZmFsc2UsIkVOQ09ERV9SRVNVTFRTIjpmYWxzZSwiSU5UX0ZJTFRFUlMiOiIiLCJTVFJfRklMVEVSUyI6IiIsIlNUUl9SRVBMQUNFIjoiIiwiU0VUX0ZJTFRFUlMiOiIiLCJJTlRTIjoiZm9vLGJhciIsIlNUUlMiOiIiLCJTRVRTIjoiIiwiU0FNUExFX0NPTFMiOiIiLCJHUk9VUFMiOiJhLGIsYyIsIkRJU1RJTkNUIjoiIiwiQUREX1JFQ09SRFMiOjAsIlRJTUUiOmZhbHNlLCJUSU1FX0NPTCI6InRpbWUiLCJUSU1FX0JVQ0tFVCI6MzYwMCwiSElTVF9CVUNLRVQiOjAsIkhEUl9ISVNUIjpmYWxzZSwiTE9HX0hJU1QiOmZhbHNlLCJUX0RJR0VTVCI6ZmFsc2UsIkZJRUxEX1NFUEFSQVRPUiI6IiwiLCJGSUxURVJfU0VQQVJBVE9SIjoiOiIsIlBSSU5UX0tFWVMiOmZhbHNlLCJMT0FEX0FORF9RVUVSWSI6dHJ1ZSwiTE9BRF9USEVOX1FVRVJZIjpmYWxzZSwiUkVBRF9JTkdFU1RJT05fTE9HIjpmYWxzZSwiUkVBRF9ST1dTVE9SRSI6ZmFsc2UsIlNLSVBfQ09NUEFDVCI6ZmFsc2UsIlNBVkVfQVNfU1JCIjpmYWxzZSwiUFJPRklMRSI6ZmFsc2UsIlBST0ZJTEVfTUVNIjpmYWxzZSwiUkVDWUNMRV9NRU0iOnRydWUsIkZBU1RfUkVDWUNMRSI6ZmFsc2UsIkNBQ0hFRF9RVUVSSUVTIjpmYWxzZSwiV0VJR0hUX0NPTCI6IiIsIkxJTUlUIjoxMDAsIk5VTV9ESVNUSU5DVCI6MCwiREVCVUciOmZhbHNlLCJKU09OIjpmYWxzZSwiR0MiOnRydWUsIkRJUiI6Ii4vZGIvIiwiU09SVCI6IiRDT1VOVCIsIlBSVU5FX0JZIjoiJENPVU5UIiwiVEFCTEUiOiJ0ZXN0YWJsZSIsIlBSSU5UX0lORk8iOmZhbHNlLCJTQU1QTEVTIjpmYWxzZSwiVVBEQVRFX1RBQkxFX0lORk8iOmZhbHNlLCJTS0lQX09VVExJRVJTIjp0cnVlfQ==
+eyJPUCI6ImF2ZyIsIlBSSU5UIjp0cnVlLCJFWFBPUlQiOmZhbHNlLCJMSVNUX1RBQkxFUyI6ZmFsc2UsIkRFQ09ERV9GTEFHUyI6ZmFsc2UsIkVOQ09ERV9GTEFHUyI6ZmFsc2UsIkVOQ09ERV9SRVNVTFRTIjpmYWxzZSwiSU5UX0ZJTFRFUlMiOiIiLCJTVFJfRklMVEVSUyI6IiIsIlNUUl9SRVBMQUNFIjoiIiwiU0VUX0ZJTFRFUlMiOiIiLCJJTlRTIjoiZm9vLGJhciIsIlNUUlMiOiIiLCJTRVRTIjoiIiwiU0FNUExFX0NPTFMiOiIiLCJHUk9VUFMiOiJhLGIsYyIsIkRJU1RJTkNUIjoiIiwiQUREX1JFQ09SRFMiOjAsIlRJTUUiOmZhbHNlLCJUSU1FX0NPTCI6InRpbWUiLCJUSU1FX0JVQ0tFVCI6MzYwMCwiSElTVF9CVUNLRVQiOjAsIkhEUl9ISVNUIjpmYWxzZSwiTE9HX0hJU1QiOmZhbHNlLCJUX0RJR0VTVCI6ZmFsc2UsIkZJRUxEX1NFUEFSQVRPUiI6IiwiLCJGSUxURVJfU0VQQVJBVE9SIjoiOiIsIlBSSU5UX0tFWVMiOmZhbHNlLCJMT0FEX0FORF9RVUVSWSI6dHJ1ZSwiTE9BRF9USEVOX1FVRVJZIjpmYWxzZSwiUkVBRF9JTkdFU1RJT05fTE9HIjpmYWxzZSwiUkVBRF9ST1dTVE9SRSI6ZmFsc2UsIlNLSVBfQ09NUEFDVCI6ZmFsc2UsIlNBVkVfQVNfU1JCIjpmYWxzZSwiUFJPRklMRSI6ZmFsc2UsIlBST0ZJTEVfTUVNIjpmYWxzZSwiUkVDWUNMRV9NRU0iOnRydWUsIkZBU1RfUkVDWUNMRSI6ZmFsc2UsIkNBQ0hFRF9RVUVSSUVTIjpmYWxzZSwiU0hPUlRFTl9LRVlfVEFCTEUiOmZhbHNlLCJXRUlHSFRfQ09MIjoiIiwiTElNSVQiOjEwMCwiTlVNX0RJU1RJTkNUIjowLCJERUJVRyI6ZmFsc2UsIkpTT04iOmZhbHNlLCJHQyI6dHJ1ZSwiRElSIjoiLi9kYi8iLCJTT1JUIjoiJENPVU5UIiwiUFJVTkVfQlkiOiIkQ09VTlQiLCJUQUJMRSI6InRlc3RhYmxlIiwiUFJJTlRfSU5GTyI6ZmFsc2UsIlNBTVBMRVMiOmZhbHNlLCJVUERBVEVfVEFCTEVfSU5GTyI6ZmFsc2UsIlNLSVBfT1VUTElFUlMiOnRydWV9