[IMP] registry: adapt LRU sizes for registries and ormcache

The registry size is now assumed to be around 10Mb, and ormcache size is
proportional to the maximum number of registries.  Statistics about ormcache
usage is shown to the log when receiving signal SIGUSR1.

openerp/modules/registry.py

@@ -305,8 +305,9 @@ def registries(cls):
                     # cannot specify the memory limit soft on windows...
                     size = 42
                 else:
-                    # On average, a clean registry take 25MB of memory + cache
-                    avgsz = 30 * 1024 * 1024
+                    # A registry takes 10MB of memory on average, so we reserve
+                    # 10Mb (registry) + 5Mb (working memory) per registry
+                    avgsz = 15 * 1024 * 1024
                     size = int(config['limit_memory_soft'] / avgsz)
 
             cls._registries = LRU(size)
@@ -319,8 +320,8 @@ def cache(cls):
         """
         with cls.lock():
             if cls._cache is None:
-                # we allocate one cache entry per 32KB of memory
-                size = max(8192, int(config['limit_memory_soft'] / 32768))
+                # we allocate 8192 cache entries per registry
+                size = 8192 * cls.registries.count
                 cls._cache = LRU(size)
             return cls._cache
 

openerp/service/server.py

@@ -38,7 +38,7 @@
 from openerp.modules.registry import RegistryManager
 from openerp.release import nt_service_name
 import openerp.tools.config as config
-from openerp.tools.misc import stripped_sys_argv, dumpstacks
+from openerp.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats
 
 _logger = logging.getLogger(__name__)
 
@@ -296,6 +296,7 @@ def start(self, stop=False):
             signal.signal(signal.SIGCHLD, self.signal_handler)
             signal.signal(signal.SIGHUP, self.signal_handler)
             signal.signal(signal.SIGQUIT, dumpstacks)
+            signal.signal(signal.SIGUSR1, log_ormcache_stats)
         elif os.name == 'nt':
             import win32api
             win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1)
@@ -389,6 +390,7 @@ def start(self):
 
         if os.name == 'posix':
             signal.signal(signal.SIGQUIT, dumpstacks)
+            signal.signal(signal.SIGUSR1, log_ormcache_stats)
 
         gevent.spawn(self.watch_parent)
         self.httpd = WSGIServer((self.interface, self.port), self.app)
@@ -510,6 +512,9 @@ def process_signals(self):
             elif sig == signal.SIGQUIT:
                 # dump stacks on kill -3
                 self.dumpstacks()
+            elif sig == signal.SIGUSR1:
+                # log ormcache stats on kill -SIGUSR1
+                log_ormcache_stats()
             elif sig == signal.SIGTTIN:
                 # increase number of workers
                 self.population += 1
@@ -586,6 +591,7 @@ def start(self):
         signal.signal(signal.SIGTTIN, self.signal_handler)
         signal.signal(signal.SIGTTOU, self.signal_handler)
         signal.signal(signal.SIGQUIT, dumpstacks)
+        signal.signal(signal.SIGUSR1, log_ormcache_stats)
 
         # listen to socket
         self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

openerp/tools/cache.py

@@ -27,7 +27,7 @@
 import lru
 import logging
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 class ormcache(object):
@@ -47,9 +47,9 @@ def __call__(self, method):
         return lookup
 
     def stat(self):
-        return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % \
+        return "lookup-stats %6d hit %6d miss %6d err %4.1f ratio" % \
             (self.stat_hit, self.stat_miss, self.stat_err,
-                (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit))
+                (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit or 1))
 
     def lru(self, model):
         return model.pool.cache, (model.pool.db_name, model._name, self.method)
@@ -73,9 +73,9 @@ def clear(self, model, *args):
         """ Remove *args entry from the cache or all keys if *args is undefined """
         d, key0 = self.lru(model)
         if args:
-            logger.warn("ormcache.clear arguments are deprecated and ignored "
-                        "(while clearing caches on (%s).%s)",
-                        model._name, self.method.__name__)
+            _logger.warn("ormcache.clear arguments are deprecated and ignored "
+                         "(while clearing caches on (%s).%s)",
+                         model._name, self.method.__name__)
         d.clear_prefix(key0)
         model.pool._any_cache_cleared = True
 
@@ -168,6 +168,25 @@ def clear(self, *l, **kw):
         pass
 
 
+def log_ormcache_stats(sig=None, frame=None):
+    """ Log statistics of ormcache usage by database, model, and method. """
+    from openerp.modules.registry import RegistryManager
+    from collections import defaultdict
+    import threading
+
+    me = threading.currentThread()
+    entries = defaultdict(int)
+    for key in RegistryManager.cache.iterkeys():
+        entries[key[:3]] += 1
+    for (dbname, model_name, method), count in sorted(entries.items()):
+        me.dbname = dbname
+        model = RegistryManager.get(dbname)[model_name]
+        func = getattr(model, method.__name__).im_func
+        ormcache = func.clear_cache.im_self
+        _logger.info("%6d entries, %s, for %s.%s",
+            count, ormcache.stat(), model_name, method.__name__)
+
+
 # For backward compatibility
 cache = ormcache