Merge multithreaded into trunk, commit memcached#2 (first commit only…

… did the

new files, not the modified ones.)


git-svn-id: http://code.sixapart.com/svn/memcached/trunk/server@509 b0b603af-a30f-0410-a34e-baf09ae79d0b

ChangeLog

@@ -69,6 +69,17 @@
 
 	* Explicitly compare against NULL or zero in many places.
 
+2007-03-05
+	* Steven Grimm <[email protected]>: Per-object-type stats collection
+	  support. Specify the object type delimiter with the -D command line
+	  option. Turn stats gathering on and off with "stats detail on" and
+	  "stats detail off". Dump the per-object-type details with
+	  "stats detail dump".
+
+2007-03-01
+	* Steven Grimm <[email protected]>: Fix an off-by-one error in the
+	  multithreaded version's message passing code.
+
 2006-12-23
 	* fix expirations of items set with absolute expiration times in
 	  the past, before the server's start time.  bug was introduced in
@@ -97,10 +108,20 @@
 	* Steve Peters <[email protected]>: OpenBSD has a malloc.h,
 	but warns to use stdlib.h instead
 
+2006-11-22
+	* Steven Grimm <[email protected]>: Add support for multithreaded
+	  execution. Run configure with "--enable-threads" to enable. See
+	  doc/threads.txt for details.
+
 2006-11-13
 	* Iain Wade <[email protected]>: Fix for UDP responses on non-"get"
 	 commands.
 
+2006-10-15
+	* Steven Grimm <[email protected]>: Dynamic sizing of hashtable to
+	  reduce collisions on very large caches and conserve memory on
+	  small caches.
+
 2006-10-13
 	* Steven Grimm <[email protected]>: New faster hash function.
 

Makefile.am

@@ -1,6 +1,6 @@
 bin_PROGRAMS = memcached memcached-debug
 
-memcached_SOURCES = memcached.c slabs.c slabs.h items.c items.h assoc.c assoc.h memcached.h 
+memcached_SOURCES = memcached.c slabs.c slabs.h items.c items.h assoc.c assoc.h memcached.h thread.c stats.c stats.h
 memcached_debug_SOURCES = $(memcached_SOURCES)
 memcached_CPPFLAGS = -DNDEBUG
 memcached_LDADD = @LIBOBJS@

assoc.c

@@ -12,10 +12,9 @@
  *
  * $Id$
  */
-#include "config.h"
-#include <sys/types.h>
+
+#include "memcached.h"
 #include <sys/stat.h>
-#include <sys/time.h>
 #include <sys/socket.h>
 #include <sys/signal.h>
 #include <sys/resource.h>
@@ -24,13 +23,9 @@
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
-#include <netinet/in.h>
 #include <errno.h>
-#include <event.h>
 #include <assert.h>
 
-#include "memcached.h"
-
 /*
  * Since the hash function does bit manipulation, it needs to know
  * whether it's big or little-endian. ENDIAN_LITTLE and ENDIAN_BIG
@@ -142,7 +137,7 @@ and these came close:
 }
 
 #if HASH_LITTLE_ENDIAN == 1
-static uint32_t hash(
+uint32_t hash(
   const void *key,       /* the key to hash */
   size_t      length,    /* length of the key */
   const uint32_t    initval)   /* initval */
@@ -323,7 +318,7 @@ static uint32_t hash(
  * from hashlittle() on all machines.  hashbig() takes advantage of
  * big-endian byte ordering.
  */
-static uint32_t hash( const void *key, size_t length, const uint32_t initval)
+uint32_t hash( const void *key, size_t length, const uint32_t initval)
 {
   uint32_t a,b,c;
   union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */
@@ -541,39 +536,41 @@ static void assoc_expand(void) {
 
     primary_hashtable = calloc(hashsize(hashpower + 1), sizeof(void *));
     if (primary_hashtable) {
-    if (settings.verbose > 1)
-        fprintf(stderr, "Hash table expansion starting\n");
+        if (settings.verbose > 1)
+            fprintf(stderr, "Hash table expansion starting\n");
         hashpower++;
         expanding = 1;
         expand_bucket = 0;
-    assoc_move_next_bucket();
+        do_assoc_move_next_bucket();
     } else {
         primary_hashtable = old_hashtable;
-    /* Bad news, but we can keep running. */
+        /* Bad news, but we can keep running. */
     }
 }
 
 /* migrates the next bucket to the primary hashtable if we're expanding. */
-void assoc_move_next_bucket(void) {
+void do_assoc_move_next_bucket(void) {
     item *it, *next;
     int bucket;
 
     if (expanding) {
         for (it = old_hashtable[expand_bucket]; NULL != it; it = next) {
-        next = it->h_next;
+            next = it->h_next;
 
             bucket = hash(ITEM_key(it), it->nkey, 0) & hashmask(hashpower);
             it->h_next = primary_hashtable[bucket];
             primary_hashtable[bucket] = it;
-    }
+        }
 
-    expand_bucket++;
-    if (expand_bucket == hashsize(hashpower - 1)) {
-        expanding = 0;
-        free(old_hashtable);
-        if (settings.verbose > 1)
-            fprintf(stderr, "Hash table expansion done\n");
-    }
+        old_hashtable[expand_bucket] = NULL;
+
+        expand_bucket++;
+        if (expand_bucket == hashsize(hashpower - 1)) {
+            expanding = 0;
+            free(old_hashtable);
+            if (settings.verbose > 1)
+                fprintf(stderr, "Hash table expansion done\n");
+        }
     }
 }
 

assoc.h

@@ -3,4 +3,5 @@ void assoc_init(void);
 item *assoc_find(const char *key, const size_t nkey);
 int assoc_insert(item *item);
 void assoc_delete(const char *key, const size_t nkey);
-void assoc_move_next_bucket(void);
+void do_assoc_move_next_bucket(void);
+uint32_t hash( const void *key, size_t length, const uint32_t initval);

configure.ac

@@ -94,6 +94,7 @@ dnl ----------------------------------------------------------------------------
 AC_SEARCH_LIBS(socket, socket)
 AC_SEARCH_LIBS(gethostbyname, nsl)
 AC_SEARCH_LIBS(mallinfo, malloc)
+AC_SEARCH_LIBS(pthread_create, pthread)
 
 AC_CHECK_FUNC(daemon,AC_DEFINE([HAVE_DAEMON],,[Define this if you have daemon()]),[AC_LIBOBJ(daemon)])
 
@@ -156,6 +157,15 @@ fi
 
 AC_C_ENDIAN
 
+dnl Check whether the user wants threads or not
+AC_ARG_ENABLE(threads,
+  [AS_HELP_STRING([--enable-threads],[support multithreaded execution])],
+  [if test "$ac_cv_search_pthread_create" != "no"; then
+    AC_DEFINE([USE_THREADS],,[Define this if you want to use pthreads])
+   else
+    AC_MSG_ERROR([Can't enable threads without the POSIX thread library.])
+   fi])
+
 AC_CHECK_FUNCS(mlockall)
 
 AC_CONFIG_FILES(Makefile doc/Makefile)

doc/memcached.1

@@ -84,6 +84,18 @@ Print memcached and libevent licenses.
 .TP
 .B \-P <filename>
 Print pidfile to <filename>, only used under -d option.
+.TP
+.B \-t <threads>
+Number of threads to use to process incoming requests. This option is only
+meaningful if memcached was compiled with thread support enabled. It is 
+typically not useful to set this higher than the number of CPU cores on the
+memcached server.
+.TP
+.B \-D <char>
+Use <char> as the delimiter between key prefixes and IDs. This is used for
+per-prefix stats reporting. The default is ":" (colon). If this option is
+specified, stats collection is turned on automatically; if not, then it may
+be turned on by sending the "stats detail on" command to the server.
 .br
 .SH LICENSE
 The memcached daemon is copyright Danga Interactive and is distributed under