initial logger code.

Logs are written to per-thread buffers. A new background thread aggregates the
logs, further processes them, then writes them to any "watchers".

Logs can have the time added to them, and all have a GID so they can be put
back into strict order.

This is an early preview. Code needs refactoring and a more complete set of
options. All watchers are also stuck viewing the global feed of logs, even if
they asked for different data.

As of this commit there's no way to toggle the "stderr" watcher.

LICENSE.bipbuffer

@@ -0,0 +1,24 @@
+Copyright (c) 2011, Willem-Hendrik Thiart
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * The names of its contributors may not be used to endorse or promote
+      products derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL WILLEM-HENDRIK THIART BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile.am

@@ -18,7 +18,9 @@ memcached_SOURCES = memcached.c memcached.h \
                     thread.c daemon.c \
                     stats.c stats.h \
                     util.c util.h \
-                    trace.h cache.h sasl_defs.h
+                    trace.h cache.h sasl_defs.h \
+                    bipbuffer.c bipbuffer.h \
+                    logger.c logger.h
 
 if BUILD_CACHE
 memcached_SOURCES += cache.c
@@ -71,7 +73,7 @@ memcached_debug_dtrace.o: $(memcached_debug_OBJECTS)
 
 SUBDIRS = doc
 DIST_DIRS = scripts
-EXTRA_DIST = doc scripts t memcached.spec memcached_dtrace.d version.m4 README.md
+EXTRA_DIST = doc scripts t memcached.spec memcached_dtrace.d version.m4 README.md LICENSE.bipbuffer
 
 MOSTLYCLEANFILES = *.gcov *.gcno *.gcda *.tcov
 

bipbuffer.c

@@ -0,0 +1,182 @@
+/**
+ * Copyright (c) 2011, Willem-Hendrik Thiart
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE.bipbuffer file.
+ *
+ * @file
+ * @author  Willem Thiart [email protected]
+ */
+
+#include "stdio.h"
+#include <stdlib.h>
+
+/* for memcpy */
+#include <string.h>
+
+#include "bipbuffer.h"
+
+static size_t bipbuf_sizeof(const unsigned int size)
+{
+    return sizeof(bipbuf_t) + size;
+}
+
+int bipbuf_unused(const bipbuf_t* me)
+{
+    if (1 == me->b_inuse)
+        /* distance between region B and region A */
+        return me->a_start - me->b_end;
+    else
+        return me->size - me->a_end;
+}
+
+int bipbuf_size(const bipbuf_t* me)
+{
+    return me->size;
+}
+
+int bipbuf_used(const bipbuf_t* me)
+{
+    return (me->a_end - me->a_start) + me->b_end;
+}
+
+void bipbuf_init(bipbuf_t* me, const unsigned int size)
+{
+    me->a_start = me->a_end = me->b_end = 0;
+    me->size = size;
+    me->b_inuse = 0;
+}
+
+bipbuf_t *bipbuf_new(const unsigned int size)
+{
+    bipbuf_t *me = malloc(bipbuf_sizeof(size));
+    if (!me)
+        return NULL;
+    bipbuf_init(me, size);
+    return me;
+}
+
+void bipbuf_free(bipbuf_t* me)
+{
+    free(me);
+}
+
+int bipbuf_is_empty(const bipbuf_t* me)
+{
+    return me->a_start == me->a_end;
+}
+
+/* find out if we should turn on region B
+ * ie. is the distance from A to buffer's end less than B to A? */
+static void __check_for_switch_to_b(bipbuf_t* me)
+{
+    if (me->size - me->a_end < me->a_start - me->b_end)
+        me->b_inuse = 1;
+}
+
+/* TODO: DOCUMENT THESE TWO FUNCTIONS */
+unsigned char *bipbuf_request(bipbuf_t* me, const int size)
+{
+    if (bipbuf_unused(me) < size)
+        return 0;
+    if (1 == me->b_inuse)
+    {
+        return (unsigned char *)me->data + me->b_end;
+    }
+    else
+    {
+        return (unsigned char *)me->data + me->a_end;
+    }
+
+    return NULL;
+}
+
+int bipbuf_push(bipbuf_t* me, const int size)
+{
+    if (bipbuf_unused(me) < size)
+        return 0;
+
+    if (1 == me->b_inuse)
+    {
+        me->b_end += size;
+    }
+    else
+    {
+        me->a_end += size;
+    }
+
+    __check_for_switch_to_b(me);
+    return size;
+}
+
+int bipbuf_offer(bipbuf_t* me, const unsigned char *data, const int size)
+{
+    /* not enough space */
+    if (bipbuf_unused(me) < size)
+        return 0;
+
+    if (1 == me->b_inuse)
+    {
+        memcpy(me->data + me->b_end, data, size);
+        me->b_end += size;
+    }
+    else
+    {
+        memcpy(me->data + me->a_end, data, size);
+        me->a_end += size;
+    }
+
+    __check_for_switch_to_b(me);
+    return size;
+}
+
+unsigned char *bipbuf_peek(const bipbuf_t* me, const unsigned int size)
+{
+    /* make sure we can actually peek at this data */
+    if (me->size < me->a_start + size)
+        return NULL;
+
+    if (bipbuf_is_empty(me))
+        return NULL;
+
+    return (unsigned char *)me->data + me->a_start;
+}
+
+unsigned char *bipbuf_peek_all(const bipbuf_t* me, unsigned int *size)
+{
+    if (bipbuf_is_empty(me))
+        return NULL;
+
+    *size = me->a_end - me->a_start;
+    return (unsigned char*)me->data + me->a_start;
+}
+
+unsigned char *bipbuf_poll(bipbuf_t* me, const unsigned int size)
+{
+    if (bipbuf_is_empty(me))
+        return NULL;
+
+    /* make sure we can actually poll this data */
+    if (me->size < me->a_start + size)
+        return NULL;
+
+    void *end = me->data + me->a_start;
+    me->a_start += size;
+
+    /* we seem to be empty.. */
+    if (me->a_start == me->a_end)
+    {
+        /* replace a with region b */
+        if (1 == me->b_inuse)
+        {
+            me->a_start = 0;
+            me->a_end = me->b_end;
+            me->b_end = me->b_inuse = 0;
+        }
+        else
+            /* safely move cursor back to the start because we are empty */
+            me->a_start = me->a_end = 0;
+    }
+
+    __check_for_switch_to_b(me);
+    return end;
+}

bipbuffer.h

@@ -0,0 +1,87 @@
+#ifndef BIPBUFFER_H
+#define BIPBUFFER_H
+
+typedef struct
+{
+    unsigned long int size;
+
+    /* region A */
+    unsigned int a_start, a_end;
+
+    /* region B */
+    unsigned int b_end;
+
+    /* is B inuse? */
+    int b_inuse;
+
+    unsigned char data[];
+} bipbuf_t;
+
+/**
+ * Create a new bip buffer.
+ *
+ * malloc()s space
+ *
+ * @param[in] size The size of the buffer */
+bipbuf_t *bipbuf_new(const unsigned int size);
+
+/**
+ * Initialise a bip buffer. Use memory provided by user.
+ *
+ * No malloc()s are performed.
+ *
+ * @param[in] size The size of the array */
+void bipbuf_init(bipbuf_t* me, const unsigned int size);
+
+/**
+ * Free the bip buffer */
+void bipbuf_free(bipbuf_t *me);
+
+/* TODO: DOCUMENTATION */
+unsigned char *bipbuf_request(bipbuf_t* me, const int size);
+int bipbuf_push(bipbuf_t* me, const int size);
+
+/**
+ * @param[in] data The data to be offered to the buffer
+ * @param[in] size The size of the data to be offered
+ * @return number of bytes offered */
+int bipbuf_offer(bipbuf_t *me, const unsigned char *data, const int size);
+
+/**
+ * Look at data. Don't move cursor
+ *
+ * @param[in] len The length of the data to be peeked
+ * @return data on success, NULL if we can't peek at this much data */
+unsigned char *bipbuf_peek(const bipbuf_t* me, const unsigned int len);
+
+/**
+ * Look at data. Don't move cursor
+ *
+ * @param[in] len The length of the data returned
+ * @return data on success, NULL if nothing available */
+unsigned char *bipbuf_peek_all(const bipbuf_t* me, unsigned int *len);
+
+/**
+ * Get pointer to data to read. Move the cursor on.
+ *
+ * @param[in] len The length of the data to be polled
+ * @return pointer to data, NULL if we can't poll this much data */
+unsigned char *bipbuf_poll(bipbuf_t* me, const unsigned int size);
+
+/**
+ * @return the size of the bipbuffer */
+int bipbuf_size(const bipbuf_t* me);
+
+/**
+ * @return 1 if buffer is empty; 0 otherwise */
+int bipbuf_is_empty(const bipbuf_t* me);
+
+/**
+ * @return how much space we have assigned */
+int bipbuf_used(const bipbuf_t* cb);
+
+/**
+ * @return bytes of unused space */
+int bipbuf_unused(const bipbuf_t* me);
+
+#endif /* BIPBUFFER_H */

items.c

@@ -841,17 +841,21 @@ static int lru_pull_tail(const int orig_id, const int cur_lru,
             case COLD_LRU:
                 it = search; /* No matter what, we're stopping */
                 if (do_evict) {
+                    logger *l;
                     if (settings.evict_to_free == 0) {
                         /* Don't think we need a counter for this. It'll OOM.  */
                         break;
                     }
+                    l = GET_LOGGER();
                     itemstats[id].evicted++;
                     itemstats[id].evicted_time = current_time - search->time;
                     if (search->exptime != 0)
                         itemstats[id].evicted_nonzero++;
                     if ((search->it_flags & ITEM_FETCHED) == 0) {
                         itemstats[id].evicted_unfetched++;
                     }
+                    if (l->log_evictions)
+                        logger_log(l, LOGGER_EVICTION, search);
                     do_item_unlink_nolock(search, hv);
                     removed++;
                     if (settings.slab_automove == 2) {