From 81ad2efec3957245fc34d0ad4c73b629c443b502 Mon Sep 17 00:00:00 2001
From: Ori Livneh <ori.livneh@gmail.com>
Date: Tue, 7 Aug 2012 09:54:25 -0700
Subject: [PATCH] Add additional module for memcached

Although there is already one memcached Python module, mine approaches things
someone differently, and adds aggregated stats about the max age of items in
slabs -- metrics that are useful to us at Wikimedia and hopefully will be
elsewhere, too.
---
 memcached_maxage/README.md                    |  21 ++
 memcached_maxage/conf.d/memcached.pyconf      | 133 ++++++++
 memcached_maxage/python_modules/every.py      |  70 +++++
 memcached_maxage/python_modules/memcached.py  | 137 +++++++++
 .../python_modules/memcached_metrics.py       | 284 ++++++++++++++++++
 5 files changed, 645 insertions(+)
 create mode 100644 memcached_maxage/README.md
 create mode 100644 memcached_maxage/conf.d/memcached.pyconf
 create mode 100644 memcached_maxage/python_modules/every.py
 create mode 100644 memcached_maxage/python_modules/memcached.py
 create mode 100644 memcached_maxage/python_modules/memcached_metrics.py

diff --git a/memcached_maxage/README.md b/memcached_maxage/README.md
new file mode 100644
index 00000000..7ba8fcef
--- /dev/null
+++ b/memcached_maxage/README.md
@@ -0,0 +1,21 @@
+python-memcached-gmond
+======================
+
+
+This is a Python Gmond module for Memcached, compatible with both Python 2 and
+3. In addition to the usual datapoints provided by "stats", this module
+aggregates max age metrics from "stats items". All metrics are available in a
+"memcached" collection group.
+
+If you've installed ganglia at the standard locations, you should be able to
+install this module by copying `memcached.pyconf` to `/etc/ganglia/conf.d` and
+`memcached.py`, `memcached_metrics.py`, and 'every.py' to
+`/usr/lib/ganglia/python_modules`. The memcached server's host and port can be
+specified in the configuration in memcached.pyconf.
+
+For more information, see the section [Gmond Python metric modules][1] in the
+Ganglia documentation.
+
+Author: Ori Livneh <ori@wikimedia.org>
+
+  [1]: http://sourceforge.net/apps/trac/ganglia/wiki/ganglia_gmond_python_modules
diff --git a/memcached_maxage/conf.d/memcached.pyconf b/memcached_maxage/conf.d/memcached.pyconf
new file mode 100644
index 00000000..1552487a
--- /dev/null
+++ b/memcached_maxage/conf.d/memcached.pyconf
@@ -0,0 +1,133 @@
+# Gmond configuration for memcached metric module
+# Install to /etc/ganglia/conf.d
+
+modules {
+  module {
+    name = "memcached"
+    language = "python"
+    param host {
+      value = "127.0.0.1"
+    }
+    param port {
+      value = "11211"
+    }
+  }
+}
+
+collection_group {
+  collect_every = 10
+  time_threshold = 60
+  
+  metric {
+    name = "curr_items"
+    title = "curr_items"
+  }
+  metric {
+    name = "total_items"
+    title = "total_items"
+  }
+  metric {
+    name = "bytes"
+    title = "bytes"
+  }
+  metric {
+    name = "curr_connections"
+    title = "curr_connections"
+  }
+  metric {
+    name = "total_connections"
+    title = "total_connections"
+  }
+  metric {
+    name = "connection_structures"
+    title = "connection_structures"
+  }
+  metric {
+    name = "cmd_get"
+    title = "cmd_get"
+  }
+  metric {
+    name = "cmd_set"
+    title = "cmd_set"
+  }
+  metric {
+    name = "get_hits"
+    title = "get_hits"
+  }
+  metric {
+    name = "get_misses"
+    title = "get_misses"
+  }
+  metric {
+    name = "delete_hits"
+    title = "delete_hits"
+  }
+  metric {
+    name = "delete_misses"
+    title = "delete_misses"
+  }
+  metric {
+    name = "incr_hits"
+    title = "incr_hits"
+  }
+  metric {
+    name = "incr_misses"
+    title = "incr_misses"
+  }
+  metric {
+    name = "decr_hits"
+    title = "decr_hits"
+  }
+  metric {
+    name = "decr_misses"
+    title = "decr_misses"
+  }
+  metric {
+    name = "cas_hits"
+    title = "cas_hits"
+  }
+  metric {
+    name = "cas_misses"
+    title = "cas_misses"
+  }
+  metric {
+    name = "evictions"
+    title = "evictions"
+  }
+  metric {
+    name = "bytes_read"
+    title = "bytes_read"
+  }
+  metric {
+    name = "bytes_written"
+    title = "bytes_written"
+  }
+  metric {
+    name = "limit_maxbytes"
+    title = "limit_maxbytes"
+  }
+  metric {
+    name = "threads"
+    title = "threads"
+  }
+  metric {
+    name = "conn_yields"
+    title = "conn_yields"
+  }
+  metric {
+    name = "age_mean"
+    title = "age_mean"
+  }
+  metric {
+    name = "age_median"
+    title = "age_median"
+  }
+  metric {
+    name = "age_min"
+    title = "age_min"
+  }
+  metric {
+    name = "age_max"
+    title = "age_max"
+  }
+}
diff --git a/memcached_maxage/python_modules/every.py b/memcached_maxage/python_modules/every.py
new file mode 100644
index 00000000..117bf6ba
--- /dev/null
+++ b/memcached_maxage/python_modules/every.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Every
+
+    Python decorator; decorated function is called on a set interval.
+
+    :author: Ori Livneh <ori@wikimedia.org>
+    :copyright: (c) 2012 Wikimedia Foundation
+    :license: GPL, version 2 or later
+"""
+from __future__ import division
+from datetime import timedelta
+import signal
+import sys
+import threading
+
+
+# pylint: disable=C0111, W0212, W0613, W0621
+
+
+__all__ = ('every', )
+
+
+def total_seconds(delta):
+    """
+    Get total seconds of timedelta object. Equivalent to
+    timedelta.total_seconds(), which was introduced in Python 2.7.
+    """
+    us = (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10**6)
+    return us / 1000000.0
+
+
+def handle_sigint(signal, frame):
+    """
+    Attempt to kill all child threads and exit. Installing this as a sigint
+    handler allows the program to run indefinitely if unmolested, but still
+    terminate gracefully on Ctrl-C.
+    """
+    for thread in threading.enumerate():
+        if thread.isAlive():
+            thread._Thread__stop()
+    sys.exit(0)
+
+
+def every(*args, **kwargs):
+    """
+    Decorator; calls decorated function on a set interval. Arguments to every()
+    are passed on to the constructor of datetime.timedelta(), which accepts the
+    following arguments: days, seconds, microseconds, milliseconds, minutes,
+    hours, weeks. This decorator is intended for functions with side effects;
+    the return value is discarded.
+    """
+    interval = total_seconds(timedelta(*args, **kwargs))
+    def decorator(func):
+        def poll():
+            func()
+            threading.Timer(interval, poll).start()
+        poll()
+        return func
+    return decorator
+
+
+def join():
+    """Pause until sigint"""
+    signal.signal(signal.SIGINT, handle_sigint)
+    signal.pause()
+
+
+every.join = join
diff --git a/memcached_maxage/python_modules/memcached.py b/memcached_maxage/python_modules/memcached.py
new file mode 100644
index 00000000..a4db94de
--- /dev/null
+++ b/memcached_maxage/python_modules/memcached.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Python Gmond Module for Memcached
+
+    This module declares a "memcached" collection group. For more information,
+    including installation instructions, see:
+
+    http://sourceforge.net/apps/trac/ganglia/wiki/ganglia_gmond_python_modules
+
+    When invoked as a standalone script, this module will attempt to use the
+    default configuration to query memcached every 10 seconds and print out the
+    results.
+
+    Based on a suggestion from Domas Mitzuas, this module also reports the min,
+    max, median and mean of the 'age' metric across slabs, as reported by the
+    "stats items" memcached command.
+
+    :copyright: (c) 2012 Wikimedia Foundation
+    :author: Ori Livneh <ori@wikimedia.org>
+    :license: GPL, v2 or later
+"""
+from __future__ import division, print_function
+
+from threading import Timer
+
+import logging
+import os
+import pprint
+import sys
+import telnetlib
+
+logging.basicConfig(level=logging.DEBUG)
+
+# Hack: load a file from the current module's directory, because gmond doesn't
+# know how to work with Python packages. (To be fair, neither does Python.)
+sys.path.insert(0, os.path.dirname(__file__))
+from memcached_metrics import descriptors
+from every import every
+sys.path.pop(0)
+
+
+# Default configuration
+config = {
+    'host' : '127.0.0.1',
+    'port' : 11211,
+}
+
+stats = {}
+client = telnetlib.Telnet()
+
+
+def median(values):
+    """Calculate median of series"""
+    values = sorted(values)
+    length = len(values)
+    mid = length // 2
+    if (length % 2):
+        return values[mid]
+    else:
+        return (values[mid - 1] + values[mid]) / 2
+
+
+def mean(values):
+    """Calculate mean (average) of series"""
+    return sum(values) / len(values)
+
+
+def cast(value):
+    """Cast value to float or int, if possible"""
+    try:
+        return float(value) if '.' in value else int(value)
+    except ValueError:
+        return value
+
+
+def query(command):
+    """Send `command` to memcached and stream response"""
+    client.write(command.encode('ascii') + b'\n')
+    while True:
+        line = client.read_until(b'\r\n').decode('ascii').strip()
+        if not line or line == 'END':
+            break
+        (_, metric, value) = line.split(None, 2)
+        yield metric, cast(value)
+
+
+@every(seconds=10)
+def update_stats():
+    """Refresh stats by polling memcached server"""
+    try:
+        client.open(**config)
+        stats.update(query('stats'))
+        ages = [v for k, v in query('stats items') if k.endswith('age')]
+        if not ages:
+            return {'age_min': 0, 'age_max': 0, 'age_mean': 0, 'age_median': 0}
+        stats.update({
+            'age_min'    : min(ages),
+            'age_max'    : max(ages),
+            'age_mean'   : mean(ages),
+            'age_median' : median(ages)
+        })
+    finally:
+        client.close()
+    logging.info("Updated stats: %s", pprint.pformat(stats, indent=4))
+
+
+#
+# Gmond Interface
+#
+
+def metric_handler(name):
+    """Get the value for a particular metric; part of Gmond interface"""
+    return stats[name]
+
+
+def metric_init(params):
+    """Initialize; part of Gmond interface"""
+    print('[memcached] memcached stats')
+    config.update(params)
+    for metric in descriptors:
+        metric['call_back'] = metric_handler
+    return descriptors
+
+
+def metric_cleanup():
+    """Teardown; part of Gmond interface"""
+    client.close()
+
+
+if __name__ == '__main__':
+    # When invoked as standalone script, run a self-test by querying each
+    # metric descriptor and printing it out.
+    for metric in metric_init({}):
+        value = metric['call_back'](metric['name'])
+        print(( "%s => " + metric['format'] ) % ( metric['name'], value ))
+    every.join()
diff --git a/memcached_maxage/python_modules/memcached_metrics.py b/memcached_maxage/python_modules/memcached_metrics.py
new file mode 100644
index 00000000..9677a94d
--- /dev/null
+++ b/memcached_maxage/python_modules/memcached_metrics.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+descriptors = [ {
+        "slope": "both",
+        "time_max": 60,
+        "description": "Current number of items stored by this instance",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "curr_items"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Total number of items stored during the life of this instance",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "total_items"
+    },
+    {
+        "slope": "both",
+        "time_max": 60,
+        "description": "Current number of bytes used by this server to store items",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "bytes"
+    },
+    {
+        "slope": "both",
+        "time_max": 60,
+        "description": "Current number of open connections",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "curr_connections"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Total number of connections opened since the server started running",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "total_connections"
+    },
+    {
+        "slope": "both",
+        "time_max": 60,
+        "description": "Number of connection structures allocated by the server",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "connection_structures"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Total number of retrieval requests (get operations)",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "cmd_get"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Total number of storage requests (set operations)",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "cmd_set"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of keys that have been requested and found present",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "get_hits"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of items that have been requested and not found",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "get_misses"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of keys that have been deleted and found present",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "delete_hits"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of items that have been delete and not found",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "delete_misses"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of keys that have been incremented and found present",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "incr_hits"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of items that have been incremented and not found",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "incr_misses"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of keys that have been decremented and found present",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "decr_hits"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of items that have been decremented and not found",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "decr_misses"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of keys that have been compared and swapped and found present",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "cas_hits"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of items that have been compared and swapped and not found",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "cas_misses"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of valid items removed from cache to free memory for new items",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "evictions"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Total number of bytes read by this server from network",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "bytes_read"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Total number of bytes sent by this server to network",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "bytes_written"
+    },
+    {
+        "slope": "zero",
+        "time_max": 60,
+        "description": "Number of bytes this server is permitted to use for storage",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "limit_maxbytes"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of worker threads requested",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "threads"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Number of yields for connections",
+        "format": "%d",
+        "value_type": "uint",
+        "groups": "memcached",
+        "units": "items",
+        "name": "conn_yields"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Age of the oldest item within slabs (mean)",
+        "format": "%.2f",
+        "value_type": "float",
+        "groups": "memcached",
+        "units": "items",
+        "name": "age_mean"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Age of the oldest item within slabs (median)",
+        "format": "%.2f",
+        "value_type": "float",
+        "groups": "memcached",
+        "units": "items",
+        "name": "age_median"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "Age of the oldest item within slabs (min)",
+        "format": "%.2f",
+        "value_type": "float",
+        "groups": "memcached",
+        "units": "items",
+        "name": "age_min"
+    },
+    {
+        "slope": "positive",
+        "time_max": 60,
+        "description": "The age of the oldest item within slabs (max)",
+        "format": "%.2f",
+        "value_type": "float",
+        "groups": "memcached",
+        "units": "items",
+        "name": "age_max"
+    }
+]