From 965d728fc9447b879c88f1c9fa64bb28b5426bce Mon Sep 17 00:00:00 2001 From: Michael Conigliaro Date: Fri, 3 Dec 2010 09:17:54 +0800 Subject: [PATCH] fix mongodb ratio reporting bug, trivial updates to diskfree and tokyo_tyrant modules --- diskfree/python_modules/diskfree.py | 26 ++--- mongodb/python_modules/mongodb.py | 113 ++++++++++++-------- tokyo_tyrant/python_modules/tokyo_tyrant.py | 68 ++++++------ 3 files changed, 118 insertions(+), 89 deletions(-) diff --git a/diskfree/python_modules/diskfree.py b/diskfree/python_modules/diskfree.py index f1fbe4ee..10837a9f 100755 --- a/diskfree/python_modules/diskfree.py +++ b/diskfree/python_modules/diskfree.py @@ -1,23 +1,24 @@ -#!/usr/bin/python +#!/usr/bin/env python +# -*- coding: utf-8 -*- import os -name_prefix = 'disk_free_' -params = { +NAME_PREFIX = 'disk_free_' +PARAMS = { 'mounts' : '/proc/mounts' } -# return a value for the requested metric def metric_handler(name): + """Return a value for the requested metric""" # parse path from name - if name == name_prefix + 'rootfs': + if name == NAME_PREFIX + 'rootfs': path = '/' else: - path = '/' + name.replace(name_prefix, '').replace('_', '/') + path = '/' + name.replace(NAME_PREFIX, '').replace('_', '/') # get fs stats try: @@ -28,18 +29,18 @@ def metric_handler(name): return (disk.f_bavail * disk.f_frsize) / 1024000000 -# initialize metric descriptors def metric_init(lparams): + """Initialize metric descriptors""" - global params + global PARAMS # set parameters for key in lparams: - params[key] = lparams[key] + PARAMS[key] = lparams[key] # read mounts file try: - f = open(params['mounts']) + f = open(PARAMS['mounts']) except IOError: return 0 @@ -56,7 +57,7 @@ def metric_init(lparams): path_key = mount_info[1][1:].replace('/', '_') descriptors.append({ - 'name': name_prefix + path_key, + 'name': NAME_PREFIX + path_key, 'call_back': metric_handler, 'time_max': 60, 'value_type': 'uint', @@ -70,8 +71,9 @@ def metric_init(lparams): return descriptors -# cleanup def metric_cleanup(): + """Cleanup""" + pass diff --git a/mongodb/python_modules/mongodb.py b/mongodb/python_modules/mongodb.py index 3ee798e4..188f3f59 100755 --- a/mongodb/python_modules/mongodb.py +++ b/mongodb/python_modules/mongodb.py @@ -1,4 +1,5 @@ -#!/usr/bin/python +#!/usr/bin/env python +# -*- coding: utf-8 -*- import json @@ -8,20 +9,21 @@ import time -name_prefix = 'mongodb_' -params = { +NAME_PREFIX = 'mongodb_' +PARAMS = { 'stats_command' : 'mongo --quiet --eval "printjson(db.serverStatus())"' } -metrics = { +METRICS = { 'time' : 0, 'values' : {} } -delta_metrics = {} -metrics_cache_max = 1 +DELTA_METRICS = {} +METRICS_CACHE_MAX = 1 -# flatten a dict (i.e. dict['a']['b']['c'] => dict['a_b_c']) def flatten(d, pre = '', sep = '_'): + """Flatten a dict (i.e. dict['a']['b']['c'] => dict['a_b_c'])""" + new_d = {} for k,v in d.items(): if type(v) == dict: @@ -31,15 +33,15 @@ def flatten(d, pre = '', sep = '_'): return new_d -# return all metrics def get_metrics(): + """Return all metrics""" - global metrics + global METRICS - if (time.time() - metrics['time']) > metrics_cache_max: + if (time.time() - METRICS['time']) > METRICS_CACHE_MAX: # get raw metric data - io = os.popen(params['stats_command']) + io = os.popen(PARAMS['stats_command']) # clean up metrics_str = ''.join(io.readlines()).strip() # convert to string @@ -49,20 +51,20 @@ def get_metrics(): fresh_metrics = flatten(json.loads(metrics_str)) # update cache - metrics['time'] = time.time() + METRICS['time'] = time.time() for name,value in fresh_metrics.items(): - metrics['values'][name] = value + METRICS['values'][name] = value - return metrics + return METRICS -# return a value for the requested metric def get_value(name): + """Return a value for the requested metric""" metrics = get_metrics() try: - name = name[len(name_prefix):] # remove prefix from name + name = name[len(NAME_PREFIX):] # remove prefix from name result = metrics['values'][name] except KeyError: result = 0 @@ -70,23 +72,23 @@ def get_value(name): return result -# return change over time for the requested metric def get_delta(name): + """Return change over time for the requested metric""" - global delta_metrics + global DELTA_METRICS # get current metrics curr_metrics = get_metrics() # get delta try: - name = name[len(name_prefix):] # remove prefix from name - delta = (curr_metrics['values'][name] - delta_metrics[name]['value'])/(curr_metrics['time'] - delta_metrics[name]['time']) + name = name[len(NAME_PREFIX):] # remove prefix from name + delta = (curr_metrics['values'][name] - DELTA_METRICS[name]['value'])/(curr_metrics['time'] - DELTA_METRICS[name]['time']) except KeyError: delta = 0 # update last metrics - delta_metrics[name] = { + DELTA_METRICS[name] = { 'value' : get_metrics()['values'][name], 'time' : get_metrics()['time'] } @@ -94,21 +96,43 @@ def get_delta(name): return delta -# initialize metric descriptors +def get_globalLock_ratio(name): + """Return the global lock ratio""" + + try: + result = get_delta(NAME_PREFIX + 'globalLock_lockTime') / get_delta(NAME_PREFIX + 'globalLock_totalTime') * 100 + except ZeroDivisionError: + result = 0 + + return result + + +def indexCounters_btree_missRatio(name): + """Return the btree miss ratio""" + + try: + result = get_delta(NAME_PREFIX + 'indexCounters_btree_misses') / get_delta(NAME_PREFIX + 'indexCounters_btree_accesses') * 100 + except ZeroDivisionError: + result = 0 + + return result + + def metric_init(lparams): + """Initialize metric descriptors""" - global params + global PARAMS # set parameters for key in lparams: - params[key] = lparams[key] + PARAMS[key] = lparams[key] # define descriptors time_max = 60 groups = 'mongodb' descriptors = [ { - 'name': name_prefix + 'opcounters_insert', + 'name': NAME_PREFIX + 'opcounters_insert', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -119,7 +143,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'opcounters_query', + 'name': NAME_PREFIX + 'opcounters_query', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -130,7 +154,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'opcounters_update', + 'name': NAME_PREFIX + 'opcounters_update', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -141,7 +165,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'opcounters_delete', + 'name': NAME_PREFIX + 'opcounters_delete', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -152,7 +176,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'opcounters_getmore', + 'name': NAME_PREFIX + 'opcounters_getmore', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -163,7 +187,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'opcounters_command', + 'name': NAME_PREFIX + 'opcounters_command', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -174,7 +198,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'backgroundFlushing_flushes', + 'name': NAME_PREFIX + 'backgroundFlushing_flushes', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -185,7 +209,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'mem_mapped', + 'name': NAME_PREFIX + 'mem_mapped', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -196,7 +220,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'mem_virtual', + 'name': NAME_PREFIX + 'mem_virtual', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -207,7 +231,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'mem_resident', + 'name': NAME_PREFIX + 'mem_resident', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -218,7 +242,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'extra_info_page_faults', + 'name': NAME_PREFIX + 'extra_info_page_faults', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -229,8 +253,8 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'globalLock_ratio', - 'call_back': get_value, + 'name': NAME_PREFIX + 'globalLock_ratio', + 'call_back': get_globalLock_ratio, 'time_max': time_max, 'value_type': 'float', 'units': '%', @@ -240,8 +264,8 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'indexCounters_btree_missRatio', - 'call_back': get_value, + 'name': NAME_PREFIX + 'indexCounters_btree_missRatio', + 'call_back': indexCounters_btree_missRatio, 'time_max': time_max, 'value_type': 'float', 'units': '%', @@ -251,7 +275,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'globalLock_currentQueue_total', + 'name': NAME_PREFIX + 'globalLock_currentQueue_total', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -262,7 +286,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'globalLock_currentQueue_readers', + 'name': NAME_PREFIX + 'globalLock_currentQueue_readers', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -273,7 +297,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'globalLock_currentQueue_writers', + 'name': NAME_PREFIX + 'globalLock_currentQueue_writers', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -284,7 +308,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'connections_current', + 'name': NAME_PREFIX + 'connections_current', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -299,8 +323,9 @@ def metric_init(lparams): return descriptors -# cleanup def metric_cleanup(): + """Cleanup""" + pass diff --git a/tokyo_tyrant/python_modules/tokyo_tyrant.py b/tokyo_tyrant/python_modules/tokyo_tyrant.py index 86e8349c..0637f320 100755 --- a/tokyo_tyrant/python_modules/tokyo_tyrant.py +++ b/tokyo_tyrant/python_modules/tokyo_tyrant.py @@ -1,4 +1,5 @@ -#!/usr/bin/python +#!/usr/bin/env python +# -*- coding: utf-8 -*- import os @@ -6,46 +7,46 @@ import time -name_prefix = 'tokyo_tyrant_' -params = { +NAME_PREFIX = 'tokyo_tyrant_' +PARAMS = { 'stats_command' : 'tcrmgr inform -st localhost' } -metrics = { +METRICS = { 'time' : 0, 'values' : {} } -delta_metrics = {} -metrics_cache_max = 1 +DELTA_METRICS = {} +METRICS_CACHE_MAX = 1 -# return all metrics def get_metrics(): + """Return all metrics""" - global metrics + global METRICS - if (time.time() - metrics['time']) > metrics_cache_max: + if (time.time() - METRICS['time']) > METRICS_CACHE_MAX: # get raw metric data - io = os.popen(params['stats_command']) + io = os.popen(PARAMS['stats_command']) # convert to list metrics_list = io.readlines() - metrics['time'] = time.time() + METRICS['time'] = time.time() for line in metrics_list: (name, value) = line.strip().split() - metrics['values'][name] = value + METRICS['values'][name] = value - return metrics + return METRICS -# return a value for the requested metric def get_value(name): + """Return a value for the requested metric""" metrics = get_metrics() try: - name = name[len(name_prefix):] # remove prefix from name + name = name[len(NAME_PREFIX):] # remove prefix from name result = metrics['values'][name] except KeyError: result = 0 @@ -53,23 +54,23 @@ def get_value(name): return result -# return change over time for the requested metric def get_delta(name): + """Return change over time for the requested metric""" - global delta_metrics + global DELTA_METRICS # get current metrics curr_metrics = get_metrics() # get delta try: - name = name[len(name_prefix):] # remove prefix from name - delta = (float(curr_metrics['values'][name]) - float(delta_metrics[name]['value']))/(curr_metrics['time'] - delta_metrics[name]['time']) + name = name[len(NAME_PREFIX):] # remove prefix from name + delta = (float(curr_metrics['values'][name]) - float(DELTA_METRICS[name]['value']))/(curr_metrics['time'] - DELTA_METRICS[name]['time']) except KeyError: delta = 0 # update last metrics - delta_metrics[name] = { + DELTA_METRICS[name] = { 'value' : get_metrics()['values'][name], 'time' : get_metrics()['time'] } @@ -77,21 +78,21 @@ def get_delta(name): return delta -# initialize metric descriptors def metric_init(lparams): + """Initialize metric descriptors""" - global params + global PARAMS # set parameters for key in lparams: - params[key] = lparams[key] + PARAMS[key] = lparams[key] # define descriptors time_max = 60 groups = 'tokyo tyrant' descriptors = [ { - 'name': name_prefix + 'rnum', + 'name': NAME_PREFIX + 'rnum', 'call_back': get_value, 'time_max': time_max, 'value_type': 'uint', @@ -102,7 +103,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'size', + 'name': NAME_PREFIX + 'size', 'call_back': get_value, 'time_max': time_max, 'value_type': 'double', @@ -113,7 +114,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'delay', + 'name': NAME_PREFIX + 'delay', 'call_back': get_value, 'time_max': time_max, 'value_type': 'float', @@ -124,7 +125,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'cnt_put', + 'name': NAME_PREFIX + 'cnt_put', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -135,7 +136,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'cnt_out', + 'name': NAME_PREFIX + 'cnt_out', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -146,7 +147,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'cnt_get', + 'name': NAME_PREFIX + 'cnt_get', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -157,7 +158,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'cnt_put_miss', + 'name': NAME_PREFIX + 'cnt_put_miss', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -168,7 +169,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'cnt_out_miss', + 'name': NAME_PREFIX + 'cnt_out_miss', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -179,7 +180,7 @@ def metric_init(lparams): 'groups': groups }, { - 'name': name_prefix + 'cnt_get_miss', + 'name': NAME_PREFIX + 'cnt_get_miss', 'call_back': get_delta, 'time_max': time_max, 'value_type': 'float', @@ -194,8 +195,9 @@ def metric_init(lparams): return descriptors -# cleanup def metric_cleanup(): + """Cleanup""" + pass