scoutapm_exporter.py

import requests
import json
import sys
import os
from time import sleep
from datetime import datetime, timedelta, timezone
from prometheus_client import start_http_server, Gauge
from scout_data import ScoutAPMAPI
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed

base_url = "https://scoutapm.com"
apikey = os.environ.get('scoutapm_apikey')
scout_email = os.environ.get("scout_email")
scout_password = os.environ.get("scout_password")

monitored_apps = os.environ.get('monitored_apps','disabled').split(',')
raw_metrics_enabled = bool(os.environ.get('raw_metrics_enabled', False))
default_metrics_enabled = bool(os.environ.get('default_metrics_enabled', True))
time_between_polls = os.environ.get("time_between_polls", 60)
prefix = os.environ.get("metric_name_prefix", "scoutapm")
port = os.environ.get("port", 8000)

headers = {
    "X-SCOUT-API": apikey
}

data = []
data_raw = []
metrics = {}

raw_api = ScoutAPMAPI(email=scout_email,password=scout_password)

def scoutapm_call(endpoint, params=None):
    result = requests.get('%s/%s' % (base_url, endpoint), headers=headers, params=params)
    if result.json()['header']['status']['code'] != 200:
        print("Error calling %s" % endpoint)
        print(json.dumps(result.json(), indent=2))
        sys.exit(1)
    return result.json()['results']

def extract_metrics(metric_info):
    app_name, app_id, metric_name = metric_info

    print("Extracting %s_%s metric for app %s" % (prefix, metric_name, app_name))
    
    # Get the time series dataset for this metric
    end = datetime.now(tz=timezone.utc)
    start = end - timedelta(minutes=5) 

    params = {
        "from": start.isoformat(),
        "to":   end.isoformat()
    }
     
    dataset = scoutapm_call('/api/v0/apps/%d/metrics/%s' % (app_id, metric_name), params=params)
    return dataset["summaries"][metric_name]

def extract_last_value_metrics(app_id):
    print(f"Extracting raw metrics for {app_id}")

    data = json.loads(raw_api.get_data(app_id=app_id))

    series_raw_metrics = {}
    for series in data['current']['series']:
        series_name = series
        series_metric = data['current']['series'][series_name]
        # we only support response_time metric right now
        if series_name == "response_time":
            series_name = f"{series_name}_detail"
            series_raw_metrics[series_name] = {}
            for metric in series_metric:
                metric_name = str(metric['name']).lower()
                try:
                    last_value = metric['data'][-1][1]
                    series_raw_metrics[series_name][metric_name.lower()] = last_value
                except:
                    logging.error(f"Can't extract latest data for app: {app_id} of {series_name} -> {metric_name}")

    return series_raw_metrics

def process_metric(metric_info):
    app_name, app_id, metric_name = metric_info
    value = extract_metrics(metric_info)
    if value is not None:
        metrics[metric_name].labels(app_name=app_name).set(value)

def process_raw_metric(metric_info):
    app_name, app_id, metric_name = metric_info
    values = extract_last_value_metrics(app_id=app_id)
    for metric in values:
        for component in values[metric]:
            if values[metric][component] is not None:
                metrics[metric_name].labels(app_name=app_name,component=component).set(values[metric_name][component])

if __name__ == '__main__':
    print("Extracting apps")
    apps = scoutapm_call('/api/v0/apps')

    print("Extracting available metrics")
    for app in apps['apps']:
        if app['name'] not in monitored_apps and monitored_apps[0] != 'disabled':
            # We don't need to extract these metrics"
            continue
        if default_metrics_enabled:
            print(f"Extracting available metrics for {app}")

            # result: {'name': 'MyPlex', 'id': 119}
            available_metrics = scoutapm_call('/api/v0/apps/%d/metrics' % app['id'])
            for metric in available_metrics['availableMetrics']:
                data.append((app['name'], app['id'], metric))
                if not metrics.get(metric):
                    metrics[metric] = Gauge("%s_%s" % (prefix,metric), metric, ['app_name'])
        
        # Get raw metrics
        if raw_metrics_enabled:
            raw_metrics = extract_last_value_metrics(app_id=app['id'])

            for metric in raw_metrics:
                data_raw.append((app['name'], app['id'], metric))
                if not metrics.get(metric):
                    metrics[metric] = Gauge("%s_%s" % (prefix,metric), metric, ['app_name','component'])

    print("Starting HTTP Server on port %s" % port)
    # Start up the server to expose the metrics.
    start_http_server(int(port))

    with ThreadPoolExecutor(max_workers=10) as executor:
        while True:
            # Parallelize the first loop
            if raw_metrics_enabled:
                futures = [executor.submit(process_raw_metric, metric_info) for metric_info in data_raw]
                for future in as_completed(futures):
                    future.result()

            # Parallelize the second loop
            if default_metrics_enabled:
                futures = [executor.submit(process_metric, metric_info) for metric_info in data]
                for future in as_completed(futures):
                    future.result()

            print("Waiting %d seconds" % time_between_polls)
            sleep(time_between_polls)