Initial commit

Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gemspec
+

README.md

@@ -0,0 +1,29 @@
+= Splunk output plugin for Fluent event collector
+
+== Overview
+
+This plugin will send your fluentd logs to a splunk server.  It can send the data in either
+key/value (k1=v1 k2=v2) or json format for easy splunk parsing.
+
+
+== Installation
+
+    gem install fluent-plugin-splunk-ex
+
+== Configuration
+
+    <match pattern>
+      type splunk_ex
+      host <splunk_host>   # default: localhost
+      port <splunk_port>   # default: 9997
+      use_time <boolean>   # default: false
+      time_key <string>    # default: time
+      format kv|json       # default: kv
+    </match>
+
+== Copyright
+
+Copyright:: Copyright (c) 2014 Trevor Gattis
+License::   Apache License, Version 2.0
+
+

Rakefile

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+    sh 'irb -rubygems -I lib'
+end
+task :c => :console
+

fluent-plugin-splunk-ex.gemspec

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+Gem::Specification.new do |gem|
+  gem.name         = "fluent-plugin-splunk-ex"
+  gem.version      = "0.0.1"
+
+  gem.authors      = ["Trevor Gattis"]
+  gem.email        = "github@trevorgattis.com"
+  gem.description  = "Splunk output plugin for Fluent event collector"
+  gem.homepage     = "https://github.com/ggatti000/fluent-plugin-splunk-ex"
+  gem.summary      = gem.description
+  gem.license      = "APLv2"
+  gem.has_rdoc = false
+
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_runtime_dependency "json"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
+end
+

lib/fluent/plugin/out_splunk_ex.rb

@@ -0,0 +1,87 @@
+require 'open-uri'
+require 'json'
+
+class Fluent::SplunkExOutput < Fluent::Output
+  Fluent::Plugin.register_output('splunk_ex', self)
+
+  config_param :host, :string, :default => 'localhost'
+  config_param :port, :string, :default => 9997
+  config_param :format, :string, :default => 'kv'
+  config_param :use_time, :bool, :default => false
+  config_param :time_key, :string, :default => 'time'
+
+  config_param :test_mode, :bool, :default => false
+
+  # To support log_level option implemented by Fluentd v0.10.43
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    super
+
+    if @format != 'json'
+      @format_fn = self.class.method(:format_kv)
+    else
+      @format_fn = self.class.method(:format_json)
+    end
+
+    @format_fn.call({"more" => "stuff"})
+
+    if test_mode
+      @send_data = proc {|text| log.info("test mode text: #{text}") }
+    else
+      @splunk_connection = TCPSocket.open(@host, @port)
+      @send_data = self.class.splunk_send
+    end
+  end
+
+
+  def shutdown
+    super
+    if !test_mode
+      @splunk_connection.close
+    end
+  end
+
+
+  def emit(tag, es, chain)
+    chain.next
+    es.each {|time,record|
+      if @use_time
+        record.merge!({@time_key => Time.at(time).to_datetime.to_s})
+      end
+
+      @send_data.call( @format_fn.call(record) )
+    }
+  end
+
+  # =================================================================
+
+  protected
+
+  def self.format_kv(record)
+    kv_out_arr = []
+    record.each { |k, v|
+      kv_out_arr << sprintf('%s=%s', URI::encode(k), URI::encode(v.to_s))
+    }
+
+    kv_out_str = kv_out_arr.join(' ')
+  end
+
+  def self.format_json(record)
+    json_str = record.to_json
+  end
+
+  def self.splunk_send(text)
+    log.debug("splunk text: #{text}")
+    @splunk_connection.puts(text)
+  end
+
+
+end
+

spec/out_splunk_ex_spec.rb

@@ -0,0 +1,78 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+require 'benchmark'
+Fluent::Test.setup
+
+def create_driver(config, tag = 'foo.bar')
+  Fluent::Test::OutputTestDriver.new(Fluent::SplunkExOutput, tag).configure(config)
+end
+
+# setup
+single_key_message = {
+  'msg' => 'testing some data'
+}
+
+multi_key_message = {
+  'msg'            => 'testing some data',
+  'chars'          => 'let"s put !@#$%^&*()-= some weird :\'?><,./ characters',
+  'dt'             => '2014/04/03T07:02:11.124202',
+  'debug_line'     => 24,
+  'debug_file'     => '/some/path/to/myFile.py',
+  'statsd_key'     => 'fluent_plugin_splunk_ex',
+  'statsd_timing'  => 0.234,
+  'statsd_type'    => 'timing',
+  'tx'             => '280c3e80-bb6c-11e3-a5e2-0800200c9a66',
+  'host'           => 'my01.cool.server.com'
+}
+
+time = Time.now.to_i
+
+driver_kv = create_driver(%[
+  log_level fatal
+
+  test_mode true
+  format kv
+])
+
+driver_json = create_driver(%[
+  log_level fatal
+
+  test_mode true
+  format json
+])
+
+driver_kv_time = create_driver(%[
+  log_level fatal
+
+  test_mode true
+  format kv
+  time_key myKey
+  use_time true
+])
+
+driver_json_time = create_driver(%[
+  log_level fatal
+
+  test_mode true
+  format json
+  time_key myKey
+  use_time true
+])
+
+
+
+# bench
+n = 10000
+Benchmark.bm(7) do |x|
+  x.report("single_kv       ") { driver_kv.run        { n.times { driver_kv.emit(       single_key_message, time) } } }
+  x.report("single_kv_time  ") { driver_kv_time.run   { n.times { driver_kv_time.emit(  single_key_message, time) } } }
+  x.report("single_json     ") { driver_json.run      { n.times { driver_json.emit(     single_key_message, time) } } }
+  x.report("single_json_time") { driver_json_time.run { n.times { driver_json_time.emit(single_key_message, time) } } }
+
+  x.report("multi_kv        ") { driver_kv.run        { n.times { driver_kv.emit(       multi_key_message, time ) } } }
+  x.report("multi_kv_time   ") { driver_kv_time.run   { n.times { driver_kv_time.emit(  multi_key_message, time ) } } }
+  x.report("multi_json      ") { driver_json.run      { n.times { driver_json.emit(     multi_key_message, time ) } } }
+  x.report("multi_json_time ") { driver_json_time.run { n.times { driver_json_time.emit(multi_key_message, time ) } } }
+
+end
+

spec/spec_helper.rb

@@ -0,0 +1,22 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+require 'fluent/test'
+
+require 'fluent/plugin/out_splunk_ex'
+