emails not sending with both mailer.rb and polymailer.rb


#1

HI I can’t get either ponymailer nor mailer to send an email out. I have sendmail installed and was able to send an email with mail -s. Below is what I have confgured for ponymailer. Any guideline is welcome.
Thanks,
Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)
amqp (1.5.0)
async_sinatra (1.2.0)
bigdecimal (1.2.0)
childprocess (0.5.3)
daemons (1.2.2)
em-redis-unified (1.0.0)
em-worker (0.0.2)
eventmachine (1.0.3)
ffi (1.9.8)
io-console (0.4.2)
ipaddress (0.8.0)
json (1.7.7)
mail (2.5.4)
mime-types (2.6.1, 1.25.1)
minitest (4.3.2)
mixlib-cli (1.5.0)
mixlib-config (2.2.1)
mixlib-log (1.6.0)
mixlib-shellout (2.1.0)
multi_json (1.11.0)
ohai (6.16.0)
polyglot (0.3.5)
pony (1.11)
psych (2.0.0)
rack (1.6.1)
rack-protection (1.5.3)
rake (0.9.6)
rdoc (4.0.0)
sensu (0.19.2)
sensu-em (2.5.2)
sensu-extension (1.1.2)
sensu-extensions (1.2.0)
sensu-logger (1.0.0)
sensu-plugin (1.1.0)
sensu-settings (1.9.0)
sensu-spawn (1.2.0)
sensu-transport (3.0.0)
sinatra (1.4.6)
systemu (2.6.5, 2.5.2)
test-unit (2.0.0.0)
thin (1.6.3)
tilt (2.0.1)
treetop (1.4.15)
uuidtools (2.1.5)
yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb
#!/usr/bin/env ruby

···

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a pre-defined recipient.

Copyright 2012 Panagiotis Papadomitsos pj@ezgr.net

Released under the same terms as Sensu (the MIT license); see LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’
require ‘sensu-handler’
require ‘timeout’
require ‘pony’
require ‘date’

class PonyMailer < Sensu::Handler
STATUSES = {
0 => ‘OK’,
1 => ‘WARNING’,
2 => ‘CRITICAL’
}

def short_name
@event[‘client’][‘name’] + ‘/’ + @event[‘check’][‘name’]
end

def action_to_string
@event[‘action’].eql?(‘resolve’) ? ‘RESOLVED’ : ‘ALERT’
end

def handle
if settings[‘ponymailer’][‘only_send_on_change’]
last_status = @event[‘check’][‘history’][-2].to_i
if last_status == @event[‘check’][‘status’]
return
end
end

mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} :: #{short_name}",
  from: "#{settings['ponymailer']['fromname']} <#{settings['ponymailer']['from']}>",
  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address: settings['ponymailer']['hostname'],
                      port: settings['ponymailer']['port'],
                      enable_starttls_auto: settings['ponymailer']['tls'],
                      user_name: settings['ponymailer']['username'],
                      password: settings['ponymailer']['password'],
                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']

mail_options[:body] = %(Sensu has detected a failed check. Event analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}
Check That Failed: #{@event[‘check’][‘name’]}
Check Command: #{@event[‘check’][‘command’]}
Check Flapping: #{@event[‘check’][‘flapping’]}
Check Occurrences: #{@event[‘occurrences’]}
Check History: #{@event[‘check’][‘history’].map { |h| STATUSES[h.to_i] }.join(’ => ‘)}
Node Name: #{@event[‘client’][‘name’]}
Node IP Address: #{@event[‘client’][‘address’]}
Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}
Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

=== Check Output ===

#{@event[‘check’][‘output’]}
)
Pony.options = mail_options

# #YELLOW
unless settings['ponymailer']['recipients'].empty? # rubocop:disable GuardClause
  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' + @event['action'] + ' an incident -- ' + short_name
    end
  end
end

end
end

/etc/sensu/conf.d/ponymailer.json
{
“ponymailer”: {
“authenticate”:true,
“only_send_on_change”:false,
“username”:"xxx@gmail.com",
“tls”:true,
“port”:“587”,
“fromname”:“Operations”,
“hostname”:“smtp.gmail.com”,
“password”:“xxxx”,
“from”:"ops@xxxx.net",
“recipients”:[
"xxxx@gmail.com"
]
}
}

/etc/sensu/conf.d/handler_ponymailer.json
{
“handlers”: {
“ponymailer”: {
“type”: “pipe”,
“command”: “/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb”
}
}
}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:“received result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:“processing result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:“storing check result”,“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}
{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:“processing event”,“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}
{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:“handling event”,“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}
{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences: pdx01.infra.domain.net/memory\n”]}


#2

Can you find events in the log where there was output from the
ponymailer command? Or any handler?
The one posted was filtered.

Try grepping for "handler output" and then grep out "only handling
every" to see the relevant log lines.

···

On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai <rtsai@rivermeadow.com> wrote:

HI I can't get either ponymailer nor mailer to send an email out. I have
sendmail installed and was able to send an email with mail -s. Below is
what I have confgured for ponymailer. Any guideline is welcome.
Thanks,
Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)
amqp (1.5.0)
async_sinatra (1.2.0)
bigdecimal (1.2.0)
childprocess (0.5.3)
daemons (1.2.2)
em-redis-unified (1.0.0)
em-worker (0.0.2)
eventmachine (1.0.3)
ffi (1.9.8)
io-console (0.4.2)
ipaddress (0.8.0)
json (1.7.7)
mail (2.5.4)
mime-types (2.6.1, 1.25.1)
minitest (4.3.2)
mixlib-cli (1.5.0)
mixlib-config (2.2.1)
mixlib-log (1.6.0)
mixlib-shellout (2.1.0)
multi_json (1.11.0)
ohai (6.16.0)
polyglot (0.3.5)
pony (1.11)
psych (2.0.0)
rack (1.6.1)
rack-protection (1.5.3)
rake (0.9.6)
rdoc (4.0.0)
sensu (0.19.2)
sensu-em (2.5.2)
sensu-extension (1.1.2)
sensu-extensions (1.2.0)
sensu-logger (1.0.0)
sensu-plugin (1.1.0)
sensu-settings (1.9.0)
sensu-spawn (1.2.0)
sensu-transport (3.0.0)
sinatra (1.4.6)
systemu (2.6.5, 2.5.2)
test-unit (2.0.0.0)
thin (1.6.3)
tilt (2.0.1)
treetop (1.4.15)
uuidtools (2.1.5)
yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb
#!/usr/bin/env ruby
#
# Sensu Handler: ponymailer
#
# This handler formats alerts as mails and sends them off to a pre-defined
recipient.
#
# Copyright 2012 Panagiotis Papadomitsos <pj@ezgr.net>
#
# Released under the same terms as Sensu (the MIT license); see LICENSE
# for details.

require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'sensu-handler'
require 'timeout'
require 'pony'
require 'date'

class PonyMailer < Sensu::Handler
  STATUSES = {
    0 => 'OK',
    1 => 'WARNING',
    2 => 'CRITICAL'
  }

  def short_name
    @event['client']['name'] + '/' + @event['check']['name']
  end

  def action_to_string
    @event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'
  end

  def handle
    if settings['ponymailer']['only_send_on_change']
      last_status = @event['check']['history'][-2].to_i
      if last_status == @event['check']['status']
        return
      end
    end

    mail_options = {
      subject: "Sensu Monitoring Alert: #{action_to_string} ::
#{short_name}",
      from: "#{settings['ponymailer']['fromname']}
<#{settings['ponymailer']['from']}>",
      via: :smtp,
      via_options: {
        address: settings['ponymailer']['hostname'],
        port: settings['ponymailer']['port'],
        enable_starttls_auto: settings['ponymailer']['tls']
      },
      charset: 'utf-8',
      sender: settings['ponymailer']['from']
    }
    mail_options.merge!(via_options: {
                          address: settings['ponymailer']['hostname'],
                          port: settings['ponymailer']['port'],
                          enable_starttls_auto:
settings['ponymailer']['tls'],
                          user_name: settings['ponymailer']['username'],
                          password: settings['ponymailer']['password'],
                          authentication: :plain
                        }) if settings['ponymailer']['authenticate']

    mail_options[:body] = %(Sensu has detected a failed check. Event
analysis follows:
Event Timestamp: #{Time.at(@event['check']['issued'].to_i)}
Check That Failed: #{@event['check']['name']}
Check Command: #{@event['check']['command']}
Check Flapping: #{@event['check']['flapping']}
Check Occurrences: #{@event['occurrences']}
Check History: #{@event['check']['history'].map { |h| STATUSES[h.to_i]
}.join(' => ')}
Node Name: #{@event['client']['name']}
Node IP Address: #{@event['client']['address']}
Node LPOL: #{Time.at(@event['client']['timestamp'].to_i)}
Node Subscriptions: #{@event['client']['subscriptions'].join(', ')}

=== Check Output ===

#{@event['check']['output']}
)
    Pony.options = mail_options

    # #YELLOW
    unless settings['ponymailer']['recipients'].empty? # rubocop:disable
GuardClause
      settings['ponymailer']['recipients'].each do |to|
        begin
          Timeout.timeout 10 do
            Pony.mail(to: to)
            puts 'mail -- sent alert for ' + short_name + ' to ' + to
          end
      rescue Timeout::Error
        puts 'mail -- timed out while attempting to ' + @event['action'] + '
an incident -- ' + short_name
        end
      end
    end
  end
end

/etc/sensu/conf.d/ponymailer.json
{
    "ponymailer": {
        "authenticate":true,
        "only_send_on_change":false,
        "username":"xxx@gmail.com",
        "tls":true,
        "port":"587",
        "fromname":"Operations",
        "hostname":"smtp.gmail.com",
        "password":"xxxx",
        "from":"ops@xxxx.net",
        "recipients":[
            "xxxx@gmail.com"
        ]
    }
}

/etc/sensu/conf.d/handler_ponymailer.json
{
    "handlers": {
        "ponymailer": {
        "type": "pipe",
        "command": "/opt/sensu/embedded/bin/ruby
/etc/sensu/handlers/ponymailer.rb"
        }
    }
}

{"timestamp":"2015-06-20T04:57:00.233492+0000","level":"debug","message":"received
result","result":{"client":"pdx01.infra.domain.net","check":{"name":"memory","issued":1434776220,"command":"/etc/sensu/plugins/check-mem.sh
-w 303800 -c
303800","handlers":["ponymailer"],"interval":10,"subscribers":["common"],"executed":1434776220,"duration":0.006,"output":"MEM
CRITICAL - free system memory: 672 MB\n","status":2}}}
{"timestamp":"2015-06-20T04:57:00.233610+0000","level":"debug","message":"processing
result","result":{"client":"pdx01.infra.domain.net","check":{"name":"memory","issued":1434776220,"command":"/etc/sensu/plugins/check-mem.sh
-w 303800 -c
303800","handlers":["ponymailer"],"interval":10,"subscribers":["common"],"executed":1434776220,"duration":0.006,"output":"MEM
CRITICAL - free system memory: 672 MB\n","status":2}}}
{"timestamp":"2015-06-20T04:57:00.234173+0000","level":"debug","message":"storing
check
result","check":{"handlers":["ponymailer"],"command":"/etc/sensu/plugins/check-mem.sh
-w 303800 -c
303800","interval":10,"subscribers":["common"],"name":"memory","issued":1434776220,"executed":1434776220,"duration":0.006,"output":"MEM
CRITICAL - free system memory: 672 MB\n","status":2}}
{"timestamp":"2015-06-20T04:57:00.235571+0000","level":"info","message":"processing
event","event":{"id":"b67033bb-9a78-4fd6-a98b-3a525f7bf552","client":{"name":"pdx01.infra.domain.net","address":"localhost","subscriptions":["common"],"version":"0.19.2","timestamp":1434776206},"check":{"handlers":["ponymailer"],"command":"/etc/sensu/plugins/check-mem.sh
-w 303800 -c
303800","interval":10,"subscribers":["common"],"name":"memory","issued":1434776220,"executed":1434776220,"duration":0.006,"output":"MEM
CRITICAL - free system memory: 672
MB\n","status":2,"history":["2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2"],"total_state_change":0},"occurrences":723,"action":"create"}}
{"timestamp":"2015-06-20T04:57:00.235927+0000","level":"debug","message":"handling
event","event_data":"{\"id\":\"b67033bb-9a78-4fd6-a98b-3a525f7bf552\",\"client\":{\"name\":\"pdx01.infra.domain.net\",\"address\":\"localhost\",\"subscriptions\":[\"common\"],\"version\":\"0.19.2\",\"timestamp\":1434776206},\"check\":{\"handlers\":[\"ponymailer\"],\"command\":\"/etc/sensu/plugins/check-mem.sh
-w 303800 -c
303800\",\"interval\":10,\"subscribers\":[\"common\"],\"name\":\"memory\",\"issued\":1434776220,\"executed\":1434776220,\"duration\":0.006,\"output\":\"MEM
CRITICAL - free system memory: 672
MB\\n\",\"status\":2,\"history\":[\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\"],\"total_state_change\":0},\"occurrences\":723,\"action\":\"create\"}","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
/etc/sensu/handlers/ponymailer.rb","name":"ponymailer"}}
{"timestamp":"2015-06-20T04:57:00.805423+0000","level":"info","message":"handler
output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
/etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}


#3

Hey Kyle,

Thanks for helping out! Below is the grepped output but I’m only seeing info messages for the handler messages. In my /etc/default/sensu, I have LOG_LEVEL=debug. I have restarted the sensu-server service. Is there another place I need to change the log level? Thanks again!!

-Rob

/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences: pdx01.infra.domain.net/memory\n”]}

{“timestamp”:“2015-06-20T20:11:30.814117+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences:pdx01.infra.domain.net/memory\n”]}

{“timestamp”:“2015-06-20T20:11:40.807064+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences: pdx01.infra.domain.net/memory\n”]}

{“timestamp”:“2015-06-20T20:14:30.838181+0000”,“level”:“info”,“message”:“handler output”,“handler”:

···

On Sat, Jun 20, 2015 at 10:53 AM, Kyle Anderson kyle@xkyle.com wrote:

Can you find events in the log where there was output from the

ponymailer command? Or any handler?

The one posted was filtered.

Try grepping for “handler output” and then grep out "only handling

every" to see the relevant log lines.

On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai rtsai@rivermeadow.com wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I have

sendmail installed and was able to send an email with mail -s. Below is

what I have confgured for ponymailer. Any guideline is welcome.

Thanks,

Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)

amqp (1.5.0)

async_sinatra (1.2.0)

bigdecimal (1.2.0)

childprocess (0.5.3)

daemons (1.2.2)

em-redis-unified (1.0.0)

em-worker (0.0.2)

eventmachine (1.0.3)

ffi (1.9.8)

io-console (0.4.2)

ipaddress (0.8.0)

json (1.7.7)

mail (2.5.4)

mime-types (2.6.1, 1.25.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

mixlib-config (2.2.1)

mixlib-log (1.6.0)

mixlib-shellout (2.1.0)

multi_json (1.11.0)

ohai (6.16.0)

polyglot (0.3.5)

pony (1.11)

psych (2.0.0)

rack (1.6.1)

rack-protection (1.5.3)

rake (0.9.6)

rdoc (4.0.0)

sensu (0.19.2)

sensu-em (2.5.2)

sensu-extension (1.1.2)

sensu-extensions (1.2.0)

sensu-logger (1.0.0)

sensu-plugin (1.1.0)

sensu-settings (1.9.0)

sensu-spawn (1.2.0)

sensu-transport (3.0.0)

sinatra (1.4.6)

systemu (2.6.5, 2.5.2)

test-unit (2.0.0.0)

thin (1.6.3)

tilt (2.0.1)

treetop (1.4.15)

uuidtools (2.1.5)

yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb

#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a pre-defined

recipient.

Copyright 2012 Panagiotis Papadomitsos pj@ezgr.net

Released under the same terms as Sensu (the MIT license); see LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’

require ‘sensu-handler’

require ‘timeout’

require ‘pony’

require ‘date’

class PonyMailer < Sensu::Handler

STATUSES = {

0 => 'OK',
1 => 'WARNING',
2 => 'CRITICAL'

}

def short_name

@event['client']['name'] + '/' + @event['check']['name']

end

def action_to_string

@event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'

end

def handle

if settings['ponymailer']['only_send_on_change']
  last_status = @event['check']['history'][-2].to_i
  if last_status == @event['check']['status']
    return
  end
end
mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} ::

#{short_name}",

  from: "#{settings['ponymailer']['fromname']}

<#{settings[‘ponymailer’][‘from’]}>",

  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address: settings['ponymailer']['hostname'],
                      port: settings['ponymailer']['port'],
                      enable_starttls_auto:

settings[‘ponymailer’][‘tls’],

                      user_name: settings['ponymailer']['username'],
                      password: settings['ponymailer']['password'],
                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']
mail_options[:body] = %(Sensu has detected a failed check. Event

analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}

Check That Failed: #{@event[‘check’][‘name’]}

Check Command: #{@event[‘check’][‘command’]}

Check Flapping: #{@event[‘check’][‘flapping’]}

Check Occurrences: #{@event[‘occurrences’]}

Check History: #{@event[‘check’][‘history’].map { |h| STATUSES[h.to_i]

}.join(’ => ')}

Node Name: #{@event[‘client’][‘name’]}

Node IP Address: #{@event[‘client’][‘address’]}

Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}

Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

====================

=== Check Output ===

====================

#{@event[‘check’][‘output’]}

)

Pony.options = mail_options
# #YELLOW
unless settings['ponymailer']['recipients'].empty? # rubocop:disable

GuardClause

  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' + @event['action'] + '

an incident – ’ + short_name

    end
  end
end

end

end

/etc/sensu/conf.d/ponymailer.json

{

"ponymailer": {
    "authenticate":true,
    "only_send_on_change":false,
    "username":"xxx@gmail.com",
    "tls":true,
    "port":"587",
    "fromname":"Operations",
    "hostname":"[smtp.gmail.com](http://smtp.gmail.com)",
    "password":"xxxx",
    "from":"ops@xxxx.net",
    "recipients":[
        "xxxx@gmail.com"
    ]
}

}

/etc/sensu/conf.d/handler_ponymailer.json

{

"handlers": {
    "ponymailer": {
    "type": "pipe",
    "command": "/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb"

    }
}

}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:"received

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:"processing

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:"storing

check

result",“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}

{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:"processing

event",“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}

{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:"handling

event",“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}

{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

Robert Tsai

RiverMeadow Operations
(650) 452 - 2020


#4

If you want to debug a handler not working, you have to get it "in the act"
You could decrease the occurences setting to "1" so it fires all the time.
Or maybe grep the logs harder? Are there logs that were logrotated + gzipd?

zgrep 'handler output' /var/log/sensu-server*

···

On Sat, Jun 20, 2015 at 1:26 PM, Robert Tsai <rtsai@rivermeadow.com> wrote:

Hey Kyle,

Thanks for helping out! Below is the grepped output but I'm only seeing
info messages for the handler messages. In my /etc/default/sensu, I have
LOG_LEVEL=debug. I have restarted the sensu-server service. Is there
another place I need to change the log level? Thanks again!!

-Rob

/ponymailer.rb","name":"ponymailer"},"output":["only handling every 180
occurrences: pdx01.infra.domain.net/memory\n"]}
{"timestamp":"2015-06-20T20:11:30.814117+0000","level":"info","message":"handler
output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
/etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
handling every 180 occurrences:pdx01.infra.domain.net/memory\n"]}
{"timestamp":"2015-06-20T20:11:40.807064+0000","level":"info","message":"handler
output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
/etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}
{"timestamp":"2015-06-20T20:14:30.838181+0000","level":"info","message":"handler
output","handler":

On Sat, Jun 20, 2015 at 10:53 AM, Kyle Anderson <kyle@xkyle.com> wrote:

Can you find events in the log where there was output from the
ponymailer command? Or any handler?
The one posted was filtered.

Try grepping for "handler output" and then grep out "only handling
every" to see the relevant log lines.

On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai <rtsai@rivermeadow.com> >> wrote:
> HI I can't get either ponymailer nor mailer to send an email out. I
> have
> sendmail installed and was able to send an email with mail -s. Below is
> what I have confgured for ponymailer. Any guideline is welcome.
> Thanks,
> Robert
>
> /opt/sensu/embedded/bin/gem list
>
> *** LOCAL GEMS ***
>
> amq-protocol (1.9.2)
> amqp (1.5.0)
> async_sinatra (1.2.0)
> bigdecimal (1.2.0)
> childprocess (0.5.3)
> daemons (1.2.2)
> em-redis-unified (1.0.0)
> em-worker (0.0.2)
> eventmachine (1.0.3)
> ffi (1.9.8)
> io-console (0.4.2)
> ipaddress (0.8.0)
> json (1.7.7)
> mail (2.5.4)
> mime-types (2.6.1, 1.25.1)
> minitest (4.3.2)
> mixlib-cli (1.5.0)
> mixlib-config (2.2.1)
> mixlib-log (1.6.0)
> mixlib-shellout (2.1.0)
> multi_json (1.11.0)
> ohai (6.16.0)
> polyglot (0.3.5)
> pony (1.11)
> psych (2.0.0)
> rack (1.6.1)
> rack-protection (1.5.3)
> rake (0.9.6)
> rdoc (4.0.0)
> sensu (0.19.2)
> sensu-em (2.5.2)
> sensu-extension (1.1.2)
> sensu-extensions (1.2.0)
> sensu-logger (1.0.0)
> sensu-plugin (1.1.0)
> sensu-settings (1.9.0)
> sensu-spawn (1.2.0)
> sensu-transport (3.0.0)
> sinatra (1.4.6)
> systemu (2.6.5, 2.5.2)
> test-unit (2.0.0.0)
> thin (1.6.3)
> tilt (2.0.1)
> treetop (1.4.15)
> uuidtools (2.1.5)
> yajl-ruby (1.2.1)
>
>
> cat /etc/sensu/handlers/ponymailer.rb
> #!/usr/bin/env ruby
> #
> # Sensu Handler: ponymailer
> #
> # This handler formats alerts as mails and sends them off to a
> pre-defined
> recipient.
> #
> # Copyright 2012 Panagiotis Papadomitsos <pj@ezgr.net>
> #
> # Released under the same terms as Sensu (the MIT license); see LICENSE
> # for details.
>
> require 'rubygems' if RUBY_VERSION < '1.9.0'
> require 'sensu-handler'
> require 'timeout'
> require 'pony'
> require 'date'
>
> class PonyMailer < Sensu::Handler
> STATUSES = {
> 0 => 'OK',
> 1 => 'WARNING',
> 2 => 'CRITICAL'
> }
>
> def short_name
> @event['client']['name'] + '/' + @event['check']['name']
> end
>
> def action_to_string
> @event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'
> end
>
> def handle
> if settings['ponymailer']['only_send_on_change']
> last_status = @event['check']['history'][-2].to_i
> if last_status == @event['check']['status']
> return
> end
> end
>
> mail_options = {
> subject: "Sensu Monitoring Alert: #{action_to_string} ::
> #{short_name}",
> from: "#{settings['ponymailer']['fromname']}
> <#{settings['ponymailer']['from']}>",
> via: :smtp,
> via_options: {
> address: settings['ponymailer']['hostname'],
> port: settings['ponymailer']['port'],
> enable_starttls_auto: settings['ponymailer']['tls']
> },
> charset: 'utf-8',
> sender: settings['ponymailer']['from']
> }
> mail_options.merge!(via_options: {
> address: settings['ponymailer']['hostname'],
> port: settings['ponymailer']['port'],
> enable_starttls_auto:
> settings['ponymailer']['tls'],
> user_name: settings['ponymailer']['username'],
> password: settings['ponymailer']['password'],
> authentication: :plain
> }) if settings['ponymailer']['authenticate']
>
> mail_options[:body] = %(Sensu has detected a failed check. Event
> analysis follows:
> Event Timestamp: #{Time.at(@event['check']['issued'].to_i)}
> Check That Failed: #{@event['check']['name']}
> Check Command: #{@event['check']['command']}
> Check Flapping: #{@event['check']['flapping']}
> Check Occurrences: #{@event['occurrences']}
> Check History: #{@event['check']['history'].map { |h|
> STATUSES[h.to_i]
> }.join(' => ')}
> Node Name: #{@event['client']['name']}
> Node IP Address: #{@event['client']['address']}
> Node LPOL: #{Time.at(@event['client']['timestamp'].to_i)}
> Node Subscriptions: #{@event['client']['subscriptions'].join(', ')}
> ====================
> === Check Output ===
> ====================
> #{@event['check']['output']}
> )
> Pony.options = mail_options
>
> # #YELLOW
> unless settings['ponymailer']['recipients'].empty? # rubocop:disable
> GuardClause
> settings['ponymailer']['recipients'].each do |to|
> begin
> Timeout.timeout 10 do
> Pony.mail(to: to)
> puts 'mail -- sent alert for ' + short_name + ' to ' + to
> end
> rescue Timeout::Error
> puts 'mail -- timed out while attempting to ' + @event['action']
> + '
> an incident -- ' + short_name
> end
> end
> end
> end
> end
>
>
>
> /etc/sensu/conf.d/ponymailer.json
> {
> "ponymailer": {
> "authenticate":true,
> "only_send_on_change":false,
> "username":"xxx@gmail.com",
> "tls":true,
> "port":"587",
> "fromname":"Operations",
> "hostname":"smtp.gmail.com",
> "password":"xxxx",
> "from":"ops@xxxx.net",
> "recipients":[
> "xxxx@gmail.com"
> ]
> }
> }
>
>
> /etc/sensu/conf.d/handler_ponymailer.json
> {
> "handlers": {
> "ponymailer": {
> "type": "pipe",
> "command": "/opt/sensu/embedded/bin/ruby
> /etc/sensu/handlers/ponymailer.rb"
> }
> }
> }
>
>
>
>
>
> {"timestamp":"2015-06-20T04:57:00.233492+0000","level":"debug","message":"received
>
> result","result":{"client":"pdx01.infra.domain.net","check":{"name":"memory","issued":1434776220,"command":"/etc/sensu/plugins/check-mem.sh
> -w 303800 -c
>
> 303800","handlers":["ponymailer"],"interval":10,"subscribers":["common"],"executed":1434776220,"duration":0.006,"output":"MEM
> CRITICAL - free system memory: 672 MB\n","status":2}}}
>
> {"timestamp":"2015-06-20T04:57:00.233610+0000","level":"debug","message":"processing
>
> result","result":{"client":"pdx01.infra.domain.net","check":{"name":"memory","issued":1434776220,"command":"/etc/sensu/plugins/check-mem.sh
> -w 303800 -c
>
> 303800","handlers":["ponymailer"],"interval":10,"subscribers":["common"],"executed":1434776220,"duration":0.006,"output":"MEM
> CRITICAL - free system memory: 672 MB\n","status":2}}}
>
> {"timestamp":"2015-06-20T04:57:00.234173+0000","level":"debug","message":"storing
> check
>
> result","check":{"handlers":["ponymailer"],"command":"/etc/sensu/plugins/check-mem.sh
> -w 303800 -c
>
> 303800","interval":10,"subscribers":["common"],"name":"memory","issued":1434776220,"executed":1434776220,"duration":0.006,"output":"MEM
> CRITICAL - free system memory: 672 MB\n","status":2}}
>
> {"timestamp":"2015-06-20T04:57:00.235571+0000","level":"info","message":"processing
>
> event","event":{"id":"b67033bb-9a78-4fd6-a98b-3a525f7bf552","client":{"name":"pdx01.infra.domain.net","address":"localhost","subscriptions":["common"],"version":"0.19.2","timestamp":1434776206},"check":{"handlers":["ponymailer"],"command":"/etc/sensu/plugins/check-mem.sh
> -w 303800 -c
>
> 303800","interval":10,"subscribers":["common"],"name":"memory","issued":1434776220,"executed":1434776220,"duration":0.006,"output":"MEM
> CRITICAL - free system memory: 672
>
> MB\n","status":2,"history":["2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2"],"total_state_change":0},"occurrences":723,"action":"create"}}
>
> {"timestamp":"2015-06-20T04:57:00.235927+0000","level":"debug","message":"handling
>
> event","event_data":"{\"id\":\"b67033bb-9a78-4fd6-a98b-3a525f7bf552\",\"client\":{\"name\":\"pdx01.infra.domain.net\",\"address\":\"localhost\",\"subscriptions\":[\"common\"],\"version\":\"0.19.2\",\"timestamp\":1434776206},\"check\":{\"handlers\":[\"ponymailer\"],\"command\":\"/etc/sensu/plugins/check-mem.sh
> -w 303800 -c
>
> 303800\",\"interval\":10,\"subscribers\":[\"common\"],\"name\":\"memory\",\"issued\":1434776220,\"executed\":1434776220,\"duration\":0.006,\"output\":\"MEM
> CRITICAL - free system memory: 672
>
> MB\\n\",\"status\":2,\"history\":[\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\"],\"total_state_change\":0},\"occurrences\":723,\"action\":\"create\"}","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
> /etc/sensu/handlers/ponymailer.rb","name":"ponymailer"}}
>
> {"timestamp":"2015-06-20T04:57:00.805423+0000","level":"info","message":"handler
> output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
> /etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
> handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

--
Robert Tsai
RiverMeadow Operations
(650) 452 - 2020


#5

Hi Kyle,

Here is what I found after I changed the occurances to 1.

{“timestamp”:“2015-06-21T05:23:10.275236+0000”,“level”:“debug”,“message”:“handling event”,“event_data”:"{“id”:“e455f544-7a70-450d-a9e6-57000e99890c”,“client”:{“name”:“infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434864189},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 200 -c 395064",“interval”:10,“occurrences”:1,“refresh”:20,“subscribers”:[“common”],“name”:“memory”,“issued”:1434864190,“executed”:1434864190,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 632 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:200,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/etc/sensu/handlers/ponymailer.rb ",“name”:“ponymailer”}}

{“timestamp”:“2015-06-21T05:17:50.847345+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/etc/sensu/handlers/ponymailer.rb “,“name”:“ponymailer”},“output”:[”/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in addr_port': undefined method+’ for nil:NilClass (NoMethodError)\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in begin_transport'\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:intransport_request’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in request'\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:inblock in request’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in start'\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:inrequest’\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in api_request'\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:instash_exists?’\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in block (2 levels) in filter_silenced'\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:intimeout’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in timeout'\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:inblock in filter_silenced’\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in each'\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:infilter_silenced’\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in filter'\n","\tfrom /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:inblock in class:Handler’\n"]}

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

bigdecimal (1.2.0)

io-console (0.4.2)

json (1.7.7)

mail (2.6.3)

mime-types (2.6.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

pony (1.11)

psych (2.0.0)

rake (0.9.6)

rdoc (4.0.0)

sendmail (0.0.1)

sensu-plugin (1.1.0)

test-unit (2.0.0.0)

timeout (0.0.1)

timeout-extensions (0.0.0)

cat /etc/sensu/conf.d/handler_ponymailer.json

{

“handlers”: {

“ponymailer”: {

“type”: “pipe”,

“command”: "/etc/sensu/handlers/ponymailer.rb "

}

}

}

cat /etc/sensu/conf.d/ponymailer.json

{

"ponymailer": {

	"authenticate":true,

	"only_send_on_change":false,

	"username":"user@gmail.com",

	"tls":true,

	"port":"587",

	"fromname":"Operations",

	"hostname":"[smtp.gmail.com](http://smtp.gmail.com)",

	"password":"xxxxx",

	"from":"ops@xxxx",

	"recipients":[

		"user@gmail.com"

	]

}

}

···

On Sat, Jun 20, 2015 at 2:46 PM, Kyle Anderson kyle@xkyle.com wrote:

If you want to debug a handler not working, you have to get it “in the act”

You could decrease the occurences setting to “1” so it fires all the time.

Or maybe grep the logs harder? Are there logs that were logrotated + gzipd?

zgrep ‘handler output’ /var/log/sensu-server*

On Sat, Jun 20, 2015 at 1:26 PM, Robert Tsai rtsai@rivermeadow.com wrote:

Hey Kyle,

Thanks for helping out! Below is the grepped output but I’m only seeing

info messages for the handler messages. In my /etc/default/sensu, I have

LOG_LEVEL=debug. I have restarted the sensu-server service. Is there

another place I need to change the log level? Thanks again!!

-Rob

/ponymailer.rb",“name”:“ponymailer”},“output”:["only handling every 180

occurrences: pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:11:30.814117+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences:pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:11:40.807064+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:14:30.838181+0000”,“level”:“info”,“message”:"handler

output",“handler”:

On Sat, Jun 20, 2015 at 10:53 AM, Kyle Anderson kyle@xkyle.com wrote:

Can you find events in the log where there was output from the

ponymailer command? Or any handler?

The one posted was filtered.

Try grepping for “handler output” and then grep out "only handling

every" to see the relevant log lines.

On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai rtsai@rivermeadow.com

wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I

have

sendmail installed and was able to send an email with mail -s. Below is

what I have confgured for ponymailer. Any guideline is welcome.

Thanks,

Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)

amqp (1.5.0)

async_sinatra (1.2.0)

bigdecimal (1.2.0)

childprocess (0.5.3)

daemons (1.2.2)

em-redis-unified (1.0.0)

em-worker (0.0.2)

eventmachine (1.0.3)

ffi (1.9.8)

io-console (0.4.2)

ipaddress (0.8.0)

json (1.7.7)

mail (2.5.4)

mime-types (2.6.1, 1.25.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

mixlib-config (2.2.1)

mixlib-log (1.6.0)

mixlib-shellout (2.1.0)

multi_json (1.11.0)

ohai (6.16.0)

polyglot (0.3.5)

pony (1.11)

psych (2.0.0)

rack (1.6.1)

rack-protection (1.5.3)

rake (0.9.6)

rdoc (4.0.0)

sensu (0.19.2)

sensu-em (2.5.2)

sensu-extension (1.1.2)

sensu-extensions (1.2.0)

sensu-logger (1.0.0)

sensu-plugin (1.1.0)

sensu-settings (1.9.0)

sensu-spawn (1.2.0)

sensu-transport (3.0.0)

sinatra (1.4.6)

systemu (2.6.5, 2.5.2)

test-unit (2.0.0.0)

thin (1.6.3)

tilt (2.0.1)

treetop (1.4.15)

uuidtools (2.1.5)

yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb

#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a

pre-defined

recipient.

Copyright 2012 Panagiotis Papadomitsos pj@ezgr.net

Released under the same terms as Sensu (the MIT license); see LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’

require ‘sensu-handler’

require ‘timeout’

require ‘pony’

require ‘date’

class PonyMailer < Sensu::Handler

STATUSES = {

0 => 'OK',
1 => 'WARNING',
2 => 'CRITICAL'

}

def short_name

@event['client']['name'] + '/' + @event['check']['name']

end

def action_to_string

@event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'

end

def handle

if settings['ponymailer']['only_send_on_change']
  last_status = @event['check']['history'][-2].to_i
  if last_status == @event['check']['status']
    return
  end
end
mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} ::

#{short_name}",

  from: "#{settings['ponymailer']['fromname']}

<#{settings[‘ponymailer’][‘from’]}>",

  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address: settings['ponymailer']['hostname'],
                      port: settings['ponymailer']['port'],
                      enable_starttls_auto:

settings[‘ponymailer’][‘tls’],

                      user_name: settings['ponymailer']['username'],
                      password: settings['ponymailer']['password'],
                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']
mail_options[:body] = %(Sensu has detected a failed check. Event

analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}

Check That Failed: #{@event[‘check’][‘name’]}

Check Command: #{@event[‘check’][‘command’]}

Check Flapping: #{@event[‘check’][‘flapping’]}

Check Occurrences: #{@event[‘occurrences’]}

Check History: #{@event[‘check’][‘history’].map { |h|

STATUSES[h.to_i]

}.join(’ => ')}

Node Name: #{@event[‘client’][‘name’]}

Node IP Address: #{@event[‘client’][‘address’]}

Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}

Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

====================

=== Check Output ===

====================

#{@event[‘check’][‘output’]}

)

Pony.options = mail_options
# #YELLOW
unless settings['ponymailer']['recipients'].empty? # rubocop:disable

GuardClause

  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' + @event['action']

an incident – ’ + short_name

    end
  end
end

end

end

/etc/sensu/conf.d/ponymailer.json

{

"ponymailer": {
    "authenticate":true,
    "only_send_on_change":false,
    "username":"xxx@gmail.com",
    "tls":true,
    "port":"587",
    "fromname":"Operations",
    "hostname":"[smtp.gmail.com](http://smtp.gmail.com)",
    "password":"xxxx",
    "from":"ops@xxxx.net",
    "recipients":[
        "xxxx@gmail.com"
    ]
}

}

/etc/sensu/conf.d/handler_ponymailer.json

{

"handlers": {
    "ponymailer": {
    "type": "pipe",
    "command": "/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb"

    }
}

}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:"received

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:"processing

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:"storing

check

result",“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}

{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:"processing

event",“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}

{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:"handling

event",“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}

{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

Robert Tsai

RiverMeadow Operations

(650) 452 - 2020

Robert Tsai

RiverMeadow Operations
(650) 452 - 2020


#6

Great, there is the smoking gun more or less. Reprinting for readability:

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in `addr_port':
undefined method `+' for nil:NilClass (NoMethodError)
        from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in
`begin_transport'
        from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:in
`transport_request'
        from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in `request'
        from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:in
`block in request'
        from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in `start'
        from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:in `request'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in
`api_request'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:in
`stash_exists?'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in
`block (2 levels) in filter_silenced'
        from /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:in `timeout'
        from /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in `timeout'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:in
`block in filter_silenced'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in
`each'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in
`filter_silenced'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in
`filter'
        from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:in
`block in <class:Handler>'

Makes me think that it isn't pony-mailer related at all.
Is the "port" setting missing from your general sensu api settings
configuration? (https://sensuapp.org/docs/0.19/install-sensu#configure-connections)
Sensu handlers use the sensu-api to tell if things are silenced or
not, and that error kinda hints that the port is empty.

···

On Sat, Jun 20, 2015 at 10:25 PM, Robert Tsai <rtsai@rivermeadow.com> wrote:

Hi Kyle,

Here is what I found after I changed the occurances to 1.

{"timestamp":"2015-06-21T05:23:10.275236+0000","level":"debug","message":"handling
event","event_data":"{\"id\":\"e455f544-7a70-450d-a9e6-57000e99890c\",\"client\":{\"name\":\"infra.domain.net\",\"address\":\"localhost\",\"subscriptions\":[\"common\"],\"version\":\"0.19.2\",\"timestamp\":1434864189},\"check\":{\"handlers\":[\"ponymailer\"],\"command\":\"/etc/sensu/plugins/check-mem.sh
-w 200 -c
395064\",\"interval\":10,\"occurrences\":1,\"refresh\":20,\"subscribers\":[\"common\"],\"name\":\"memory\",\"issued\":1434864190,\"executed\":1434864190,\"duration\":0.006,\"output\":\"MEM
CRITICAL - free system memory: 632
MB\\n\",\"status\":2,\"history\":[\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\"],\"total_state_change\":0},\"occurrences\":200,\"action\":\"create\"}","handler":{"type":"pipe","command":"/etc/sensu/handlers/ponymailer.rb
","name":"ponymailer"}}

{"timestamp":"2015-06-21T05:17:50.847345+0000","level":"info","message":"handler
output","handler":{"type":"pipe","command":"/etc/sensu/handlers/ponymailer.rb
","name":"ponymailer"},"output":["/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in
`addr_port': undefined method `+' for nil:NilClass
(NoMethodError)\n","\tfrom
/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in
`begin_transport'\n","\tfrom
/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:in
`transport_request'\n","\tfrom
/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in `request'\n","\tfrom
/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:in `block in
request'\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in
`start'\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:in
`request'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in
`api_request'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:in
`stash_exists?'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in
`block (2 levels) in filter_silenced'\n","\tfrom
/opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:in `timeout'\n","\tfrom
/opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in `timeout'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:in
`block in filter_silenced'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in
`each'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in
`filter_silenced'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in
`filter'\n","\tfrom
/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:in
`block in <class:Handler>'\n"]}

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

bigdecimal (1.2.0)
io-console (0.4.2)
json (1.7.7)
mail (2.6.3)
mime-types (2.6.1)
minitest (4.3.2)
mixlib-cli (1.5.0)
pony (1.11)
psych (2.0.0)
rake (0.9.6)
rdoc (4.0.0)
sendmail (0.0.1)
sensu-plugin (1.1.0)
test-unit (2.0.0.0)
timeout (0.0.1)
timeout-extensions (0.0.0)

cat /etc/sensu/conf.d/handler_ponymailer.json
{
    "handlers": {
        "ponymailer": {
        "type": "pipe",
        "command": "/etc/sensu/handlers/ponymailer.rb "
        }
    }
}

cat /etc/sensu/conf.d/ponymailer.json
{
"ponymailer": {
"authenticate":true,
"only_send_on_change":false,
"username":"user@gmail.com",
"tls":true,
"port":"587",
"fromname":"Operations",
"hostname":"smtp.gmail.com",
"password":"xxxxx",
"from":"ops@xxxx",
"recipients":[
"user@gmail.com"
]
}
}

On Sat, Jun 20, 2015 at 2:46 PM, Kyle Anderson <kyle@xkyle.com> wrote:

If you want to debug a handler not working, you have to get it "in the
act"
You could decrease the occurences setting to "1" so it fires all the time.
Or maybe grep the logs harder? Are there logs that were logrotated +
gzipd?

zgrep 'handler output' /var/log/sensu-server*

On Sat, Jun 20, 2015 at 1:26 PM, Robert Tsai <rtsai@rivermeadow.com> >> wrote:
> Hey Kyle,
>
> Thanks for helping out! Below is the grepped output but I'm only seeing
> info messages for the handler messages. In my /etc/default/sensu, I
> have
> LOG_LEVEL=debug. I have restarted the sensu-server service. Is there
> another place I need to change the log level? Thanks again!!
>
> -Rob
>
> /ponymailer.rb","name":"ponymailer"},"output":["only handling every 180
> occurrences: pdx01.infra.domain.net/memory\n"]}
>
> {"timestamp":"2015-06-20T20:11:30.814117+0000","level":"info","message":"handler
> output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
> /etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
> handling every 180 occurrences:pdx01.infra.domain.net/memory\n"]}
>
> {"timestamp":"2015-06-20T20:11:40.807064+0000","level":"info","message":"handler
> output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
> /etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
> handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}
>
> {"timestamp":"2015-06-20T20:14:30.838181+0000","level":"info","message":"handler
> output","handler":
>
> On Sat, Jun 20, 2015 at 10:53 AM, Kyle Anderson <kyle@xkyle.com> wrote:
>>
>> Can you find events in the log where there was output from the
>> ponymailer command? Or any handler?
>> The one posted was filtered.
>>
>> Try grepping for "handler output" and then grep out "only handling
>> every" to see the relevant log lines.
>>
>> On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai <rtsai@rivermeadow.com> >> >> wrote:
>> > HI I can't get either ponymailer nor mailer to send an email out. I
>> > have
>> > sendmail installed and was able to send an email with mail -s. Below
>> > is
>> > what I have confgured for ponymailer. Any guideline is welcome.
>> > Thanks,
>> > Robert
>> >
>> > /opt/sensu/embedded/bin/gem list
>> >
>> > *** LOCAL GEMS ***
>> >
>> > amq-protocol (1.9.2)
>> > amqp (1.5.0)
>> > async_sinatra (1.2.0)
>> > bigdecimal (1.2.0)
>> > childprocess (0.5.3)
>> > daemons (1.2.2)
>> > em-redis-unified (1.0.0)
>> > em-worker (0.0.2)
>> > eventmachine (1.0.3)
>> > ffi (1.9.8)
>> > io-console (0.4.2)
>> > ipaddress (0.8.0)
>> > json (1.7.7)
>> > mail (2.5.4)
>> > mime-types (2.6.1, 1.25.1)
>> > minitest (4.3.2)
>> > mixlib-cli (1.5.0)
>> > mixlib-config (2.2.1)
>> > mixlib-log (1.6.0)
>> > mixlib-shellout (2.1.0)
>> > multi_json (1.11.0)
>> > ohai (6.16.0)
>> > polyglot (0.3.5)
>> > pony (1.11)
>> > psych (2.0.0)
>> > rack (1.6.1)
>> > rack-protection (1.5.3)
>> > rake (0.9.6)
>> > rdoc (4.0.0)
>> > sensu (0.19.2)
>> > sensu-em (2.5.2)
>> > sensu-extension (1.1.2)
>> > sensu-extensions (1.2.0)
>> > sensu-logger (1.0.0)
>> > sensu-plugin (1.1.0)
>> > sensu-settings (1.9.0)
>> > sensu-spawn (1.2.0)
>> > sensu-transport (3.0.0)
>> > sinatra (1.4.6)
>> > systemu (2.6.5, 2.5.2)
>> > test-unit (2.0.0.0)
>> > thin (1.6.3)
>> > tilt (2.0.1)
>> > treetop (1.4.15)
>> > uuidtools (2.1.5)
>> > yajl-ruby (1.2.1)
>> >
>> >
>> > cat /etc/sensu/handlers/ponymailer.rb
>> > #!/usr/bin/env ruby
>> > #
>> > # Sensu Handler: ponymailer
>> > #
>> > # This handler formats alerts as mails and sends them off to a
>> > pre-defined
>> > recipient.
>> > #
>> > # Copyright 2012 Panagiotis Papadomitsos <pj@ezgr.net>
>> > #
>> > # Released under the same terms as Sensu (the MIT license); see
>> > LICENSE
>> > # for details.
>> >
>> > require 'rubygems' if RUBY_VERSION < '1.9.0'
>> > require 'sensu-handler'
>> > require 'timeout'
>> > require 'pony'
>> > require 'date'
>> >
>> > class PonyMailer < Sensu::Handler
>> > STATUSES = {
>> > 0 => 'OK',
>> > 1 => 'WARNING',
>> > 2 => 'CRITICAL'
>> > }
>> >
>> > def short_name
>> > @event['client']['name'] + '/' + @event['check']['name']
>> > end
>> >
>> > def action_to_string
>> > @event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'
>> > end
>> >
>> > def handle
>> > if settings['ponymailer']['only_send_on_change']
>> > last_status = @event['check']['history'][-2].to_i
>> > if last_status == @event['check']['status']
>> > return
>> > end
>> > end
>> >
>> > mail_options = {
>> > subject: "Sensu Monitoring Alert: #{action_to_string} ::
>> > #{short_name}",
>> > from: "#{settings['ponymailer']['fromname']}
>> > <#{settings['ponymailer']['from']}>",
>> > via: :smtp,
>> > via_options: {
>> > address: settings['ponymailer']['hostname'],
>> > port: settings['ponymailer']['port'],
>> > enable_starttls_auto: settings['ponymailer']['tls']
>> > },
>> > charset: 'utf-8',
>> > sender: settings['ponymailer']['from']
>> > }
>> > mail_options.merge!(via_options: {
>> > address:
>> > settings['ponymailer']['hostname'],
>> > port: settings['ponymailer']['port'],
>> > enable_starttls_auto:
>> > settings['ponymailer']['tls'],
>> > user_name:
>> > settings['ponymailer']['username'],
>> > password:
>> > settings['ponymailer']['password'],
>> > authentication: :plain
>> > }) if settings['ponymailer']['authenticate']
>> >
>> > mail_options[:body] = %(Sensu has detected a failed check. Event
>> > analysis follows:
>> > Event Timestamp: #{Time.at(@event['check']['issued'].to_i)}
>> > Check That Failed: #{@event['check']['name']}
>> > Check Command: #{@event['check']['command']}
>> > Check Flapping: #{@event['check']['flapping']}
>> > Check Occurrences: #{@event['occurrences']}
>> > Check History: #{@event['check']['history'].map { |h|
>> > STATUSES[h.to_i]
>> > }.join(' => ')}
>> > Node Name: #{@event['client']['name']}
>> > Node IP Address: #{@event['client']['address']}
>> > Node LPOL: #{Time.at(@event['client']['timestamp'].to_i)}
>> > Node Subscriptions: #{@event['client']['subscriptions'].join(', ')}
>> > ====================
>> > === Check Output ===
>> > ====================
>> > #{@event['check']['output']}
>> > )
>> > Pony.options = mail_options
>> >
>> > # #YELLOW
>> > unless settings['ponymailer']['recipients'].empty? #
>> > rubocop:disable
>> > GuardClause
>> > settings['ponymailer']['recipients'].each do |to|
>> > begin
>> > Timeout.timeout 10 do
>> > Pony.mail(to: to)
>> > puts 'mail -- sent alert for ' + short_name + ' to ' + to
>> > end
>> > rescue Timeout::Error
>> > puts 'mail -- timed out while attempting to ' +
>> > @event['action']
>> > + '
>> > an incident -- ' + short_name
>> > end
>> > end
>> > end
>> > end
>> > end
>> >
>> >
>> >
>> > /etc/sensu/conf.d/ponymailer.json
>> > {
>> > "ponymailer": {
>> > "authenticate":true,
>> > "only_send_on_change":false,
>> > "username":"xxx@gmail.com",
>> > "tls":true,
>> > "port":"587",
>> > "fromname":"Operations",
>> > "hostname":"smtp.gmail.com",
>> > "password":"xxxx",
>> > "from":"ops@xxxx.net",
>> > "recipients":[
>> > "xxxx@gmail.com"
>> > ]
>> > }
>> > }
>> >
>> >
>> > /etc/sensu/conf.d/handler_ponymailer.json
>> > {
>> > "handlers": {
>> > "ponymailer": {
>> > "type": "pipe",
>> > "command": "/opt/sensu/embedded/bin/ruby
>> > /etc/sensu/handlers/ponymailer.rb"
>> > }
>> > }
>> > }
>> >
>> >
>> >
>> >
>> >
>> >
>> > {"timestamp":"2015-06-20T04:57:00.233492+0000","level":"debug","message":"received
>> >
>> >
>> > result","result":{"client":"pdx01.infra.domain.net","check":{"name":"memory","issued":1434776220,"command":"/etc/sensu/plugins/check-mem.sh
>> > -w 303800 -c
>> >
>> >
>> > 303800","handlers":["ponymailer"],"interval":10,"subscribers":["common"],"executed":1434776220,"duration":0.006,"output":"MEM
>> > CRITICAL - free system memory: 672 MB\n","status":2}}}
>> >
>> >
>> > {"timestamp":"2015-06-20T04:57:00.233610+0000","level":"debug","message":"processing
>> >
>> >
>> > result","result":{"client":"pdx01.infra.domain.net","check":{"name":"memory","issued":1434776220,"command":"/etc/sensu/plugins/check-mem.sh
>> > -w 303800 -c
>> >
>> >
>> > 303800","handlers":["ponymailer"],"interval":10,"subscribers":["common"],"executed":1434776220,"duration":0.006,"output":"MEM
>> > CRITICAL - free system memory: 672 MB\n","status":2}}}
>> >
>> >
>> > {"timestamp":"2015-06-20T04:57:00.234173+0000","level":"debug","message":"storing
>> > check
>> >
>> >
>> > result","check":{"handlers":["ponymailer"],"command":"/etc/sensu/plugins/check-mem.sh
>> > -w 303800 -c
>> >
>> >
>> > 303800","interval":10,"subscribers":["common"],"name":"memory","issued":1434776220,"executed":1434776220,"duration":0.006,"output":"MEM
>> > CRITICAL - free system memory: 672 MB\n","status":2}}
>> >
>> >
>> > {"timestamp":"2015-06-20T04:57:00.235571+0000","level":"info","message":"processing
>> >
>> >
>> > event","event":{"id":"b67033bb-9a78-4fd6-a98b-3a525f7bf552","client":{"name":"pdx01.infra.domain.net","address":"localhost","subscriptions":["common"],"version":"0.19.2","timestamp":1434776206},"check":{"handlers":["ponymailer"],"command":"/etc/sensu/plugins/check-mem.sh
>> > -w 303800 -c
>> >
>> >
>> > 303800","interval":10,"subscribers":["common"],"name":"memory","issued":1434776220,"executed":1434776220,"duration":0.006,"output":"MEM
>> > CRITICAL - free system memory: 672
>> >
>> >
>> > MB\n","status":2,"history":["2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2"],"total_state_change":0},"occurrences":723,"action":"create"}}
>> >
>> >
>> > {"timestamp":"2015-06-20T04:57:00.235927+0000","level":"debug","message":"handling
>> >
>> >
>> > event","event_data":"{\"id\":\"b67033bb-9a78-4fd6-a98b-3a525f7bf552\",\"client\":{\"name\":\"pdx01.infra.domain.net\",\"address\":\"localhost\",\"subscriptions\":[\"common\"],\"version\":\"0.19.2\",\"timestamp\":1434776206},\"check\":{\"handlers\":[\"ponymailer\"],\"command\":\"/etc/sensu/plugins/check-mem.sh
>> > -w 303800 -c
>> >
>> >
>> > 303800\",\"interval\":10,\"subscribers\":[\"common\"],\"name\":\"memory\",\"issued\":1434776220,\"executed\":1434776220,\"duration\":0.006,\"output\":\"MEM
>> > CRITICAL - free system memory: 672
>> >
>> >
>> > MB\\n\",\"status\":2,\"history\":[\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\",\"2\"],\"total_state_change\":0},\"occurrences\":723,\"action\":\"create\"}","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
>> > /etc/sensu/handlers/ponymailer.rb","name":"ponymailer"}}
>> >
>> >
>> > {"timestamp":"2015-06-20T04:57:00.805423+0000","level":"info","message":"handler
>> >
>> > output","handler":{"type":"pipe","command":"/opt/sensu/embedded/bin/ruby
>> >
>> > /etc/sensu/handlers/ponymailer.rb","name":"ponymailer"},"output":["only
>> > handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}
>
>
>
>
> --
> Robert Tsai
> RiverMeadow Operations
> (650) 452 - 2020
>

--
Robert Tsai
RiverMeadow Operations
(650) 452 - 2020


#7

nope it looks good.

{

“rabbitmq”: {

“host”: “localhost”,

“vhost”: “/sensu”,

“user”: “sensu”,

“password”: “xxxxxxx”

},

“redis”: {

“host”: “localhost”

},

“api”: {

“port”: 4567

}

}

···

On Sat, Jun 20, 2015 at 10:34 PM, Kyle Anderson kyle@xkyle.com wrote:

Great, there is the smoking gun more or less. Reprinting for readability:

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in `addr_port’:

undefined method `+’ for nil:NilClass (NoMethodError)

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in

`begin_transport’

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:in

`transport_request’

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in `request'

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:in

`block in request’

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in `start'

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:in `request'

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in

`api_request’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:in

`stash_exists?’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in

`block (2 levels) in filter_silenced’

    from /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:in `timeout'

    from /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in `timeout'

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:in

`block in filter_silenced’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`each’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`filter_silenced’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in

`filter’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:in

`block in class:Handler

Makes me think that it isn’t pony-mailer related at all.

Is the “port” setting missing from your general sensu api settings

configuration? (https://sensuapp.org/docs/0.19/install-sensu#configure-connections)

Sensu handlers use the sensu-api to tell if things are silenced or

not, and that error kinda hints that the port is empty.

On Sat, Jun 20, 2015 at 10:25 PM, Robert Tsai rtsai@rivermeadow.com wrote:

Hi Kyle,

Here is what I found after I changed the occurances to 1.

{“timestamp”:“2015-06-21T05:23:10.275236+0000”,“level”:“debug”,“message”:"handling

event",“event_data”:"{“id”:“e455f544-7a70-450d-a9e6-57000e99890c”,“client”:{“name”:“infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434864189},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 200 -c

395064",“interval”:10,“occurrences”:1,“refresh”:20,“subscribers”:[“common”],“name”:“memory”,“issued”:1434864190,“executed”:1434864190,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 632

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:200,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/etc/sensu/handlers/ponymailer.rb

",“name”:“ponymailer”}}

{“timestamp”:“2015-06-21T05:17:50.847345+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/etc/sensu/handlers/ponymailer.rb

“,“name”:“ponymailer”},“output”:[”/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in

addr_port': undefined method+’ for nil:NilClass

(NoMethodError)\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in

`begin_transport’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:in

`transport_request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in `request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:in `block in

request’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in

`start’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:in

`request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in

`api_request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:in

`stash_exists?’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in

`block (2 levels) in filter_silenced’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:in `timeout’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in `timeout’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:in

`block in filter_silenced’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`each’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`filter_silenced’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in

`filter’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:in

`block in class:Handler’\n"]}

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

bigdecimal (1.2.0)

io-console (0.4.2)

json (1.7.7)

mail (2.6.3)

mime-types (2.6.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

pony (1.11)

psych (2.0.0)

rake (0.9.6)

rdoc (4.0.0)

sendmail (0.0.1)

sensu-plugin (1.1.0)

test-unit (2.0.0.0)

timeout (0.0.1)

timeout-extensions (0.0.0)

cat /etc/sensu/conf.d/handler_ponymailer.json

{

"handlers": {
    "ponymailer": {
    "type": "pipe",
    "command":  "/etc/sensu/handlers/ponymailer.rb "
    }
}

}

cat /etc/sensu/conf.d/ponymailer.json

{

“ponymailer”: {

“authenticate”:true,

“only_send_on_change”:false,

“username”:"user@gmail.com",

“tls”:true,

“port”:“587”,

“fromname”:“Operations”,

“hostname”:“smtp.gmail.com”,

“password”:“xxxxx”,

“from”:“ops@xxxx”,

“recipients”:[

"user@gmail.com"

]

}

}

On Sat, Jun 20, 2015 at 2:46 PM, Kyle Anderson kyle@xkyle.com wrote:

If you want to debug a handler not working, you have to get it "in the

act"

You could decrease the occurences setting to “1” so it fires all the time.

Or maybe grep the logs harder? Are there logs that were logrotated +

gzipd?

zgrep ‘handler output’ /var/log/sensu-server*

On Sat, Jun 20, 2015 at 1:26 PM, Robert Tsai rtsai@rivermeadow.com

wrote:

Hey Kyle,

Thanks for helping out! Below is the grepped output but I’m only seeing

info messages for the handler messages. In my /etc/default/sensu, I

have

LOG_LEVEL=debug. I have restarted the sensu-server service. Is there

another place I need to change the log level? Thanks again!!

-Rob

/ponymailer.rb",“name”:“ponymailer”},“output”:["only handling every 180

occurrences: pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:11:30.814117+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences:pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:11:40.807064+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:14:30.838181+0000”,“level”:“info”,“message”:"handler

output",“handler”:

On Sat, Jun 20, 2015 at 10:53 AM, Kyle Anderson kyle@xkyle.com wrote:

Can you find events in the log where there was output from the

ponymailer command? Or any handler?

The one posted was filtered.

Try grepping for “handler output” and then grep out "only handling

every" to see the relevant log lines.

On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai rtsai@rivermeadow.com

wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I

have

sendmail installed and was able to send an email with mail -s. Below

is

what I have confgured for ponymailer. Any guideline is welcome.

Thanks,

Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)

amqp (1.5.0)

async_sinatra (1.2.0)

bigdecimal (1.2.0)

childprocess (0.5.3)

daemons (1.2.2)

em-redis-unified (1.0.0)

em-worker (0.0.2)

eventmachine (1.0.3)

ffi (1.9.8)

io-console (0.4.2)

ipaddress (0.8.0)

json (1.7.7)

mail (2.5.4)

mime-types (2.6.1, 1.25.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

mixlib-config (2.2.1)

mixlib-log (1.6.0)

mixlib-shellout (2.1.0)

multi_json (1.11.0)

ohai (6.16.0)

polyglot (0.3.5)

pony (1.11)

psych (2.0.0)

rack (1.6.1)

rack-protection (1.5.3)

rake (0.9.6)

rdoc (4.0.0)

sensu (0.19.2)

sensu-em (2.5.2)

sensu-extension (1.1.2)

sensu-extensions (1.2.0)

sensu-logger (1.0.0)

sensu-plugin (1.1.0)

sensu-settings (1.9.0)

sensu-spawn (1.2.0)

sensu-transport (3.0.0)

sinatra (1.4.6)

systemu (2.6.5, 2.5.2)

test-unit (2.0.0.0)

thin (1.6.3)

tilt (2.0.1)

treetop (1.4.15)

uuidtools (2.1.5)

yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb

#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a

pre-defined

recipient.

Copyright 2012 Panagiotis Papadomitsos pj@ezgr.net

Released under the same terms as Sensu (the MIT license); see

LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’

require ‘sensu-handler’

require ‘timeout’

require ‘pony’

require ‘date’

class PonyMailer < Sensu::Handler

STATUSES = {

0 => 'OK',
1 => 'WARNING',
2 => 'CRITICAL'

}

def short_name

@event['client']['name'] + '/' + @event['check']['name']

end

def action_to_string

@event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'

end

def handle

if settings['ponymailer']['only_send_on_change']
  last_status = @event['check']['history'][-2].to_i
  if last_status == @event['check']['status']
    return
  end
end
mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} ::

#{short_name}",

  from: "#{settings['ponymailer']['fromname']}

<#{settings[‘ponymailer’][‘from’]}>",

  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address:

settings[‘ponymailer’][‘hostname’],

                      port: settings['ponymailer']['port'],
                      enable_starttls_auto:

settings[‘ponymailer’][‘tls’],

                      user_name:

settings[‘ponymailer’][‘username’],

                      password:

settings[‘ponymailer’][‘password’],

                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']
mail_options[:body] = %(Sensu has detected a failed check. Event

analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}

Check That Failed: #{@event[‘check’][‘name’]}

Check Command: #{@event[‘check’][‘command’]}

Check Flapping: #{@event[‘check’][‘flapping’]}

Check Occurrences: #{@event[‘occurrences’]}

Check History: #{@event[‘check’][‘history’].map { |h|

STATUSES[h.to_i]

}.join(’ => ')}

Node Name: #{@event[‘client’][‘name’]}

Node IP Address: #{@event[‘client’][‘address’]}

Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}

Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

====================

=== Check Output ===

====================

#{@event[‘check’][‘output’]}

)

Pony.options = mail_options
# #YELLOW
unless settings['ponymailer']['recipients'].empty? #

rubocop:disable

GuardClause

  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' +

@event[‘action’]

an incident – ’ + short_name

    end
  end
end

end

end

/etc/sensu/conf.d/ponymailer.json

{

"ponymailer": {
    "authenticate":true,
    "only_send_on_change":false,
    "username":"xxx@gmail.com",
    "tls":true,
    "port":"587",
    "fromname":"Operations",
    "hostname":"[smtp.gmail.com](http://smtp.gmail.com)",
    "password":"xxxx",
    "from":"ops@xxxx.net",
    "recipients":[
        "xxxx@gmail.com"
    ]
}

}

/etc/sensu/conf.d/handler_ponymailer.json

{

"handlers": {
    "ponymailer": {
    "type": "pipe",
    "command": "/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb"

    }
}

}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:"received

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:"processing

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:"storing

check

result",“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}

{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:"processing

event",“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}

{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:"handling

event",“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}

{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

Robert Tsai

RiverMeadow Operations

(650) 452 - 2020

Robert Tsai

RiverMeadow Operations

(650) 452 - 2020

Robert Tsai

RiverMeadow Operations
(650) 452 - 2020


#8

I set the port manually in /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb. It worked but know I’m getting smtp auth errors which is easy to fix on my end. I wonder why the port isn’t being picked up but at least I’m close to figuring out. Again, hanks for help, Kyle!

···

On Sat, Jun 20, 2015 at 10:48 PM, Robert Tsai rtsai@rivermeadow.com wrote:

nope it looks good.

{

“rabbitmq”: {

“host”: “localhost”,

“vhost”: “/sensu”,

“user”: “sensu”,

“password”: “xxxxxxx”

},

“redis”: {

“host”: “localhost”

},

“api”: {

“port”: 4567

}

}

On Sat, Jun 20, 2015 at 10:34 PM, Kyle Anderson kyle@xkyle.com wrote:

Great, there is the smoking gun more or less. Reprinting for readability:

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in `addr_port’:

undefined method `+’ for nil:NilClass (NoMethodError)

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in

`begin_transport’

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:in

`transport_request’

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in `request'

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:in

`block in request’

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in `start'

    from /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:in `request'

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in

`api_request’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:in

`stash_exists?’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in

`block (2 levels) in filter_silenced’

    from /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:in `timeout'

    from /opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in `timeout'

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:in

`block in filter_silenced’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`each’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`filter_silenced’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in

`filter’

    from /opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:in

`block in class:Handler

Makes me think that it isn’t pony-mailer related at all.

Is the “port” setting missing from your general sensu api settings

configuration? (https://sensuapp.org/docs/0.19/install-sensu#configure-connections)

Sensu handlers use the sensu-api to tell if things are silenced or

not, and that error kinda hints that the port is empty.

On Sat, Jun 20, 2015 at 10:25 PM, Robert Tsai rtsai@rivermeadow.com wrote:

Hi Kyle,

Here is what I found after I changed the occurances to 1.

{“timestamp”:“2015-06-21T05:23:10.275236+0000”,“level”:“debug”,“message”:"handling

event",“event_data”:"{“id”:“e455f544-7a70-450d-a9e6-57000e99890c”,“client”:{“name”:“infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434864189},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 200 -c

395064",“interval”:10,“occurrences”:1,“refresh”:20,“subscribers”:[“common”],“name”:“memory”,“issued”:1434864190,“executed”:1434864190,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 632

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:200,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/etc/sensu/handlers/ponymailer.rb

",“name”:“ponymailer”}}

{“timestamp”:“2015-06-21T05:17:50.847345+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/etc/sensu/handlers/ponymailer.rb

“,“name”:“ponymailer”},“output”:[”/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1533:in

addr_port': undefined method+’ for nil:NilClass

(NoMethodError)\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1466:in

`begin_transport’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1408:in

`transport_request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1382:in `request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1375:in `block in

request’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:852:in

`start’\n","\tfrom /opt/sensu/embedded/lib/ruby/2.0.0/net/http.rb:1373:in

`request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:85:in

`api_request’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:120:in

`stash_exists?’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:132:in

`block (2 levels) in filter_silenced’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:66:in `timeout’\n","\tfrom

/opt/sensu/embedded/lib/ruby/2.0.0/timeout.rb:97:in `timeout’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:131:in

`block in filter_silenced’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`each’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:129:in

`filter_silenced’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:31:in

`filter’\n","\tfrom

/opt/sensu/embedded/lib/ruby/gems/2.0.0/gems/sensu-plugin-1.1.0/lib/sensu-handler.rb:54:in

`block in class:Handler’\n"]}

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

bigdecimal (1.2.0)

io-console (0.4.2)

json (1.7.7)

mail (2.6.3)

mime-types (2.6.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

pony (1.11)

psych (2.0.0)

rake (0.9.6)

rdoc (4.0.0)

sendmail (0.0.1)

sensu-plugin (1.1.0)

test-unit (2.0.0.0)

timeout (0.0.1)

timeout-extensions (0.0.0)

cat /etc/sensu/conf.d/handler_ponymailer.json

{

"handlers": {
    "ponymailer": {
    "type": "pipe",
    "command":  "/etc/sensu/handlers/ponymailer.rb "
    }
}

}

cat /etc/sensu/conf.d/ponymailer.json

{

“ponymailer”: {

“authenticate”:true,

“only_send_on_change”:false,

“username”:"user@gmail.com",

“tls”:true,

“port”:“587”,

“fromname”:“Operations”,

“hostname”:“smtp.gmail.com”,

“password”:“xxxxx”,

“from”:“ops@xxxx”,

“recipients”:[

"user@gmail.com"

]

}

}

On Sat, Jun 20, 2015 at 2:46 PM, Kyle Anderson kyle@xkyle.com wrote:

If you want to debug a handler not working, you have to get it "in the

act"

You could decrease the occurences setting to “1” so it fires all the time.

Or maybe grep the logs harder? Are there logs that were logrotated +

gzipd?

zgrep ‘handler output’ /var/log/sensu-server*

On Sat, Jun 20, 2015 at 1:26 PM, Robert Tsai rtsai@rivermeadow.com

wrote:

Hey Kyle,

Thanks for helping out! Below is the grepped output but I’m only seeing

info messages for the handler messages. In my /etc/default/sensu, I

have

LOG_LEVEL=debug. I have restarted the sensu-server service. Is there

another place I need to change the log level? Thanks again!!

-Rob

/ponymailer.rb",“name”:“ponymailer”},“output”:["only handling every 180

occurrences: pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:11:30.814117+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences:pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:11:40.807064+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

{“timestamp”:“2015-06-20T20:14:30.838181+0000”,“level”:“info”,“message”:"handler

output",“handler”:

On Sat, Jun 20, 2015 at 10:53 AM, Kyle Anderson kyle@xkyle.com wrote:

Can you find events in the log where there was output from the

ponymailer command? Or any handler?

The one posted was filtered.

Try grepping for “handler output” and then grep out "only handling

every" to see the relevant log lines.

On Fri, Jun 19, 2015 at 10:17 PM, Robert Tsai rtsai@rivermeadow.com

wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I

have

sendmail installed and was able to send an email with mail -s. Below

is

what I have confgured for ponymailer. Any guideline is welcome.

Thanks,

Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)

amqp (1.5.0)

async_sinatra (1.2.0)

bigdecimal (1.2.0)

childprocess (0.5.3)

daemons (1.2.2)

em-redis-unified (1.0.0)

em-worker (0.0.2)

eventmachine (1.0.3)

ffi (1.9.8)

io-console (0.4.2)

ipaddress (0.8.0)

json (1.7.7)

mail (2.5.4)

mime-types (2.6.1, 1.25.1)

minitest (4.3.2)

mixlib-cli (1.5.0)

mixlib-config (2.2.1)

mixlib-log (1.6.0)

mixlib-shellout (2.1.0)

multi_json (1.11.0)

ohai (6.16.0)

polyglot (0.3.5)

pony (1.11)

psych (2.0.0)

rack (1.6.1)

rack-protection (1.5.3)

rake (0.9.6)

rdoc (4.0.0)

sensu (0.19.2)

sensu-em (2.5.2)

sensu-extension (1.1.2)

sensu-extensions (1.2.0)

sensu-logger (1.0.0)

sensu-plugin (1.1.0)

sensu-settings (1.9.0)

sensu-spawn (1.2.0)

sensu-transport (3.0.0)

sinatra (1.4.6)

systemu (2.6.5, 2.5.2)

test-unit (2.0.0.0)

thin (1.6.3)

tilt (2.0.1)

treetop (1.4.15)

uuidtools (2.1.5)

yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb

#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a

pre-defined

recipient.

Copyright 2012 Panagiotis Papadomitsos pj@ezgr.net

Released under the same terms as Sensu (the MIT license); see

LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’

require ‘sensu-handler’

require ‘timeout’

require ‘pony’

require ‘date’

class PonyMailer < Sensu::Handler

STATUSES = {

0 => 'OK',
1 => 'WARNING',
2 => 'CRITICAL'

}

def short_name

@event['client']['name'] + '/' + @event['check']['name']

end

def action_to_string

@event['action'].eql?('resolve') ? 'RESOLVED' : 'ALERT'

end

def handle

if settings['ponymailer']['only_send_on_change']
  last_status = @event['check']['history'][-2].to_i
  if last_status == @event['check']['status']
    return
  end
end
mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} ::

#{short_name}",

  from: "#{settings['ponymailer']['fromname']}

<#{settings[‘ponymailer’][‘from’]}>",

  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address:

settings[‘ponymailer’][‘hostname’],

                      port: settings['ponymailer']['port'],
                      enable_starttls_auto:

settings[‘ponymailer’][‘tls’],

                      user_name:

settings[‘ponymailer’][‘username’],

                      password:

settings[‘ponymailer’][‘password’],

                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']
mail_options[:body] = %(Sensu has detected a failed check. Event

analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}

Check That Failed: #{@event[‘check’][‘name’]}

Check Command: #{@event[‘check’][‘command’]}

Check Flapping: #{@event[‘check’][‘flapping’]}

Check Occurrences: #{@event[‘occurrences’]}

Check History: #{@event[‘check’][‘history’].map { |h|

STATUSES[h.to_i]

}.join(’ => ')}

Node Name: #{@event[‘client’][‘name’]}

Node IP Address: #{@event[‘client’][‘address’]}

Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}

Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

====================

=== Check Output ===

====================

#{@event[‘check’][‘output’]}

)

Pony.options = mail_options
# #YELLOW
unless settings['ponymailer']['recipients'].empty? #

rubocop:disable

GuardClause

  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' +

@event[‘action’]

an incident – ’ + short_name

    end
  end
end

end

end

/etc/sensu/conf.d/ponymailer.json

{

"ponymailer": {
    "authenticate":true,
    "only_send_on_change":false,
    "username":"xxx@gmail.com",
    "tls":true,
    "port":"587",
    "fromname":"Operations",
    "hostname":"[smtp.gmail.com](http://smtp.gmail.com)",
    "password":"xxxx",
    "from":"ops@xxxx.net",
    "recipients":[
        "xxxx@gmail.com"
    ]
}

}

/etc/sensu/conf.d/handler_ponymailer.json

{

"handlers": {
    "ponymailer": {
    "type": "pipe",
    "command": "/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb"

    }
}

}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:"received

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:"processing

result",“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}}

{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:"storing

check

result",“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672 MB\n",“status”:2}}

{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:"processing

event",“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}

{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:"handling

event",“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh

-w 303800 -c

303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:"MEM

CRITICAL - free system memory: 672

MB\n",“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}

{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:"handler

output",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby

/etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:["only

handling every 180 occurrences: pdx01.infra.domain.net/memory\n"]}

Robert Tsai

RiverMeadow Operations

(650) 452 - 2020

Robert Tsai

RiverMeadow Operations

(650) 452 - 2020


Robert Tsai

RiverMeadow Operations
(650) 452 - 2020

Robert Tsai

RiverMeadow Operations
(650) 452 - 2020


#9

Did you ever figure this one out? I have a similar problem that might be related.

···

On Saturday, June 20, 2015 at 12:17:38 AM UTC-5, Robert Tsai wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I have sendmail installed and was able to send an email with mail -s. Below is what I have confgured for ponymailer. Any guideline is welcome.
Thanks,
Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)
amqp (1.5.0)
async_sinatra (1.2.0)
bigdecimal (1.2.0)
childprocess (0.5.3)
daemons (1.2.2)
em-redis-unified (1.0.0)
em-worker (0.0.2)
eventmachine (1.0.3)
ffi (1.9.8)
io-console (0.4.2)
ipaddress (0.8.0)
json (1.7.7)
mail (2.5.4)
mime-types (2.6.1, 1.25.1)
minitest (4.3.2)
mixlib-cli (1.5.0)
mixlib-config (2.2.1)
mixlib-log (1.6.0)
mixlib-shellout (2.1.0)
multi_json (1.11.0)
ohai (6.16.0)
polyglot (0.3.5)
pony (1.11)
psych (2.0.0)
rack (1.6.1)
rack-protection (1.5.3)
rake (0.9.6)
rdoc (4.0.0)
sensu (0.19.2)
sensu-em (2.5.2)
sensu-extension (1.1.2)
sensu-extensions (1.2.0)
sensu-logger (1.0.0)
sensu-plugin (1.1.0)
sensu-settings (1.9.0)
sensu-spawn (1.2.0)
sensu-transport (3.0.0)
sinatra (1.4.6)
systemu (2.6.5, 2.5.2)
test-unit (2.0.0.0)
thin (1.6.3)
tilt (2.0.1)
treetop (1.4.15)
uuidtools (2.1.5)
yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb
#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a pre-defined recipient.

Copyright 2012 Panagiotis Papadomitsos p...@ezgr.net

Released under the same terms as Sensu (the MIT license); see LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’
require ‘sensu-handler’
require ‘timeout’
require ‘pony’
require ‘date’

class PonyMailer < Sensu::Handler
STATUSES = {
0 => ‘OK’,
1 => ‘WARNING’,
2 => ‘CRITICAL’
}

def short_name
@event[‘client’][‘name’] + ‘/’ + @event[‘check’][‘name’]
end

def action_to_string
@event[‘action’].eql?(‘resolve’) ? ‘RESOLVED’ : ‘ALERT’
end

def handle
if settings[‘ponymailer’][‘only_send_on_change’]
last_status = @event[‘check’][‘history’][-2].to_i
if last_status == @event[‘check’][‘status’]
return
end
end

mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} :: #{short_name}",
  from: "#{settings['ponymailer']['fromname']} <#{settings['ponymailer']['from']}>",
  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address: settings['ponymailer']['hostname'],
                      port: settings['ponymailer']['port'],
                      enable_starttls_auto: settings['ponymailer']['tls'],
                      user_name: settings['ponymailer']['username'],
                      password: settings['ponymailer']['password'],
                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']

mail_options[:body] = %(Sensu has detected a failed check. Event analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}
Check That Failed: #{@event[‘check’][‘name’]}
Check Command: #{@event[‘check’][‘command’]}
Check Flapping: #{@event[‘check’][‘flapping’]}
Check Occurrences: #{@event[‘occurrences’]}
Check History: #{@event[‘check’][‘history’].map { |h| STATUSES[h.to_i] }.join(’ => ‘)}
Node Name: #{@event[‘client’][‘name’]}
Node IP Address: #{@event[‘client’][‘address’]}
Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}
Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

=== Check Output ===

#{@event[‘check’][‘output’]}
)
Pony.options = mail_options

# #YELLOW
unless settings['ponymailer']['recipients'].empty? # rubocop:disable GuardClause
  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' + @event['action'] + ' an incident -- ' + short_name
    end
  end
end

end
end

/etc/sensu/conf.d/ponymailer.json
{
“ponymailer”: {
“authenticate”:true,
“only_send_on_change”:false,
“username”:"x...@gmail.com",
“tls”:true,
“port”:“587”,
“fromname”:“Operations”,
“hostname”:“smtp.gmail.com”,
“password”:“xxxx”,
“from”:"o...@xxxx.net",
“recipients”:[
"xx...@gmail.com"
]
}
}

/etc/sensu/conf.d/handler_ponymailer.json
{
“handlers”: {
“ponymailer”: {
“type”: “pipe”,
“command”: “/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb”
}
}
}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:“received result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:“processing result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:“storing check result”,“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}
{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:“processing event”,“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}
{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:“handling event”,“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}
{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences: pdx01.infra.domain.net/memory\n”]}


#10

Try adding “host”: “localhost”, in api config and restart

“api”: {
"host": “localhost”, “port”: 4567
}

···

On Jul 23, 2015, at 1:22 PM, James Jelinek jelinek@gmail.com wrote:

Did you ever figure this one out? I have a similar problem that might be related.

On Saturday, June 20, 2015 at 12:17:38 AM UTC-5, Robert Tsai wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I have sendmail installed and was able to send an email with mail -s. Below is what I have confgured for ponymailer. Any guideline is welcome.
Thanks,
Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)
amqp (1.5.0)
async_sinatra (1.2.0)
bigdecimal (1.2.0)
childprocess (0.5.3)
daemons (1.2.2)
em-redis-unified (1.0.0)
em-worker (0.0.2)
eventmachine (1.0.3)
ffi (1.9.8)
io-console (0.4.2)
ipaddress (0.8.0)
json (1.7.7)
mail (2.5.4)
mime-types (2.6.1, 1.25.1)
minitest (4.3.2)
mixlib-cli (1.5.0)
mixlib-config (2.2.1)
mixlib-log (1.6.0)
mixlib-shellout (2.1.0)
multi_json (1.11.0)
ohai (6.16.0)
polyglot (0.3.5)
pony (1.11)
psych (2.0.0)
rack (1.6.1)
rack-protection (1.5.3)
rake (0.9.6)
rdoc (4.0.0)
sensu (0.19.2)
sensu-em (2.5.2)
sensu-extension (1.1.2)
sensu-extensions (1.2.0)
sensu-logger (1.0.0)
sensu-plugin (1.1.0)
sensu-settings (1.9.0)
sensu-spawn (1.2.0)
sensu-transport (3.0.0)
sinatra (1.4.6)
systemu (2.6.5, 2.5.2)
test-unit (2.0.0.0)
thin (1.6.3)
tilt (2.0.1)
treetop (1.4.15)
uuidtools (2.1.5)
yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb
#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a pre-defined recipient.

Copyright 2012 Panagiotis Papadomitsos p...@ezgr.net

Released under the same terms as Sensu (the MIT license); see LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’
require ‘sensu-handler’
require ‘timeout’
require ‘pony’
require ‘date’

class PonyMailer < Sensu::Handler
STATUSES = {
0 => ‘OK’,
1 => ‘WARNING’,
2 => ‘CRITICAL’
}

def short_name
@event[‘client’][‘name’] + ‘/’ + @event[‘check’][‘name’]
end

def action_to_string
@event[‘action’].eql?(‘resolve’) ? ‘RESOLVED’ : ‘ALERT’
end

def handle
if settings[‘ponymailer’][‘only_send_on_change’]
last_status = @event[‘check’][‘history’][-2].to_i
if last_status == @event[‘check’][‘status’]
return
end
end

mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} :: #{short_name}",
  from: "#{settings['ponymailer']['fromname']} <#{settings['ponymailer']['from']}>",
  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address: settings['ponymailer']['hostname'],
                      port: settings['ponymailer']['port'],
                      enable_starttls_auto: settings['ponymailer']['tls'],
                      user_name: settings['ponymailer']['username'],
                      password: settings['ponymailer']['password'],
                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']

mail_options[:body] = %(Sensu has detected a failed check. Event analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}
Check That Failed: #{@event[‘check’][‘name’]}
Check Command: #{@event[‘check’][‘command’]}
Check Flapping: #{@event[‘check’][‘flapping’]}
Check Occurrences: #{@event[‘occurrences’]}
Check History: #{@event[‘check’][‘history’].map { |h| STATUSES[h.to_i] }.join(’ => ‘)}
Node Name: #{@event[‘client’][‘name’]}
Node IP Address: #{@event[‘client’][‘address’]}
Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}
Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

=== Check Output ===

#{@event[‘check’][‘output’]}
)
Pony.options = mail_options

# #YELLOW
unless settings['ponymailer']['recipients'].empty? # rubocop:disable GuardClause
  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' + @event['action'] + ' an incident -- ' + short_name
    end
  end
end

end
end

/etc/sensu/conf.d/ponymailer.json
{
“ponymailer”: {
“authenticate”:true,
“only_send_on_change”:false,
“username”:"x...@gmail.com",
“tls”:true,
“port”:“587”,
“fromname”:“Operations”,
“hostname”:“smtp.gmail.com”,
“password”:“xxxx”,
“from”:"o...@xxxx.net",
“recipients”:[
"xx...@gmail.com"
]
}
}

/etc/sensu/conf.d/handler_ponymailer.json
{
“handlers”: {
“ponymailer”: {
“type”: “pipe”,
“command”: “/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb”
}
}
}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:“received result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:“processing result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:“storing check result”,“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}
{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:“processing event”,“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}
{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:“handling event”,“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}
{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences: pdx01.infra.domain.net/memory\n”]}


#11

This was the problem I experienced. It’s not documented that the sensu-plugin doesn’t pick that field up. The Sensu team is going to update the plugin doc to notate that. I spent like 6 hours debugging this today :frowning:

···

Sent from my iPhone

On Jul 23, 2015, at 19:51, San s warpdrive47@gmail.com wrote:

Try adding “host”: “localhost”, in api config and restart

“api”: {
"host": “localhost”, “port”: 4567
}

On Jul 23, 2015, at 1:22 PM, James Jelinek jelinek@gmail.com wrote:

Did you ever figure this one out? I have a similar problem that might be related.

On Saturday, June 20, 2015 at 12:17:38 AM UTC-5, Robert Tsai wrote:

HI I can’t get either ponymailer nor mailer to send an email out. I have sendmail installed and was able to send an email with mail -s. Below is what I have confgured for ponymailer. Any guideline is welcome.
Thanks,
Robert

/opt/sensu/embedded/bin/gem list

*** LOCAL GEMS ***

amq-protocol (1.9.2)
amqp (1.5.0)
async_sinatra (1.2.0)
bigdecimal (1.2.0)
childprocess (0.5.3)
daemons (1.2.2)
em-redis-unified (1.0.0)
em-worker (0.0.2)
eventmachine (1.0.3)
ffi (1.9.8)
io-console (0.4.2)
ipaddress (0.8.0)
json (1.7.7)
mail (2.5.4)
mime-types (2.6.1, 1.25.1)
minitest (4.3.2)
mixlib-cli (1.5.0)
mixlib-config (2.2.1)
mixlib-log (1.6.0)
mixlib-shellout (2.1.0)
multi_json (1.11.0)
ohai (6.16.0)
polyglot (0.3.5)
pony (1.11)
psych (2.0.0)
rack (1.6.1)
rack-protection (1.5.3)
rake (0.9.6)
rdoc (4.0.0)
sensu (0.19.2)
sensu-em (2.5.2)
sensu-extension (1.1.2)
sensu-extensions (1.2.0)
sensu-logger (1.0.0)
sensu-plugin (1.1.0)
sensu-settings (1.9.0)
sensu-spawn (1.2.0)
sensu-transport (3.0.0)
sinatra (1.4.6)
systemu (2.6.5, 2.5.2)
test-unit (2.0.0.0)
thin (1.6.3)
tilt (2.0.1)
treetop (1.4.15)
uuidtools (2.1.5)
yajl-ruby (1.2.1)

cat /etc/sensu/handlers/ponymailer.rb
#!/usr/bin/env ruby

Sensu Handler: ponymailer

This handler formats alerts as mails and sends them off to a pre-defined recipient.

Copyright 2012 Panagiotis Papadomitsos p...@ezgr.net

Released under the same terms as Sensu (the MIT license); see LICENSE

for details.

require ‘rubygems’ if RUBY_VERSION < ‘1.9.0’
require ‘sensu-handler’
require ‘timeout’
require ‘pony’
require ‘date’

class PonyMailer < Sensu::Handler
STATUSES = {
0 => ‘OK’,
1 => ‘WARNING’,
2 => ‘CRITICAL’
}

def short_name
@event[‘client’][‘name’] + ‘/’ + @event[‘check’][‘name’]
end

def action_to_string
@event[‘action’].eql?(‘resolve’) ? ‘RESOLVED’ : ‘ALERT’
end

def handle
if settings[‘ponymailer’][‘only_send_on_change’]
last_status = @event[‘check’][‘history’][-2].to_i
if last_status == @event[‘check’][‘status’]
return
end
end

mail_options = {
  subject: "Sensu Monitoring Alert: #{action_to_string} :: #{short_name}",
  from: "#{settings['ponymailer']['fromname']} <#{settings['ponymailer']['from']}>",
  via: :smtp,
  via_options: {
    address: settings['ponymailer']['hostname'],
    port: settings['ponymailer']['port'],
    enable_starttls_auto: settings['ponymailer']['tls']
  },
  charset: 'utf-8',
  sender: settings['ponymailer']['from']
}
mail_options.merge!(via_options: {
                      address: settings['ponymailer']['hostname'],
                      port: settings['ponymailer']['port'],
                      enable_starttls_auto: settings['ponymailer']['tls'],
                      user_name: settings['ponymailer']['username'],
                      password: settings['ponymailer']['password'],
                      authentication: :plain
                    }) if settings['ponymailer']['authenticate']

mail_options[:body] = %(Sensu has detected a failed check. Event analysis follows:

Event Timestamp: #{Time.at(@event[‘check’][‘issued’].to_i)}
Check That Failed: #{@event[‘check’][‘name’]}
Check Command: #{@event[‘check’][‘command’]}
Check Flapping: #{@event[‘check’][‘flapping’]}
Check Occurrences: #{@event[‘occurrences’]}
Check History: #{@event[‘check’][‘history’].map { |h| STATUSES[h.to_i] }.join(’ => ‘)}
Node Name: #{@event[‘client’][‘name’]}
Node IP Address: #{@event[‘client’][‘address’]}
Node LPOL: #{Time.at(@event[‘client’][‘timestamp’].to_i)}
Node Subscriptions: #{@event[‘client’][‘subscriptions’].join(’, ')}

=== Check Output ===

#{@event[‘check’][‘output’]}
)
Pony.options = mail_options

# #YELLOW
unless settings['ponymailer']['recipients'].empty? # rubocop:disable GuardClause
  settings['ponymailer']['recipients'].each do |to|
    begin
      Timeout.timeout 10 do
        Pony.mail(to: to)
        puts 'mail -- sent alert for ' + short_name + ' to ' + to
      end
  rescue Timeout::Error
    puts 'mail -- timed out while attempting to ' + @event['action'] + ' an incident -- ' + short_name
    end
  end
end

end
end

/etc/sensu/conf.d/ponymailer.json
{
“ponymailer”: {
“authenticate”:true,
“only_send_on_change”:false,
“username”:"x...@gmail.com",
“tls”:true,
“port”:“587”,
“fromname”:“Operations”,
“hostname”:“smtp.gmail.com”,
“password”:“xxxx”,
“from”:"o...@xxxx.net",
“recipients”:[
"xx...@gmail.com"
]
}
}

/etc/sensu/conf.d/handler_ponymailer.json
{
“handlers”: {
“ponymailer”: {
“type”: “pipe”,
“command”: “/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb”
}
}
}

{“timestamp”:“2015-06-20T04:57:00.233492+0000”,“level”:“debug”,“message”:“received result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.233610+0000”,“level”:“debug”,“message”:“processing result”,“result”:{“client”:“pdx01.infra.domain.net”,“check”:{“name”:“memory”,“issued”:1434776220,“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“handlers”:[“ponymailer”],“interval”:10,“subscribers”:[“common”],“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}}
{“timestamp”:“2015-06-20T04:57:00.234173+0000”,“level”:“debug”,“message”:“storing check result”,“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2}}
{“timestamp”:“2015-06-20T04:57:00.235571+0000”,“level”:“info”,“message”:“processing event”,“event”:{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}}
{“timestamp”:“2015-06-20T04:57:00.235927+0000”,“level”:“debug”,“message”:“handling event”,“event_data”:"{“id”:“b67033bb-9a78-4fd6-a98b-3a525f7bf552”,“client”:{“name”:“pdx01.infra.domain.net”,“address”:“localhost”,“subscriptions”:[“common”],“version”:“0.19.2”,“timestamp”:1434776206},“check”:{“handlers”:[“ponymailer”],“command”:"/etc/sensu/plugins/check-mem.sh -w 303800 -c 303800",“interval”:10,“subscribers”:[“common”],“name”:“memory”,“issued”:1434776220,“executed”:1434776220,“duration”:0.006,“output”:“MEM CRITICAL - free system memory: 672 MB\n”,“status”:2,“history”:[“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”,“2”],“total_state_change”:0},“occurrences”:723,“action”:“create”}",“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”}}
{“timestamp”:“2015-06-20T04:57:00.805423+0000”,“level”:“info”,“message”:“handler output”,“handler”:{“type”:“pipe”,“command”:"/opt/sensu/embedded/bin/ruby /etc/sensu/handlers/ponymailer.rb",“name”:“ponymailer”},“output”:[“only handling every 180 occurrences: pdx01.infra.domain.net/memory\n”]}