[Xymon] Clariion monitoring

[Betsy Schwartz]

There's this rather crude script
I learned much more perl since I wrote it and looking at it now
there's a bunch of stuff I want to fix, but you can play with it
(I am going to clean it up, and add custom per-device storage levels,
ask me in a while if I've done that :-)

Assumes youv'e set up sshkey access for xymon user -> "nasadmin" on your devices

#!/usr/bin/perl
#
# Script to test EMC storage devices in xymon
#
#  2011/11/08  1.0   - betsys - initial install
#
#
use strict;
use warnings;
use DBI;
use Time::Local;

#use Net::SSH::Perl;

use constant false => 0;
use constant true  => 1;

my $REDLEVEL    = 95;
my $YELLOWLEVEL = 90;

my $REDDOT    = '<IMG SRC="/xymon/gifs/red-recent.gif">';
my $YELLOWDOT = '<IMG SRC="/xymon/gifs/yellow-recent.gif">';

#-------- Configure
my $BBHOME    = ENV{BBHOME};     # Path to the BBServer software.
my $BBTMP     = "$BBHOME/tmp";
my $BB         = $ENV{'BB'};
my $BBDISP     = $ENV{'BBDISP'};
my $BBDISPLAYS = $ENV{'BBDISPLAYS'};
my $TESTNAME  = "temp";                        # Test name as
displayed on the main page (bb.html).

#-------- End Configure

my $COLOR     = "clear";
my $BBTMPLOG  = "$BBHOME/tmp/TESTNAME.tmp";
my $BBSTATLOG = "$BBHOME/tmp/TESTNAME";
my $epoch     = time();
my $date      = localtime($epoch);
my $status    = "clear";
my $gored     = false;
my $goyellow  = false;
my $nasadmin   = "nasadmin";

my $server = "";    # we're using strict
my @serverlist = ( "server1.example.com", "server2.example.com",
"server3.example.com" );

foreach $server (@serverlist)
{
    test_uptime($server);
    test_storage($server);
    test_replication($server);

}

sub test_replication {
    my ($serv) = @_;
    $COLOR    = "green";
    $gored    = false;
    $goyellow = false;
    my @warnings = ("\n");

    my $TESTNAME = "nas_rep";    # Test name as displayed on the main
page (bb.html).
    my $nascmd = "export NAS_DB=/nas;/nas/bin/nas_replicate -list";
    my $sshcmd = "ssh $nasadmin\@$serv \"$nascmd\"";

    my @output   = `$sshcmd  >/tmp/$serv.nas_replicate.out `;
    my @dataline = `cat /tmp/$serv.nas_replicate.out`;

    open( FH, "< /tmp/$serv.nas_replicate.out" );
    while (<FH>)
    {
        my @line = (split);
        next if ( @line == 0 );    # skip blanks
        next unless ( $line[1] =~ /filesystem/ );
        if ( $line[5] ne "OK" )
        {
            $goyellow = true;
            push( @warnings, "Problem with replication @line \n" );
        }
    }
    if    ( $gored == true )    { $COLOR = "red"; }
    elsif ( $goyellow == true ) { $COLOR = "yellow"; }
    else
    {
        $COLOR = "green";
        push( @warnings, "Replication looks OK on $serv\n\n" );
    }

    my $bbcmd = "$BB $BBDISPLAY 'status+2h $serv.$TESTNAME $COLOR
$date \n @warnings \n @dataline' ";

    #    print "command is";
    #    print $bbcmd;
    #    print "\n";

    system("$bbcmd");

}

sub test_uptime {
    my ($serv) = @_;
    $COLOR = "green";
    my $TESTNAME = "uptime";                                        #
Test name as displayed on the main page (bb.html).
    my $nascmd   = "export NAS_DB=/nas;/nas/bin/server_uptime ALL";
    my $sshcmd   = "ssh nasadmin\@$serv \"$nascmd\"";

    #   my @output = `$sshcmd  >/tmp/$serv.uptime.out 2 >& /dev/null`;
    my @output = `$sshcmd  >/tmp/$serv.uptime.out `;

    #   print "$server uptime output is  @output \n";
    open( FH, "< /tmp/$serv.uptime.out" );
    while (<FH>)
    {
        $COLOR = "red" if /fault/;
    }

    my @dataline = `cat /tmp/$serv.uptime.out`;
    my $bbcmd    = "$BB $BBDISPLAY 'status+2h $serv.$TESTNAME $COLOR
$date \n @dataline' ";

    #    print "command is";
    #    print $bbcmd;
    #    print "\n";
    system("$bbcmd");
}

sub test_storage {
    my ($serv) = @_;
    $COLOR    = "green";
    $gored    = false;
    $goyellow = false;
    my @warnings = ("\n");
    my $TESTNAME = "storage";    # Test name as displayed on the main
page (bb.html).
    my $scratch  = "";
    my ( $fs, $kb, $used, $avail, $cap, $mount );

    my $nascmd =
      "export NAS_DB=/nas;/nas/bin/server_df ALL |grep -v iscsi| grep
-v root_ | grep -v ckpt | grep -v automaticND";
    my $sshcmd   = "ssh nasadmin\@$serv \"$nascmd\"";
    my @output   = `$sshcmd  >/tmp/$serv.storage.out `;
    my @dataline = `cat /tmp/$serv.storage.out`;

    #       print "$serv storage output is  @dataline \n";
    open( FH, "< /tmp/$serv.storage.out" );

    while (<FH>)
    {
        next if /server_/;
        next if /Filesystem/;
        my @line = (split);
        if ( @line == 0 ) { next }    # skip blanks
        if ( @line == 1 )
        {                             # we might be on the first line
of a wrapped line
            $scratch = $line[0];      # but some one-word lines are
ckpts followed by real lines
            next;
        }
        elsif ( ( @line >= 5 ) && ( $line[4] =~ /\%/ ) )
        {                             # we have a filesystem data line
            ( $fs, $kb, $used, $avail, $cap, $mount ) = @line;
        }
        elsif ( ( $scratch ne '' ) && ( $line[3] =~ /\%/ ) )
        {                             # we have the second line of a wrap
            $fs = $scratch;
            ( $kb, $used, $avail, $cap, $mount ) = @line;
        }
        else
        {
            $scratch = "";
            next;
        }    #at the moment, not interested in non-data lines

        my $full = $cap;    # number with pesky "%" appended
        $full =~ s/%//;     # is there a more elegant way to do this?

        #               print "used space is $full % \n";
        if ( $full >= $REDLEVEL )
        {
            $gored = true;
            push( @warnings,
                "$REDDOT  filesystem $fs ($cap used) has reached the
PANIC level ($REDLEVEL%) on mountpoint $mount \n"
            );
        }
        elsif ( $full >= $YELLOWLEVEL )
        {
            $goyellow = true;
            push( @warnings,
"$YELLOWDOT  filesystem $fs ($cap used) has reached the WARNING level
($YELLOWLEVEL%) on mountpoint $mount \n"
            );
        }

        #               my $i=0;
        #               for ( $i=0; $i<@line; $i++){
        #                       print "element #$i is $line[$i]\n";
        #               }
    }

    #!#     print "$serv output is  @output";
    if    ( $gored == true )    { $COLOR = "red"; }
    elsif ( $goyellow == true ) { $COLOR = "yellow"; }
    else                        { $COLOR = "green"; }
    my $bbcmd = "$BB $BBDISPLAY 'status+2h $serv.$TESTNAME $COLOR
$date \n @warnings \n @dataline' ";

    #    print "command is";
    #    print $bbcmd;
    #    print "\n";

    system("$bbcmd");

}