[Xymon] Perl script to test EMC storage devices in xymon

[Elizabeth Schwartz]

Here's my first crack at a script to test EMC storage in xymon. I need
to expand it to take a list of exceptions to customize disk full
alerts for different volumes
 I think it needs more error checking and a way to check xymon env
variables and get them from the client dir if they aren't set. But so
far it works.

Comments very welcome.  It's my first real perl script.
-------------- next part --------------
#!/usr/bin/perl
#
# Script to test EMC storage devices in xymon
#
#  2011/11/08  1.0   - betsys - initial install
#
#
use strict;
use warnings;
use DBI;
use Time::Local;

#use Net::SSH::Perl;

use constant false => 0;
use constant true  => 1;

my $REDLEVEL    = 95;
my $YELLOWLEVEL = 90;

my $REDDOT    = '<IMG SRC="/xymon/gifs/red-recent.gif">';
my $YELLOWDOT = '<IMG SRC="/xymon/gifs/yellow-recent.gif">';

#-------- Configure
my $BBHOME = "/usr/local/xymon/server";    # Path to the BBServer software.
my $BBTMP  = "$BBHOME/tmp";
my $BB     = "$BBHOME/bin/bb";
my $BBDISPLAY =
  "10.100.5.42 ";    # FQDN, resolvable hostname, or, IP of the BBServer.
my $MACHINE =
  "netmon2.example.com";    # BBClient machine where this script is running.
my $TESTNAME = "temp";      # Test name as displayed on the main page (bb.html).

#-------- End Configure

my $COLOR     = "clear";
my $BBTMPLOG  = "$BBHOME/tmp/TESTNAME.tmp";
my $BBSTATLOG = "$BBHOME/tmp/TESTNAME";
my $epoch     = time();
my $date      = localtime($epoch);
my $status    = "clear";
my $gored     = false;
my $goyellow  = false;

my $server = "";    # we're using strict
my @serverlist = ( "vg8-cs.example.com", "ns80-cs.example.com", "vg8-cs.com" );

foreach $server (@serverlist) {
    test_uptime($server);
    test_storage($server);
    test_replication($server);

}

sub test_replication {
    my ($serv) = @_;
    $COLOR    = "green";
    $gored    = false;
    $goyellow = false;
    my @warnings = ("\n");

    my $TESTNAME =
      "nas_rep";    # Test name as displayed on the main page (bb.html).
    my $nascmd = "export NAS_DB=/nas;/nas/bin/nas_replicate -list";
    my $sshcmd = "ssh nasadmin\@$serv \"$nascmd\"";

    my @output   = `$sshcmd  >/tmp/$serv.nas_replicate.out `;
    my @dataline = `cat /tmp/$serv.nas_replicate.out`;

    open( FH, "< /tmp/$serv.nas_replicate.out" );
    while (<FH>) {
        my @line = (split);
        next if ( @line == 0 );    # skip blanks
        next unless ( $line[1] =~ /filesystem/ );
        if ( $line[5] ne "OK" ) {
            $goyellow = true;
            push( @warnings, "Problem with replication @line \n" );
        }
    }
    if    ( $gored == true )    { $COLOR = "red"; }
    elsif ( $goyellow == true ) { $COLOR = "yellow"; }
    else {
        $COLOR = "green";
        push( @warnings, "Replication looks OK on $serv\n\n" );
    }

    my $bbcmd =
"$BB $BBDISPLAY 'status+2h $serv.$TESTNAME $COLOR $date \n @warnings \n @dataline' ";

    #    print "command is";
    #    print $bbcmd;
    #    print "\n";

    system("$bbcmd");

}

sub test_uptime {
    my ($serv) = @_;
    $COLOR = "green";
    my $TESTNAME =
      "uptime";    # Test name as displayed on the main page (bb.html).
    my $nascmd = "export NAS_DB=/nas;/nas/bin/server_uptime ALL";
    my $sshcmd = "ssh nasadmin\@$serv \"$nascmd\"";

    #	my @output = `$sshcmd  >/tmp/$serv.uptime.out 2 >& /dev/null`;
    my @output = `$sshcmd  >/tmp/$serv.uptime.out `;

    #	print "$server uptime output is  @output \n";
    open( FH, "< /tmp/$serv.uptime.out" );
    while (<FH>) {
        $COLOR = "red" if /fault/;
    }

    my @dataline = `cat /tmp/$serv.uptime.out`;
    my $bbcmd =
      "$BB $BBDISPLAY 'status+2h $serv.$TESTNAME $COLOR $date \n @dataline' ";

    #    print "command is";
    #    print $bbcmd;
    #    print "\n";
    system("$bbcmd");
}

sub test_storage {
    my ($serv) = @_;
    $COLOR    = "green";
    $gored    = false;
    $goyellow = false;
    my @warnings = ("\n");
    my $TESTNAME =
      "storage";    # Test name as displayed on the main page (bb.html).
    my $scratch = "";
    my ( $fs, $kb, $used, $avail, $cap, $mount );

    my $nascmd =
"export NAS_DB=/nas;/nas/bin/server_df ALL |grep -v iscsi| grep -v root_ | grep -v ckpt | grep -v automaticND";
    my $sshcmd   = "ssh nasadmin\@$serv \"$nascmd\"";
    my @output   = `$sshcmd  >/tmp/$serv.storage.out `;
    my @dataline = `cat /tmp/$serv.storage.out`;

    #       print "$serv storage output is  @dataline \n";
    open( FH, "< /tmp/$serv.storage.out" );

    while (<FH>) {
        next if /server_/;
        next if /Filesystem/;
        my @line = (split);
        if ( @line == 0 ) { next }    # skip blanks
        if ( @line == 1 ) {    # we might be on the first line of a wrapped line
            $scratch = $line[0]
              ;    # but some one-word lines are ckpts followed by real lines
            next;
        }
        elsif ( ( @line >= 5 ) && ( $line[4] =~ /\%/ ) )
        {          # we have a filesystem data line
            ( $fs, $kb, $used, $avail, $cap, $mount ) = @line;
        }
        elsif ( ( $scratch ne '' ) && ( $line[3] =~ /\%/ ) )
        {          # we have the second line of a wrap
            $fs = $scratch;
            ( $kb, $used, $avail, $cap, $mount ) = @line;
        }
        else {
            $scratch = "";
            next;
        }    #at the moment, not interested in non-data lines

        my $full = $cap;    # number with pesky "%" appended
        $full =~ s/%//;     # is there a more elegant way to do this?

        #               print "used space is $full % \n";
        if ( $full >= $REDLEVEL ) {
            $gored = true;
            push( @warnings,
"$REDDOT  filesystem $fs ($cap used) has reached the PANIC level ($REDLEVEL%) on mountpoint $mount \n"
            );
        }
        elsif ( $full >= $YELLOWLEVEL ) {
            $goyellow = true;
            push( @warnings,
"$YELLOWDOT  filesystem $fs ($cap used) has reached the WARNING level ($YELLOWLEVEL%) on mountpoint $mount \n"
            );
        }

        #               my $i=0;
        #               for ( $i=0; $i<@line; $i++){
        #                       print "element #$i is $line[$i]\n";
        #               }
    }

    #!#     print "$serv output is  @output";
    if    ( $gored == true )    { $COLOR = "red"; }
    elsif ( $goyellow == true ) { $COLOR = "yellow"; }
    else                        { $COLOR = "green"; }
    my $bbcmd =
"$BB $BBDISPLAY 'status+2h $serv.$TESTNAME $COLOR $date \n @warnings \n @dataline' ";

    #    print "command is";
    #    print $bbcmd;
    #    print "\n";

    system("$bbcmd");

}