LogCheck

Put this script at /opt/scripts/logcheck.pl

It will check the logs each time it is run from cron. A 'offset' is recorded when run so that it does not analyse the same log entries twice.

#!/usr/bin/perl

use strict;

# Script to check log files for error messages and provide additional
# alerting to NetCool

# Script checks if log has been rotated and if not will only scan for new
# lines in the log

# Author - Gordon Johnston
# Date - 05/03/2007
# Version 1.0

my $LOGFILE = "/var/adm/messages"; # Log file to test

my $OFFSETFILE = "/opt/scripts/offset"; # Offset to start checking the log
my $FIRSTLINEFILE = "/opt/scripts/firstline"; # First line last seen in log file

my @ALERTON = ('error', 'warning', 'online', 'offline', 'reboot'); # List of strings to search for

my $EMAILTO = 'mail@address.com'; # Email address to send alerts to

my $HOSTNAME = `hostname`;
chomp $HOSTNAME;

my $LOGGER = '/usr/bin/logger';
my $MAILER = '/usr/bin/mail';

# First check we can read from log file

if (-r $LOGFILE) {

        open (LOG, "< $LOGFILE") or die "Could not open $LOGFILE: $!\n";;
        my $firstLine = <LOG>;
        my $offset = 0;
        # Now check that line agast the FIRSTLINEFILE if any
        if (-r $FIRSTLINEFILE) {
                open (FIRST, "< $FIRSTLINEFILE") or die "Could not open $FIRSTLINEFILE: $!\n";
                my $oldFirstLine = <FIRST>;
                if ($oldFirstLine eq $firstLine) {
                        # Log file is same file as last checked
                        if (-r $OFFSETFILE) {
                                open (OFFSET, "< $OFFSETFILE") or die "Could not open $OFFSETFILE: $!\n";
                                $offset = <OFFSET>;
                                chomp $offset;
                                close OFFSET;
                        } else {
                                print STDERR "Same file but offset not recorded from previous run at $OFFSETFILE\n";
                        }
                } else {
                        # Log file has been rotated
                }
                close FIRST;
        }

        # Now read in the log into an array but throw away upto $offset
        my @lines;
        my $linesSeen = 1; # The line we already read
        if (!$offset) {
                # Add the line already read to the file
                push @lines, $firstLine;
        }
        while ($linesSeen < $offset) {
                my $junk = <LOG>; # Throw away lines
                $linesSeen++;
        }
        while (my $line = <LOG>) {
                push @lines, $line;
                $linesSeen++;
        }
        
        # Update the 'state' files
        open (FIRST, "> $FIRSTLINEFILE") or die "Could not open $FIRSTLINEFILE for writing: $!\n";
        print FIRST $firstLine;
        close FIRST;
        open (OFFSET, "> $OFFSETFILE") or die "Could not open $OFFSETFILE for writing: $!\n";
        print OFFSET $linesSeen;
        close OFFSET;
        
        # Now check the new lines for the error strings
        
        foreach my $line (@lines) {
                if (grep ($line =~ /$_/i, @ALERTON)) {
                        # We got a match;
                        &sendAlert ($line);
                }
        }
        close LOG;

} else {
        print "Unable to read from $LOGFILE\n";
}

sub sendAlert {
        my $alert = shift;
        # Send the alert
        `$LOGGER -i -p user.err Alert: A critical alert has been found in the syslog. Please check\n `;
        
        open (MAIL, "| $MAILER $EMAILTO");
        print MAIL "Subject: $HOSTNAME\n";
        print MAIL "##################################################################\n";
        print MAIL "  Found the following text in $alert on $HOSTNAME\n";
        print MAIL "  PLEASE CHECK\n\n";
        print MAIL "##################################################################\n\n";
        close MAIL;
}