Sunday, March 10, 2013

WebSphere MQ hardcore process cleanup in PERL



Note: this script does a kill -TERM first, then waits to issue a kill -9, terminating processes in the correct order according to the IBM Websphere MQ reference manual.







#!/usr/bin/perl -w
use strict;
##########################################################################
#
# Filename: killallmq.pl
# Version:              1.0
# Date:                 03/28/2012
# Author(s):            Matt Feenstra
# Last updated by:      <>
#
# INFO: This script forcefully terminates MQSeries (WMQ) by:
# 1. Retrieving a list of PIDs for all MQ processes
# 2. For each process, in the correct order:
# a. Send interrupt signal SIGTERM (15)
# b. Wait $sleepwait (10) seconds
# c. If process still exists, issue SIGKILL (9)
# 3. Cleanup all IPC resources belonging to MQM
#
# USAGE: 1. Must be ran as ROOT
# 2. Kill output sent to /tmp/stdio.out and /tmp/stderr.out
#
# NOTES: 1. The initial SIGTERM signal on the processes may
# cause an automatic cascade of process termmination
# due to the nature of MQ process interdependencies.
# 2. It may be helpful to backup and clear out the
# /var/mqm/errors directory such that new AMQERR logs
# are generated on restart.
# 3. When the queue manager is stopped manually in this way,
# FFSTs might be taken, and FDC files placed in
# /var/mqm/errors.  However, this is not a defect in
# the queue manager.
# 4. There is no risk of losing persistent messages.  However
# non-persistent messages, or those with expiry may be
# lost as normal.  Persistent messaging involves a
# complex handshaking protocol that ensures the message
# is safely stored before forwarding.
#
##########################################################################

my $who = `whoami`;
chomp($who);
## if($who ne "root") { die "This program must be ran as root."; }

##########################################################################
# Main()
##########################################################################

my $sleepwait = 10; # seconds to wait after kill 15

&killmq();
&cleanIPC();

##########################################################################
# Subroutines
##########################################################################


# Description: Issues a SIGTERM (15) to the PID
# Requires: PID number
# Returns: N/A
sub kill15cmd {
# signal 15 is a SIGTERM or Termination, which is properly trapped
# by the WMQ code.  We always try this first.
my $pid = shift;
system("kill -15 $pid 1>>/tmp/stdio.out 2>>/tmp/stderr.out");
sleep($sleepwait);
}

# Description: Issues a SIGKILL (9) to the PID
# Requires: PID number
# Returns: N/A
sub kill9cmd {
my $pid = shift;
print "\tInfo:\t\tForcefully removing $pid..\n";
system("kill -9 $pid 1>>/tmp/stdio.out 2>>/tmp/stderr.out");
}

# Description: Determines if a Process ID number is running
# Requires: PID number
# Returns: Boolean (true/false)
sub isproc {

my $pid = shift;
my @procs = `ps -eo pid,args`;
chomp(@procs);

foreach(@procs) {
if($_ =~ /$pid/) {
return(1);
}
}
return(0);
}

# Description: Issues an IPC RM to remove stale Inter Process Communication
# Websphere MQ utilizes System V (SysV) IPC resources which are
# zombied once the parent process is killed.
# Requires: N/A
# Returns: N/A
sub cleanIPC {

my $userid = "mqm";
my @ipcs_s = `ipcs -s`;
my @ipcs_m = `ipcs -m`;
my @ipcs_q = `ipcs -q`;

print "\n***\tStarting IPC Cleanup\t***\n";

# Semaphores
foreach(@ipcs_s) {
if($_ =~ /\S+\s+(\S+)\s+($userid)/) {
#print "semid = $1\towner = $2\n";
print "\tInfo:\t\tRemoving Semaphore ID $1\n";
system("ipcrm -s $1");
}
}
     
# Shared Memory
foreach(@ipcs_m) {
                if($_ =~ /\S+\s+(\S+)\s+($userid)/) {
                        #print "memid = $1\towner = $2\n";
print "\tInfo:\t\tRemoving Shared Memory ID $1\n";
system("ipcrm -m $1");
                }
        }

# IPC SysV Message Queues (not to be confused with MQ)
        foreach(@ipcs_q) {
                if($_ =~ /\S+\s+(\S+)\s+($userid)/) {
                        #print "qid = $1\towner = $2\n";
print "\tInfo:\t\tRemoving Message Queue ID $1\n";
system("ipcrm -q $1");
                }
        }
}

# Description: This is the parent subroutine that actually retrieves and
# kills the WMQ processes in the correct order.
# Requires: N/A
# Returns: STDOUT
sub killmq() {

my @procs = `ps -eo pid,args`;
chomp(@procs);
my @sortedprocs = sort(@procs);
my @mqprocs;
my @mqpids;
my $numpids;

foreach(@sortedprocs) {

if($_ =~ /(\d+)\s+.*?(amq.....)/) {
#print "found: $1\t$2\n";
push(@mqpids, $1);
push(@mqprocs, $2);
}
if($_ =~ /(\d+)\s+.*?(runmq...)/) {
#print "found: $1\t$2\n";
push(@mqpids, $1);
push(@mqprocs, $2);
}
}

$numpids = @mqpids;

# Proper kill order listed in ranked priority:
#
# runmqlsr
# runmqchi
#
# amqzmuc0      Critical process manager
# amqzxma0      Execution controller
# amqzfuma      OAM process
# amqzlaa0      LQM agents
# amqzlsa0      LQM agents
# amqzmgr0      Process controller
# amqzmur0      Restartable process manager
# amqrmppa      Process pooling process
# amqrrmfa      The repository process (for clusters)
# amqzdmaa      Deferred message processor
# amqpcsea      The command server

print "\n***\tStarting Termination Sequence\t***\n";
print "\n***\tInfo:\t\tTerminating Listeners, Channel Initiators, and default MQ broker..\t***\n";
for(my $i = 0; $i < $numpids; $i++) {
if($mqprocs[$i] =~ /runmq/) {

print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
kill15cmd($mqpids[$i]);
if(&isproc($mqpids[$i])) {
print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
&kill9cmd($mqpids[$i]);
}
else {
print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n-\n";
}

splice(@mqpids, $i, 1);
splice(@mqprocs, $i, 1);
$numpids--;
$i--;

}

}
print "\n***\tInfo:\t\tTerminating Critical Process Manager (amqzmuc0)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzmuc./) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

print "\n***\tInfo:\t\tTerminating Execution Controller (amqzxma0)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzxma./) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }


                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating OAM process (amqzfuma)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzfuma/) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }


                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating LQM agents1 (amqzlaa0)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzlaa./) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }


                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating LQM agents2 (amqzlsa0)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzlsa./) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating Process Controller (amqzmgr0)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzmgr./) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating Restartable Process Manager (amqzmur0)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzmur./) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }


        print "\n***\tInfo:\t\tTerminating Process Pooling Process (amqrmppa)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqrmppa/) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating The Repository Process for clusters (amqrrmfa)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqrrmfa/) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating Deferred Message Processor (amqzdmaa)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqzdmaa/) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating The Command Server (amqpcsea)\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {
                if($mqprocs[$i] =~ /amqpcsea/) {

                        print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                        kill15cmd($mqpids[$i]);
                        if(&isproc($mqpids[$i])) {
                                print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                                &kill9cmd($mqpids[$i]);
                        }
                        else {
                                print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                        }

                        splice(@mqpids, $i, 1);
                        splice(@mqprocs, $i, 1);
                        $numpids--;
                        $i--;
                }
        }

        print "\n***\tInfo:\t\tTerminating UNKNOWN processes\t***\n";
        for(my $i = 0; $i < $numpids; $i++) {

                print "-\n\tInfo:\t\tSending termination signal to process \"$mqprocs[$i]\" PID $mqpids[$i]..\n";
                kill15cmd($mqpids[$i]);
                if(&isproc($mqpids[$i])) {
                        print "\tWarning:\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] is still running..\n";
                        &kill9cmd($mqpids[$i]);
                }
                else {
                        print "\tInfo:\t\tProcess \"$mqprocs[$i]\" PID $mqpids[$i] terminated gracefully.\n";
                }

                splice(@mqpids, $i, 1);
                splice(@mqprocs, $i, 1);
                $numpids--;
                $i--;
        }


} ## end killmq()

2 comments: