Elastic Search cluster status primary check script (ruby/json)

#!/usr/bin/ruby

require "rubygems"
require "json"
require "socket"


# This script is to be ran on an Elastic Search server and returns 1 or 0 depending on
# if the system is the current primary for a cluster.
# Matt Feenstra 11/25/13

#@file = open("curl.out")
#@json = @file.read

# @json = `curl -XGET http://#{myhostname}:9200/_cluster/state`;
# myhostname = "paz02app8825"


# get my local hostname
myhostname = Socket.gethostbyname(Socket.gethostname).first
# get JSON output from ES server
@json = `curl -XGET http://#{myhostname}:9200/_cluster/state 2>/dev/null`;
# parse it into ruby hashes
@parsed = JSON.parse(@json)
retcode = 0


# return the nodes hash element
def get_hostnames
nodeshost = @parsed["nodes"]
return nodeshost
end

# return the routing_nodes->nodes hash element
def get_isprimary
nodesprimary = @parsed["routing_nodes"]["nodes"]
return nodesprimary
end

# construct new hash (hosts) to link the data elements from the
# previous 2 hashes together
#
# new hash looks like
#                    array
# encrypted name -> |-- hostname (string)
#                -> |-- isPrimaryNode? (0/1) (boolean)

hosts = Hash.new


# somenewvar is the "nodes" hash element from the original JSON
somenewvar = get_hostnames
# search through and grab the hostname
somenewvar.each do |key,value|
hosts[key] = Array.new
hosts[key][0] = somenewvar[key]["name"]
end

# othernewvar is the "routing_nodes->nodes" hash element from original JSOn
othernewvar = get_isprimary
# search through and get the is "primary" value (yes/no)
othernewvar.each do |key,value|
othernewvar[key].each do |key2, value2|
if key2["primary"]
hosts[key][1] = 1
next
else
hosts[key][1] = 0
next
end
end
end

# Print out new hash created
# puts "---final arry---\n"
# hosts.each do |key,value|
# puts "----#{key}\n"
# puts "\t#{value}\n"
# end


# Check if my host is listed as primary in the new hash
hosts.each do |key,value|
# is primary?
if hosts[key][0] =~ /^#{myhostname}/ && hosts[key][1].to_int
print "I am Primary. (#{hosts[key][0]})\n"
retcode = 1
next

elseif hosts[key][1].to_int
print "#{hosts[key][0]} is Primary."

else
print "#{hosts[key][0]} not Primary.\n"
#print "***" + hosts[key][0] + " does not match #{myhostname}\n"

end
end

# 1 == I'm primary, 0 == seondary
print "#{retcode}\n"
retcode

httpd status code / health check

#!/usr/bin/perl -w
use strict;

# Matt Feenstra 09/30/13

my $result = `/usr/bin/curl -IL -m 10 --write-out \"\%{http_code}\" --silent --output /dev/null $ARGV[0]`;

if($result =~ /(\d{1,3})/) {
        if($1 == 0) {                           print "Error, httpd not working: $ARGV[0]\n"; exit(1); }
        if($1 < 200 && $1 > 0) {       print "informational: $result\n"; exit(0); }
        if($1 >= 200 && $1 < 300) { print "Success: $result\n"; exit(0); }
        if($1 >= 300 && $1 < 400) { print "redirected: $result\n"; exit(0); }
        if($1 >= 400 && $1 < 500) { print "client error: $result\n"; exit(1); }
        if($1 >= 500 && $1 < 600) { print "server error: $result\n"; exit(1); }
}
else { print "unknown result: $result\n"; exit(1); }

Chaotic Trajectory decomposition into periodic loops (MATLAB, theoretical physics)

The following is a recursive algorithm that strips the fundamental walks from a chaotic transition matrix.




%%%%%%%%%%%%%%%%%%%%
% fundamentals.m
% Matt Feenstra
%%%%%%%%%%%%%%%%%%%%

% This recursive function returns a cell array of routes that are the
% fundamental routes for a given transition matrix.

% Parameters: starting vertex (usually 1) or route of vertices, and
%             the transition matrix



function all_routes = fundamentals(route, Tmatrix)

if(~exist('route') || ~exist('Tmatrix'))
    error 'Incorrect syntax,  fundamentals(starting node, transition matrix)';
end  


all_routes = {};
[rows,cols] = size(Tmatrix);
vertex = route(end);
all_adjacencies = find(Tmatrix(:, vertex));



for adjacency = all_adjacencies'
     
        % is this adjacency the first vertex in the route? if so then
        % we're done with this route.  Add route to cell array and exit.

   
        if(route(1) == adjacency)
         
            all_routes(end+1) = {route};
            continue;
        end

        % is this adjacency some other part of the route already?  if so
        % then skip it
     
        if(is_repeat(adjacency, route) || adjacency < route(1) )
            continue;
     
        else

        % then its not repeated and adjacency is not in the route? get
        % the routes from here and add those to the end of the cell array
            all_routes_temp = fundamentals([route, adjacency], Tmatrix);
            all_routes = [all_routes; all_routes_temp];
        end

end




% has this vertex already been listed in this route?
function decision = is_repeat(vertex, my_route)

[rows,cols] = size(my_route);
decision = 0;

for i = 1:1:cols

    if(vertex <= 0)
        error('zero or negative vertex')
    end
 
    % are we repeating?
    if(my_route(i) == vertex)
        decision = 1;
        break;
    end
end




%%%%%% end of program








%%%%%%%%%%%%%%%%%%%%
% get_partitions.m
% Matt Feenstra
%%%%%%%%%%%%%%%%%%%%

function all_partitions = get_partitions(all_funds, graph)

all_partitions = {};
num_funds = length(all_funds);

for n = 1:num_funds

    fun_cycle = all_funds{n};

    % is fun_cycle in this graph?

    % if not, continue to the next fun_cycle
    if(fun_exist(fun_cycle, graph))

        % if cycle exists in graph
        recurs_graph = subtract_cycle_from_graph(fun_cycle, graph);
        recurs_all_funds = all_funds(n:end);

        % if not empty
       
        if any(any((recurs_graph)))
            % recurse_graph(:) to collapse
        %if any(recurs_graph)
       

            recurs_all_partitions = get_partitions(recurs_all_funds, recurs_graph);

            all_partitions_temp = prefix_partitions_with_cycle(fun_cycle, recurs_all_partitions);
           
           
        else

            all_partitions_temp = {{fun_cycle}};
        end
       
        all_partitions = [all_partitions; all_partitions_temp];

    end

end



% remove fundamental cycles prior to n from list
function sublist = subtract_previous_cycles(n, all_funds)

sublist = {};
len_all_funds = length(all_funds);

i = 0;

for j = n:1:len_all_funds
    sublist(i+1) = all_funds(j);
    i = i + 1;
end

sublist = sublist';



% does this fundamental cycle exist in the graph?
function proceed = fun_exist(fun_cycle, graph_matrix)

proceed = 1;
fun_cycle(end+1) = fun_cycle(1);
for ind = 1:length(fun_cycle) - 1
    proceed = proceed*graph_matrix(fun_cycle(ind+1),fun_cycle(ind));
end



% remove the fundamental cycle from the graph
function graph = subtract_cycle_from_graph(fun_cycle, graph)
% we know its in here because fun_exist said so


cycle_len = length(fun_cycle);

for ind = 1:cycle_len
   
    % then we are at the end of this cycle
    if(ind == cycle_len)
        graph(fun_cycle(1), fun_cycle(cycle_len)) = graph(fun_cycle(1), fun_cycle(cycle_len)) - 1;
    else
        graph(fun_cycle(ind+1), fun_cycle(ind)) = graph(fun_cycle(ind+1), fun_cycle(ind)) - 1;
    end


end



% this function inserts this fundamental cycle at the beginning
% of each partition in the list

function all_partitions = prefix_partitions_with_cycle(cycle, all_partitions)


for ind = 1:length(all_partitions)

    all_partitions{ind} = [cycle, all_partitions{ind}];
end

Remote SSH admin on Linux with Net::SSH::Perl

#!/usr/bin/perl -w
##########################################################################
# Filename:             probe.pm
# Version:              2.1
# Date:                 06/01/2012
# Author(s):            Matt Feenstra
# Last Updated by:      <>
#
# INFO:         This package contains the modules necessary to connect to
#               and perform remote SSH commands on remote hosts with logging.
#
# SUBROUTINES:
# probe()
# probe_mq()
# probe_disk()
# probe_mq_maxchl()
#
# USAGE:
#
# NOTES:
# Modified 07/20/12
# Matt Feenstra
#
##########################################################################

use strict;
package probe;

use Net::SSH::Perl;

my($CONFIG_FILE, $HOSTS_FILE, @CONFIG_KEYS, @CONFIG_VALUES);


sub init {
my $argz = shift;
probe::check_params($argz);
probe::set_config();
probe::log("[INFO] BEGIN $0");
}


sub set_config {
#$CONFIG_FILE = shift;
if(!(-T $CONFIG_FILE)) { die "$CONFIG_FILE is not a text file. $!"; }
@CONFIG_KEYS = ();
@CONFIG_VALUES = ();
        open(CFGFILE, "<$CONFIG_FILE") || die "could not open $CONFIG_FILE. $!";

        while(<CFGFILE>) {
                if($_ =~ /^([\w\d\_\-\.]+)\s+(.+)\n$/) {
my($key,$val) = ($1, $2);
chomp($key);
chomp($val);
                        push(@CONFIG_KEYS, $1);
                        push(@CONFIG_VALUES, $2);
                }
        }
        close(CFGFILE);
}

sub get_value {
        my $keyname = shift;
        my $numkeys = @CONFIG_KEYS;

        for(my $i = 0; $i < $numkeys; $i++) {
                if($CONFIG_KEYS[$i] =~ /^$keyname/) {
my $retvalue = $CONFIG_VALUES[$i];
if($keyname =~ /runCommand/) {
splice(@CONFIG_VALUES, $i, 1);
splice(@CONFIG_KEYS, $i, 1);
}
                        return($retvalue);
                }
        }
        #die("CONFIG KEY NOT FOUND: $keyname $!");
}

sub print_config {

        if(@CONFIG_KEYS) {
                my $numkeys = @CONFIG_KEYS;
                print "config file: $CONFIG_FILE\n";
                for(my $i = 0; $i < $numkeys; $i++) {
                        print "index $i:\tkey: \"$CONFIG_KEYS[$i]\"\tvalue: \"$CONFIG_VALUES[$i]\"\n";
                }
        }

        else {

                print "CONFIG_KEYS not defined\n";
        }
}



sub check_params {

my $paramref = shift;
        my @params = @$paramref;
my $paramsize = @params;
if($paramsize != 2) {
die "incorrect number of parameters $paramsize\nusage:\n\t\t$0 <config file> <hosts file>\n";
}
if( (-e $params[0]) && (-T $params[0]) && (-e $params[1]) && (-T $params[1]) ) {
$CONFIG_FILE = $params[0];
$HOSTS_FILE = $params[1];
}
else {
die "invalid config or hosts file ($params[0], $params[1])";
}
}

# REQUIRES: hosts file name
sub get_urls {
        my $urlsfile = $HOSTS_FILE;
        print "reading hosts file $urlsfile..\n";
        probe::log("[INFO] reading hosts file $urlsfile");
        open(URLLIST, "<$urlsfile") || die "$! $urlsfile";
        my @url_list = <URLLIST>;
        close(URLLIST);
        chomp(@url_list);
        my (@username, @hostname, @port);
        my $num_lines = @url_list;
        for(my $i = 0; $i < $num_lines; $i++) {
                if(!($url_list[$i] =~ /^#/) && $url_list[$i] =~ /^(.+)?@(.+)?\:(\d+)?$/) {

#print "\t\tget_urls - $1 - $2 - $3\n";
                        push(@username, $1);
                        push(@hostname, $2);
                        push(@port, $3);
                }
                else {
                        my $linenum = $i + 1;
                        print "ignoring line $linenum in $urlsfile\n";
                }
        }


        return(\@username, \@hostname, \@port);
}



sub connect {

        my $user = $_[0];
        my $host = $_[1];
        my $port = $_[2];
        my $cmdref =  $_[3];

        my @cmds = @$cmdref;

my $connect_timeout = probe::get_value("connectTimeout");
my $private_key = probe::get_value("privateKey");
my $login_timeout = probe::get_value("loginTimeout");
my @priv_key_ary;
$priv_key_ary[0] = $private_key;


print "*** creating SSH handle for $user\@$host:$port\n";
probe::log("[INFO] creating SSH handle for $user\@$host:$port");

        my %params = (  priviledge      => '0',
                        #protocol       => '2,1',
                        protocol        => '2',
                        ciphers         => 'RC4',
                        port            => $port,
                        identity_files  => \@priv_key_ary,
                        #debug          => 1,
                        #interactive    => 1,
                        );

        my $ssh;
        my $ret = 0;

$SIG{ALRM} = \&timed_out;

        eval {
alarm($connect_timeout);
                $ssh = Net::SSH::Perl->new($host, %params);
alarm(0);
};

if($@) {

chomp($@);
print "[ERROR] connect failure on $host as $user because $@ Reason: $!\n";
probe::log("[ERROR] connect failure on $host as $user because $@ Reason: $!");
die "$@, $!";
}

eval {
print "*** logging in as $user ...\n";
alarm($login_timeout);
                $ret = $ssh->login($user);
alarm(0);
};

if($@) {

chomp($@);
print "[ERROR] login failure for $user on $host because $@ Reason: $!\n";
probe::log("[ERROR] login failure for $user on $host because $@ Reason: $!");
die "$@, $!";
}

if($ret) {
print "*** login complete.\n";
        probe::log("[INFO] return on ssh->login($user) $ret host $host");
        }
else {
print "*** login problem, return $ret.\n";
probe::log("[WARNING] login problem, return $ret");
}

        ### Logged in -- Run commands here

        my(@stdouts, @stderrs, @returns);
        my $numcmds = @cmds;

        # time to wait for getting hostname
        my $hostname_timeout = probe::get_value("hostnameTimeout");
$hostname_timeout = $hostname_timeout * 1;

        # assign alarm signal to the timed_out function so we can timeout on commands

        print "*** querying hostname ..\n";
#my $hostname = "unknown";
        my($hostname, $stderr2, $return2);

        eval {
                alarm($hostname_timeout);
                ($hostname, $stderr2, $return2) = $ssh->cmd("hostname");
                alarm(0);
        };

if($@) {
chomp($@);
print "[ERROR] could not retrieve hostname for $host: $@, $!\n";
probe::log("[ERROR] could not retrieve hostname for $host: $@, $!");
die "could not retrieve hostname for $host";
}

if(defined($hostname)) { chomp($hostname); }
if(!defined($hostname)) { $hostname = "unknown"; }

        # run each of the commands in the cmds array

        # seconds to wait for command to return
        my $timeout_secs = probe::get_value("commandTimeout");
$timeout_secs = $timeout_secs * 1;

        for(my $i = 0; $i < $numcmds; $i++) {

                print "*** remote executing -> $cmds[$i]\n";

                eval {
                        alarm($timeout_secs);
                        ($stdouts[$i],$stderrs[$i],$returns[$i]) = $ssh->cmd("$cmds[$i]");
                        alarm(0);
                };

                if($@ =~ /TIMEOUT/) {
                        print("[WARNING] [$hostname] execution of \"$cmds[$i]\" left to run after $timeout_secs secs, moving on\n");
                        probe::log("[WARNING] [$hostname] execution of \"$cmds[$i]\" left to run after $timeout_secs secs, moving on");
                }
elsif($@ && $!) {
chomp($@);
print "[ERROR] problem executing \"$cmds[$i]\" $@, $!\n";
probe::log("[ERROR] problem executing \"$cmds[$i]\" $@, $!");
}
                elsif($@) {
chomp($@);
                        print("[WARNING] [$hostname] execution of \"$cmds[$i]\" incomplete\n");
                        probe::log("[WARNING] [$hostname] execution of \"$cmds[$i]\" incomplete");
                }

        }
        undef $ssh;
        return($hostname, \@stdouts, \@stderrs, \@returns);
}

sub timed_out {
        die "TIMEOUT";
}

sub probe {
my @cmds = ();
while(my $cmd_value = probe::get_value("runCommand")) {
push(@cmds, $cmd_value);
undef $cmd_value;
}
if(@cmds == 0) { die "[ERROR] no runCommand(s) listed in $CONFIG_FILE"; }
        # Read in a urls file and fill the arrays with host connect info
        my($userref, $hostref, $portref) = probe::get_urls($HOSTS_FILE);
        my @users = @$userref;
        my @hosts = @$hostref;
        my @ports = @$portref;
        my $num_hosts = @hosts;
my(@stdouts, @stderrs, @returns);
        for(my $i = 0; $i < $num_hosts; $i++) {
                my $linenum = $i + 1;
probe::log("---");
print "\n---\n";
                probe::log("[INFO] STARTING ($HOSTS_FILE) [ line: $linenum ] $hosts[$i] port $ports[$i] as $users[$i]");
                print("*** ($HOSTS_FILE) [ line: $linenum ] connecting to host $hosts[$i]\n");
                my($host, $ref1, $ref2, $ref3);
                eval {
                        ($host, $ref1, $ref2, $ref3) = probe::connect($users[$i],
                                                        $hosts[$i], $ports[$i], \@cmds);
                };
                if($@) {
# the connect subroutine should have printed the [ERROR]
chomp($@);
                        print "[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@";
                        probe::log("[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@");
                }
                # we have connected ok
my(@stdouts, @stderrs, @returns) = ();

                if(defined($host) && $host ne "unknown") {
                        print("*** found hostname: $host\n\n");
                        if(defined(@$ref1)) {
@stdouts = @$ref1;
                        @stderrs = @$ref2;
                        @returns = @$ref3;
}




                        my $num_outputs = @returns;
                        for(my $y = 0; $y < $num_outputs; $y++) {
                                if(defined($stdouts[$y])) { print "\n### stdout ($cmds[$y]) (ret $returns[$y]):\n\n$stdouts[$y]\n"; }
                                if(defined($stderrs[$y])) { print "+++ stderr:\n$stderrs[$y]\n+++\n"; }
                                probe::log("[INFO] [$host] [COMMAND] [$cmds[$y]]");
                                probe::log("[INFO] [$host] [Return Code] [$returns[$y]]");
                                if(defined($stdouts[$y])) {
                                        my @stdlines = split(/\n/,$stdouts[$y]);
                                        foreach(@stdlines) {
                                                probe::log("[INFO] [$host] [stdout] $_"); }
                                }
                                else {
                                        probe::log("[INFO] [$host] [stdout] NONE");
                                }
                                # add some check logic here
# we are at $hosts[$i]
                                if(defined($stderrs[$y])) {
                                        my @stderrlines = split(/\n/, $stderrs[$y]);
                                        foreach(@stderrlines) {
                                                probe::log("[WARNING] [$host] [stderr] $_"); }
                                }
                        }
                } # end if
        } # end for
}




sub probe_mq {
        my @cmds = ();
        while(my $cmd_value = probe::get_value("runCommand")) {
                push(@cmds, $cmd_value);
                undef $cmd_value;
        }
        if(@cmds == 0) { die "[ERROR] no runCommand(s) listed in $CONFIG_FILE"; }
        # Read in a urls file and fill the arrays with host connect info
        my($userref, $hostref, $portref) = probe::get_urls($HOSTS_FILE);
        my @users = @$userref;
        my @hosts = @$hostref;
        my @ports = @$portref;
        my $num_hosts = @hosts;
        my(@stdouts, @stderrs, @returns);
        for(my $i = 0; $i < $num_hosts; $i++) {
                my $linenum = $i + 1;
                probe::log("---");
                print "\n---\n";
                probe::log("[INFO] STARTING ($HOSTS_FILE) [ line: $linenum ] $hosts[$i] port $ports[$i] as $users[$i]");
                print("*** ($HOSTS_FILE) [ line: $linenum ] connecting to host $hosts[$i]\n");
                my($host, $ref1, $ref2, $ref3);
                eval {
                        ($host, $ref1, $ref2, $ref3) = probe::connect($users[$i],
                                                        $hosts[$i], $ports[$i], \@cmds);
                };
                if($@) {
chomp($@);
                        print "[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@";
                        probe::log("[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@");
                }
                # we have connected ok
                if(defined($host) && $host ne "unknown") {
                        print("connect complete, found host: $host\n");
                        my @stdouts = @$ref1;
                        my @stderrs = @$ref2;
                        my @returns = @$ref3;
                        my $num_outputs = @returns;

                        for(my $y = 0; $y < $num_outputs; $y++) {
                                if(defined($stdouts[$y])) { print "\n### stdout ($cmds[$y]) (ret $returns[$y]):\n\n$stdouts[$y]\n"; }
                                if(defined($stderrs[$y])) { print "+++ stderr:\n$stderrs[$y]\n+++\n"; }
                                probe::log("[INFO] [$host] [COMMAND] [$cmds[$y]]");
                                probe::log("[INFO] [$host] [Return Code] [$returns[$y]]");
                                if(defined($stdouts[$y])) {
                                        my @stdlines = split(/\n/,$stdouts[$y]);
                                        foreach(@stdlines) {
                                                probe::log("[INFO] [$host] [stdout] $_"); }
                                }
                                else {
                                        probe::log("[INFO] [$host] [stdout] NONE");
                                }
                                # add some check logic here
# we are at $hosts[$i]
check_mq_chs($host, $stdouts[$y]);
                                if(defined($stderrs[$y])) {
                                        my @stderrlines = split(/\n/, $stderrs[$y]);
                                        foreach(@stderrlines) {
                                                probe::log("[WARNING] [$host] [stderr] $_"); }
                                }
                        }
                } # end if
        } # end for
}


sub check_mq_chs {

        my $host = shift;
        my $stdout = shift;
        my @lines;
        if(defined($stdout)) { @lines = split(/\n/,$stdout); }

        my $num_lines = @lines;

        for(my $i = 0; $i < $num_lines; $i++) {

                #my($channel, $type, $conname, $port, $rqmname, $status, $substate);
                my $channel = "null";
                my $type = "null";
                my $conname = "null";
                my $port = "1414";
                my $rqmname = "null";
                my $status = "null";
                my $substate = "null";

                # easy case -- no channels running
                if($lines[$i] =~ /^AMQ8420/) {
                        print "[$host] [RESULT] ok - channels inactive\n";
                        probe::log("[INFO] [$host] [RESULT] OK - channels inactive");
                }

                # need to get channel report blocks starting with channel name line

                if($lines[$i] =~ /CHANNEL\((.+)?\).+?CHLTYPE\((.+)?\)/) {
#print "found\tchannel\t$1\tchltype\t$2\n";

                        $channel = $1;
                        $type = $2;

                        if($lines[$i+1] =~ /CONNAME\((.+)?\((.+)?\)\)/ ||
$lines[$i+1] =~ /CONNAME\((.+)?\)/ ) {

                                $conname = $1;
                                if($2) { $port = $2; }

#print "\tfound\tconname\t$1\tport\t$port\n";

                                if($lines[$i+2] =~ /RQMNAME\((.+)?\).+STATUS\((.+)?\)/) {
#print "\t\tfound\trqmname\t$1\tstatus\t$2\n";

                                        $rqmname = $1;
                                        $status = $2

                                        }
                                        if($lines[$i+3] =~ /SUBSTATE\((.+)?\).+XMITQ/) {
#print "\t\t\tfound\tsubstate\t$1\n";

                                                $substate = $1;
                                        }

                                        if($status =~ /RUNNING/) {
                                                probe::log("[INFO] [$host] [RESULT] OK - $channel is $status");
                                                print "[INFO] [$host] [RESULT] ok - $channel is $status\n";
                                        }
                                        else {
                                                probe::log("[WARNING] [$host] [RESULT] Warning - $channel is $status in $substate");
                                                print "[WARNING] [$host] [RESULT] warning - $channel is $status in $substate\n";
                                        }
                                        if($status =~ /RETRYING/) {
                                                probe::log("[ERROR] [$host] [RESULT] Error - $channel to $conname:$port is $status");
                                                print "[ERROR] [$host] [RESULT] Error - $channel to $conname:$port is $status\n";
                                        }
                        }

                }


        }

}


sub probe_disk {
        my @cmds = ();
        while(my $cmd_value = probe::get_value("runCommand")) {
                push(@cmds, $cmd_value);
                undef $cmd_value;
        }
        if(@cmds == 0) { die "[ERROR] no runCommand(s) listed in $CONFIG_FILE"; }
        # Read in a urls file and fill the arrays with host connect info
        my($userref, $hostref, $portref) = probe::get_urls($HOSTS_FILE);
        my @users = @$userref;
        my @hosts = @$hostref;
        my @ports = @$portref;
        my $num_hosts = @hosts;
        my(@stdouts, @stderrs, @returns);
        for(my $i = 0; $i < $num_hosts; $i++) {
                my $linenum = $i + 1;
                probe::log("---");
                print "\n---\n";
                probe::log("[INFO] STARTING ($HOSTS_FILE) [ line: $linenum ] $hosts[$i] port $ports[$i] as $users[$i]");
                print("*** ($HOSTS_FILE) [ line: $linenum ] connecting to host $hosts[$i]\n");
                my($host, $ref1, $ref2, $ref3);
                eval {
                        ($host, $ref1, $ref2, $ref3) = probe::connect($users[$i],
                                                        $hosts[$i], $ports[$i], \@cmds);
                };
                if($@) {
chomp($@);
                        print "[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@";
                        probe::log("[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@");
                }
                # we have connected ok
                my(@stdouts, @stderrs, @returns) = ();

                if(defined($host) && $host ne "unknown") {
                        print("*** found hostname: $host\n\n");
                        if(defined(@$ref1)) {
                                @stdouts = @$ref1;
                                @stderrs = @$ref2;
                                @returns = @$ref3;
                        }




                        my $num_outputs = @returns;
                        for(my $y = 0; $y < $num_outputs; $y++) {
                                if(defined($stdouts[$y])) { print "\n### stdout ($cmds[$y]) (ret $returns[$y]):\n\n$stdouts[$y]\n"; }
                                if(defined($stderrs[$y])) { print "+++ stderr:\n$stderrs[$y]\n+++\n"; }
                                probe::log("[INFO] [$host] [COMMAND] [$cmds[$y]]");
                                probe::log("[INFO] [$host] [Return Code] [$returns[$y]]");
                                if(defined($stdouts[$y])) {
                                        my @stdlines = split(/\n/,$stdouts[$y]);
                                        foreach(@stdlines) {
                                                probe::log("[INFO] [$host] [stdout] $_"); }
                                }
                                else {
                                        probe::log("[INFO] [$host] [stdout] NONE");
                                }
                                # add some check logic here
check_disk($host, $hosts[$i], $stdouts[$y]);
                                if(defined($stderrs[$y])) {
                                        my @stderrlines = split(/\n/, $stderrs[$y]);
                                        foreach(@stderrlines) {
                                                probe::log("[WARNING] [$host] [stderr] $_"); }
                                }
                        }
                } # end if
        } # end for
}


sub check_disk {
        # anything less than this amount free will generate an [ERROR].  Value should be 49 pct or less
        my $pct_threshold = probe::get_value("diskcheckPctThreshold");

        # if we found an error we change the return code to 1 (true)
        my $retcode = 0;

        my $host = shift;
        my $ipname = shift;
        my $stdout = shift;
        my @lines;
        if(defined($stdout)) { @lines = split(/\n/,$stdout); }
        my $num_lines = @lines;
        for(my $i = 0; $i < $num_lines; $i++) {
                if(!($lines[$i] =~ /^Filesystem/) && $lines[$i] =~ /(\S+)?\s+(\S+)?\s+(\S+)?\s+(\S+)?\s+(\S+)?\s+(\S+)/) {
                        my($filesystem, $size, $used, $avail, $use, $mounted) = ($1, $2, $3, $4, $5, $6);
                        my($pct_used, $pct_free);
                        if($use =~ /(\d+)?\%/) {
                                $pct_used = $1;
                        }
                        $pct_free = 100 - $pct_used;
                        if(!defined($pct_free)) {
print "[WARNING] [$host] [RESULT] disk percent free not calculated, undefined\n";
                                probe::log("[WARNING] [$host] [RESULT] disk percent free not calculated, undefined");
                        }
                        if($pct_free > 50) {
print "[INFO] [$host] [RESULT] disk ok, free space $pct_free% on $mounted\n";
                                probe::log("[INFO] [$host] [RESULT] disk ok, free space $pct_free% on $mounted");
                        }
                        if(($pct_free <= 50) && ($pct_free >= $pct_threshold)) {
print "[WARNING] [$host] [RESULT] host $ipname disk in trouble, only $pct_free% available on $mounted\n";
                                probe::log("[WARNING] [$host] [RESULT] host $ipname disk in trouble, only $pct_free% available on $mounted");
                        }
                        if($pct_free < $pct_threshold) {
print "[ERROR] [$host] [RESULT] host $ipname disk is filling up! $pct_free% available on $mounted\n";
                                probe::log("[ERROR] [$host] [RESULT] host $ipname disk is filling up! $pct_free% available on $mounted");
                                $retcode++;
                        }
                }

        }
        return $retcode;
}

sub probe_mq_maxchl {
        my @cmds = ();
        while(my $cmd_value = probe::get_value("runCommand")) {
                push(@cmds, $cmd_value);
                undef $cmd_value;
        }
        if(@cmds == 0) { die "[ERROR] no runCommand(s) listed in $CONFIG_FILE"; }
        # Read in a urls file and fill the arrays with host connect info
        my($userref, $hostref, $portref) = probe::get_urls($HOSTS_FILE);
        my @users = @$userref;
        my @hosts = @$hostref;
        my @ports = @$portref;
        my $num_hosts = @hosts;
        my(@stdouts, @stderrs, @returns);
        for(my $i = 0; $i < $num_hosts; $i++) {
                my $linenum = $i + 1;
                probe::log("---");
                print "\n---\n";
                probe::log("[INFO] STARTING ($HOSTS_FILE) [ line: $linenum ] $hosts[$i] port $ports[$i] as $users[$i]");
                print("*** ($HOSTS_FILE) [ line: $linenum ] connecting to host $hosts[$i]\n");
                my($host, $ref1, $ref2, $ref3);
                eval {
                        ($host, $ref1, $ref2, $ref3) = probe::connect($users[$i],
                                                        $hosts[$i], $ports[$i], \@cmds);
                };
                if($@) {
                        # the connect subroutine should have printed the [ERROR]
                        chomp($@);
                        print "[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@";
                        probe::log("[WARNING] unable to connect to $hosts[$i]:$ports[$i] because $@");
                }
                # we have connected ok
                my(@stdouts, @stderrs, @returns) = ();

                if(defined($host) && $host ne "unknown") {
                        print("*** found hostname: $host\n\n");
                        if(defined(@$ref1)) {
                                @stdouts = @$ref1;
                                @stderrs = @$ref2;
                                @returns = @$ref3;
                        }

                        my $num_outputs = @returns;
                        for(my $y = 0; $y < $num_outputs; $y++) {
                                if(defined($stdouts[$y])) { print "\n### stdout ($cmds[$y]) (ret $returns[$y]):\n\n$stdouts[$y]\n"; }
                                if(defined($stderrs[$y])) { print "+++ stderr:\n$stderrs[$y]\n+++\n"; }
                                probe::log("[INFO] [$host] [COMMAND] [$cmds[$y]]");
                                probe::log("[INFO] [$host] [Return Code] [$returns[$y]]");
                                if(defined($stdouts[$y])) {
                                        my @stdlines = split(/\n/,$stdouts[$y]);
                                        foreach(@stdlines) {
                                                probe::log("[INFO] [$host] [stdout] $_"); }
                                }
                                else {
                                        probe::log("[INFO] [$host] [stdout] NONE");
                                }
                                # add some check logic here
                                # we are at $hosts[$i]

                                check_mq_maxchl($host, $hosts[$i], $stdouts[$y]);

                                if(defined($stderrs[$y])) {
                                        my @stderrlines = split(/\n/, $stderrs[$y]);
                                        foreach(@stderrlines) {
                                                probe::log("[WARNING] [$host] [stderr] $_"); }
                                }
                        }
                } # end if
        } # end for
}


sub check_mq_maxchl {

        # if we found an error we change the return code to 1 (true)

        my $host = shift;
        my $ipname = shift;
        my $stdout = shift;
        my @lines;
my $retcode = 0;
        if(defined($stdout)) { @lines = split(/\n/,$stdout); }
        my $times = $lines[0] * 1;
if($times > 0) {
print "[ERROR] [$host] [RESULT] Max number of channels reached $times times on $ipname\n";
probe::log("[ERROR] [$host] [RESULT] Max number of channels reached $times times on $ipname");
$retcode++;
}
else {
print "[INFO] [$host] [RESULT] MQ channel count on $host is OK.\n";
probe::log("[INFO] [$host] [RESULT] MQ channel count on $host is OK.");
}
        return $retcode;
}




sub log {

        my $text = shift;
        my ($logsec, $logmin, $loghour, $logmday, $logmon, $logyear, $logwday, $logyday, $logisdst)=localtime(time);
        my $timestamp = sprintf("[%02d-%02d-%4d %02d:%02d:%02d]",$logmon+1,$logmday,$logyear+1900,$loghour,$logmin,$logsec);
        $logmon++;
my $logdir = probe::get_value("logDirectory");
my $logpostfix = probe::get_value("logPostfix");
$logyear = $logyear + 1900;
        my $logfile="$logdir\/$logmon-$logmday-$logyear-$logpostfix.log";
        eval {
                open(LOGOUT, ">>$logfile") || die "$logfile";
        };
        if($@) {
chomp($@);
                print "[ERROR] could not write logfile \"$logfile\" $!\n";
                print "[WARNING] log text was \"$text\"\n";
        }
else {
my $latest_symlink = "$logdir" . "/" . "$logpostfix.latest";
system("rm -rf $latest_symlink 1>>/dev/null 2>>/dev/null; ln -s $logfile $latest_symlink 1>>/dev/null 2>>/dev/null");
}
        print LOGOUT "$timestamp $text\n";
        close(LOGOUT);
}





1;

Puppet notes for RHEL 6

Installing Puppet / Puppet Server on RHEL

grab the repositories:

puppet -

[root@localhost ~]# sudo rpm -ivh http://yum.puppetlabs.com/el/6/products/i386/puppetlabs-release-6-7.noarch.rpm

puppet enterprise -

[root@localhost ~]# sudo rpm -ivh http://yum-enterprise.puppetlabs.com/el/6/extras/i386/puppetlabs-enterprise-release-extras-6-2.noarch.rpm

getzippy.pl usage example

#!/usr/bin/perl -w
use strict;

system("rm -rf ./index.html*");
system("wget http://muzeno.pl");
open(IN,"<index.html");
my @zippyurls;

foreach(<IN>) {
        if($_ =~ /a\shref=\"(.+file\.html)?\"\s/) {
                push(@zippyurls, $1);
        }
}
close(IN);

foreach(@zippyurls) {
        system("./getzippy.pl $_");
}

system("rm -rf ./index.html*");

z1ppy sh4re automatic downloader (getzippy.pl)

Disclaimer:  Code is for educational use only.

command line syntax: getzippy.pl http://www13.zippyshare.com/v/83632345/file.html



#!/usr/bin/perl -w
use strict;

my $useragent = "Mozilla/5.0";

if(!defined($ARGV[0])) { die "no args. requires url\n"; }

system("mkdir files 2>/dev/null");

my $zippyurl = $ARGV[0];
chomp($zippyurl);

my $magic_b;
my $url;

system("wget -O zippy.page --keep-session-cookies --save-cookies zippy.cookies \"$zippyurl\"");
print "\n\n";

my @filepage = `cat zippy.page`;

foreach(@filepage) {

        # looks like
        # document.getElementById('dlbutton').href = "/d/83632345/"+(a/a+b+b%10)+"/Deadmau5%20-%20Ghost%20N%20Stuff%20%28Original%20Mix%29.mp3";

        my($prefix, $postfix, $zippyserv);

        if($zippyurl =~ /(http:\/\/ww.+?\.zippyshare\.com)/) {
                $zippyserv = $1;
        }

        if($_ =~ /var\sb\s=\s(.+)?\;/) {
                $magic_b = $1 * 1;
                print "magic b =>\t$magic_b\n";
        }

        #if(!defined($magic_b)) { print "ERROR: couldn't find magic var numbers\n"; }

        if($_ =~ /.+?dlbutton.+?\s=\s\"(.+?)\"(\+.+?)\"(.+?)\"/) {

                $prefix = $1;
                my $magicnumformula = $2;
                $postfix = $3;

                print "prefix ->\t$prefix\n";
                print "formula ->\t$magicnumformula\n";
                print "postfix ->\t$postfix\n\n";

                my $temp1 = 1 + $magic_b;
                my $temp2 = $magic_b % 10;

                my $magicmath = $temp1 + $temp2;

                $url = "$zippyserv" . "$prefix" . "$magicmath" . "$postfix";

                print "\nurl ->\t$url\ngetting file..\n\n";

                system("mkdir ./files 2>>/dev/null");
                system("wget --directory-prefix=./files --load-cookies ./zippy.cookies --user-agent \"$useragent\" $url");

                print "\n";


        }
}



# cleanup temp files
system("rm zippy.cookies");
system("rm zippy.page");

Installing Chef Server on CentOS 5 / 6

Requires bootstrap method, starting with chef-solo / chef-client

First steps, system prep

vi /etc/sysconfig/network

vi /etc/selinux/config

chkconfig iptables off

/etc/init.d/iptables stop

(reboot)

vi /etc/hosts – add ip

yum update

rpm -Uvh http://rbel.frameos.org/rbel5

wget -O /etc/yum.repos.d/aegisco.repo http://rpm.aegisco.com/aegisco/el5/aegisco.repo

rpm -Uvh http://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm

or

rpm -Uvh http://rbel.frameos.org/rbel6

rpm -Uvh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm

yum install java-1.6.0-openjdk gecode-devel telnet ruby rubygems ruby-devel ruby-ri ruby-rdoc ruby-shadow gcc gcc-c++ automake autoconf make curl dmidecode

curl -O http://production.cf.rubygems.org/rubygems/rubygems-1.8.10.tgz

tar -zxvf rubygems-1.8.10.tgz

ruby setup.rb --no-format-executable

Second steps, install chef client and chef solo

gem install chef –no-ri –no-rdoc

Create chef-solo (bootstrap)

mkdir /etc/chef

vi /etc/chef/solo.rb

file_cache_path "/tmp/chef-solo"

cookbook_path "/tmp/chef-solo/cookbooks"

vi /etc/chef/chef.json

{

  "chef_server": {

    "server_url": "http://localhost:4000",

    "webui_enabled": true

  },

  "run_list": [ "recipe[chef-server::rubygems-install]" ]

}

If you find the error message

  /usr/lib/ruby/1.8/openssl/cipher.rb:22: Cipher is not a module (TypeError)

  then download the ruby source files (for your version!) from e.g., 

  ftp://ftp.ruby-lang.org/pub/ruby/1.8/ruby-1.8.7-p352.tar.bz2, 

  untar it and change dir to ruby-XXXXX/ext/openssl. There, run 

  ruby extconf.rb, 

  make,

  and make install. 

Afterwards, the bootstrap should overcome this error.

Running into library dependency errors on running chef-solo for CentOS 5 ...

yum install openssl-devel

yum install glibc glibc-devel glibc-headers

#yum install gcc44
#yum install gcc44-c++

export LD_LIBRARY_PATH=/usr/local/lib

(configure, make, make install - in order)

ftp://gcc.gnu.org/pub/gcc/infrastructure/gmp-4.3.2.tar.bz2

ftp://gcc.gnu.org/pub/gcc/infrastructure/mpfr-2.4.2.tar.bz2

ftp://gcc.gnu.org/pub/gcc/infrastructure/mpc-0.8.1.tar.gz

ftp://gcc.gnu.org/pub/gcc/releases/gcc-4.7.2/gcc-4.7.2.tar.gz

chef-solo -c /etc/chef/solo.rb -j /etc/chef/chef.json -r http://s3.amazonaws.com/chef-solo/bootstrap-latest.tar.gz

Known CentOS error ??


Recipe: gecode::source
  * execute[ldconfig] action run
    - execute ldconfig

[2013-03-09T14:01:15-08:00] ERROR: Running exception handlers
[2013-03-09T14:01:15-08:00] ERROR: Exception handlers complete
Chef Client failed. 6 resources updated
[2013-03-09T14:01:15-08:00] FATAL: Stacktrace dumped to /tmp/chef-solo/chef-stacktrace.out
[2013-03-09T14:01:15-08:00] FATAL: NoMethodError: gem_package[chef-server-api] (chef-server::rubygems-install line 83) had an error: NoMethodError: undefined method `full_name' for nil:NilClass

Try -

gem install chef-server

Throws -


(::) (::) (::) (::) (::) (::) (::) (::) (::) (::) (::) (::) (::) (::) (::)
Fetching: merb-assets-1.1.3.gem (100%)
Fetching: merb-helpers-1.1.3.gem (100%)
Fetching: merb-param-protection-1.1.3.gem (100%)
Fetching: dep_selector-0.0.8.gem (100%)
Building native extensions.  This could take a while...
ERROR:  Error installing chef-server:
        ERROR: Failed to build gem native extension.

        /usr/bin/ruby extconf.rb
checking for main() in -lgecodesearch... no
=========================================================================================
Gecode >3.5 must be installed (http://www.gecode.org/).

OSX:
  brew install gecode

For convenience, we have built Gecode for Debian/Ubuntu (<release> is lucid or maverick):
  Add the following two lines to /etc/apt/sources.list.d/opscode.list:
    deb http://apt.opscode.com <release> main
    deb-src http://apt.opscode.com <release> main
  Then run:
    curl http://apt.opscode.com/packages@opscode.com.gpg.key | sudo apt-key add -
    sudo apt-get update
    sudo apt-get install libgecode-dev

Other distributions can build from source.

Try -

yum install gecode

yum install gecode-devel

Throws -

Package gecode-3.5.0-1.el5.x86_64 already installed and latest version

Package gecode-devel-3.5.0-1.el5.x86_64 already installed and latest version

Try installing from source -

http://www.gecode.org/download/gecode-3.7.3.tar.gz

(configure, make, make install)


Try -

gem install chef-server

Throws -

needs gecode > 3.5

Try -

chef-solo -c /etc/chef/solo.rb -j /etc/chef/chef.json -r http://s3.amazonaws.com/chef-solo/bootstrap-latest.tar.gz


same

Try -

yum erase gecode gecode-devel

cd gecode-3.7.3
make install

gem install chef-server

Verify components installed and running

Name	Listen Port	Example Program Name in ps (Erlang programs truncated)
Chef Server	4000	merb : chef-server (api) : worker (port 4000)
Chef Server WebUI	4040	merb : chef-server-webui : worker (port 4040)
CouchDB	5984	beam.smp -Bd -K true – -root /usr/local/lib/erlang -progname erl – -noshell -noinput -couch_ini /usr/local/etc/couchdb/default.ini /usr/local/etc/couchdb/local.ini -s couch
RabbitMQ	5672	{{beam.smp -W w -K true -A30 – -root /usr/local/lib/erlang -progname erl – -noshell -noinput -s rabbit -sname rabbit -rabbit tcp_listeners [{"0.0.0.0", 5672}]}}
Chef Solr	8983	/usr/bin/java -Xmx250M -Xms250M -Dsolr.data.dir=/opscode/chef/features/data/solr/data -Dsolr.solr.home=/opscode/chef/features/data/solr/home -jar /opscode/chef/features/data/solr/jetty/start.jar
Chef Expander	none	ruby ./chef-solr/bin/chef-expander -c /etc/chef/solr.rb -l debug