GitLab: Clone all the repos!

#!/home/matt_feenstra/.rvm/rubies/ruby-2.3.1/bin/ruby -w
require 'json'
gitlab        = '10.252.1.229'
private_token = 'gog3AkrumotxU2m6wqSP'
###
json_obj = JSON.parse(`curl --header "PRIVATE-TOKEN: #{private_token}" "http://#{gitlab}/api/v3/projects/all?per_page=1000"`)
repos = Array.new
# has format "Group_name/Project_name"
json_obj.each do |record|
repos.push(record['path_with_namespace'])
end
repos.sort!
git_paths = Array.new
groups    = Array.new
# construct git clone command
repos.each do |repo|
groupname, projectname = repo.split('/')
groups.push(groupname)
git_paths.push("git clone git@#{gitlab}:#{groupname}/#{projectname}.git")
end
groups.uniq!
# put them in separate directories per group
groups.each do |dir|
system("mkdir #{dir}")
end
# git clone them all into 'groupname' subdirs
git_paths.each do |clone|
groups.each do |groupname|
# grab the project name
projectname = String
if clone =~ /#{groupname}\/(.*)\.git/ then
projectname = $1
end
# drop into subdirectories and clone
if clone =~ /#{groupname}/ then
system("cd #{groupname} && #{clone}")
# if clone_cookbooks.rb exists, run it for sub-cookbooks
subdir = "./#{groupname}/#{projectname}"
if File.exist?("#{subdir}/clone_cookbooks.rb") then
puts "\tRunning #{subdir}/clone_cookbooks.rb.."
system("cd #{subdir} && ./clone_cookbooks.rb")
end
end
end
end

HOWTO Generate ANSI color on your Linux terminal

#!/bin/bash
#  Put this in a shell script as an example
#  It will print two lines with all the available colors in words
#  matt.a.feenstra@gmail.com


BLACK='\033[0;30m'
RED='\033[0;31m'
GREEN='\033[0;32m'
ORANGE='\033[0;33m'
BLUE='\033[0;34m'
PURP='\033[0;35m'
CYAN='\033[0;36m'
LGRAY='\033[0;37m'

DGRAY='\033[1;30m'
LRED='\033[1;31m'
LGREEN='\033[1;32m'
YELLOW='\033[1;33m'
LBLUE='\033[1;34m'
LPURP='\033[1;35m'
LCYAN='\033[1;36m'
WHITE='\033[1;37m'

NC='\033[0m' # No Color

printf "     ${RED}RED ${GREEN}GREEN ${ORANGE}ORANGE${BLUE}BLUE ${PURP}PURP ${CYAN}CYAN ${LGRAY}LGRAY\n"
printf "${DGRAY}DGRAY${LRED}LRED${LGREEN}LGREEN${YELLOW}YELLOW${LBLUE}LBLUE${LPURP}LPURP${LCYAN}LCYAN${WHITE}WHITE\n"

List your AWS EC2 instances, show the size

#!/home/matt_feenstra/.rvm/rubies/ruby-2.3.1/bin/ruby

puts 'Querying AWS command line..'

instanceids = Array.new
instancetypes = Array.new

output = Array.new
system('aws ec2 describe-instances >output.txt')
rawfile = File.open('output.txt', 'r')
rawfile.each do |l|
        output.push(l)
end

i = 0
output.each do |line|

        if line =~ /InstanceId\": \"(i-........)\",/ then
                instanceids[i] = $1
                print "InstanceID: #{instanceids[i]} "

                # find the next occurance of instancetype and break out
                subarray = output[i+1..output.length]

                subarray.each do |info|
                        if info =~ /InstanceType\": \"(.*)\",/ then
                                instancetypes[i] = $1
                                puts "Type: #{instancetypes[i]}"
                                break
                        end
                end
        end
        i += 1
end

Two Excellent Sites (RegEx and JSON)

I highly reccomend this site for evaulating Regular Expressions

https://regex101.com/

Awesome site for evaluating your JSON

https://www.jsonlint.com


--Matt

New favorite PS1 prompt

# .bash_profile
.
.
.
.
.

PS1='\[\033[0;35m\]\u\[\033[0;31m\]@\[\033[0;32m\]\h\[\033[1;37m\]:\[\033[1;34m\]\w\[\033[1;33m\]\$\[\033[00m\] '

Super handy tool: findwgrep.sh (find with grep)


#!/bin/bash
# findwgrep <word>
# Search through the files from here (./) onward for a word

find ./ -type f -exec grep -Hi "$1" {} 2>>/dev/null \;

How to enable networking for CentOS 7 on Oracle VirtualBox

1. Go to settings->networking for your VM and create a NAT network (using your computer's link)
2. Use 'nmtui' to configure hostname and "active" connection
3. Edit the /etc/sysconfig/network-scripts/ifcfg-enp0s3 and set ONBOOT=yes
4. Reboot!

Ruby - Retrieve stock market data and graph moving averages with gnuplot

#!/usr/bin/ruby
#
# Matt Feenstra 04/11/16
# original work
##########################



require "yahoo-finance"

$TICKER = nil
$maxDaysHistory = 3650   # 10 years default
$start_day = nil


### syntax check

if $ARGV[0] =~ /\w+/ && $ARGV[1] =~ /\d+/ then
$TICKER = $ARGV[0]
$maxDaysHistory = Integer($ARGV[1])
print "max days history is #{$maxDaysHistory}\n"
elsif $ARGV[0] =~ /\w+/ then
$TICKER = $ARGV[0]
else
STDERR.puts "incorrect syntax:\t#{$0} #{$ARGV[0]} #{$ARGV[1]}\n"
abort "correct syntax:\t\t#{$0} <TICKER SYMBOL> <NUMBER of DAYS>"
end


################### functions

def get_data
STDERR.puts "getting data for #{$TICKER} from yahoo finance..\n"
yahoo_client = YahooFinance::Client.new
data_download = yahoo_client.historical_quotes($TICKER)

data = Array.new
$start_day = data_download.reverse[$maxDaysHistory].trade_date
STDERR.puts "starting day 0 on #{$start_day}\n"
i = 0
data_download.reverse.each do |value|
#puts "#{i} #{value.adjusted_close}"

if i < $maxDaysHistory.to_f then
data.push value.adjusted_close
end

i += 1
end # data_download.each
data
end


def get_avg(data_ary, radius)

i = 0
y = 0
fiftyday = Array.new

for num in 0..radius
fiftyday[num] = 0
end

data_ary.each do |price|

if i > (radius - 1) && i < (data_ary.length - radius) then

subary = Array.new
#puts "subary will be data_ary from #{i-radius} to #{i+radius}\n"
subary = data_ary[i-radius..i+radius]
sum = 0
#puts "averaging sub ary indexes #{i-radius} to #{i+radius}\n"
y = 0
subary.each do |pricenum|
#puts "  adding subary(#{y}) value: #{pricenum} to #{sum}\n"
sum = sum + pricenum.to_f
y += 1
end
subary.slice(0)
#puts "sum is #{sum} average is #{sum/(2*radius)}\n"
fiftyday[i] = sum / (2*radius)

end
i += 1
end

fiftyday
end

def make_graph_file(data_ary, filename)

puts "writing #{filename}\n"
`rm -rf #{filename} 1>>/dev/null 2>>/dev/null`
file = File.open(filename, "w")
i = 0
data_ary.each do |value|
file.puts "#{i} #{value}"
i += 1
end
file.close

end

################## main


data = get_data
dataFile = "#{$TICKER}.data"
make_graph_file(data, dataFile)

STDERR.puts "calculating 20 day average graph..\n"
twentyDayAvg = get_avg(data, 10)
twentyDayFile = "#{$TICKER}.20.data"
make_graph_file(twentyDayAvg, twentyDayFile)

STDERR.puts "calculating 50 day average graph..\n"
fiftyDayAvg = get_avg(data, 25)
fiftyDayFile = "#{$TICKER}.50.data"
make_graph_file(fiftyDayAvg, fiftyDayFile)

STDERR.puts "calculating 100 day average graph..\n"
hundredDayAvg = get_avg(data, 50)
hundredDayFile = "#{$TICKER}.100.data"
make_graph_file(hundredDayAvg, hundredDayFile)

STDERR.puts "calculating 200 day average graph..\n"
twohundredDayAvg = get_avg(data,100)
twohundredDayFile = "#{$TICKER}.200.data"
make_graph_file(twohundredDayAvg, twohundredDayFile)

STDERR.puts "calculating 500 day average graph..\n"
fivehundredDayAvg = get_avg(data,250)
fivehundredDayFile = "#{$TICKER}.500.data"
make_graph_file(fivehundredDayAvg, fivehundredDayFile)

STDERR.puts "calculating 800 day average graph..\n"
eighthundredDayAvg = get_avg(data,400)
eighthundredDayFile = "#{$TICKER}.800.data"
make_graph_file(eighthundredDayAvg, eighthundredDayFile)

STDERR.puts "calculating 1000 day average graph..\n"
thousandDayAvg = get_avg(data,500)
thousandDayFile = "#{$TICKER}.1000.data"
make_graph_file(thousandDayAvg, thousandDayFile)

# gnuplot script

gnuFilename = "#{$TICKER}.script"
gnuPlot = "plot \"#{dataFile}\", \"#{twentyDayFile}\" with lines, "\
 "\"#{fiftyDayFile}\" with lines, \"#{hundredDayFile}\" with lines lw 2, "\
 "\"#{twohundredDayFile}\" with lines lw 3 lt -1, "\
 "\"#{fivehundredDayFile}\" with lines lw 4 lt 5, "\
 "\"#{eighthundredDayFile}\" with lines lw 3 lt rgb \"grey\", "\
 "\"#{thousandDayFile}\" with lines lw 5 lt rgb \"magenta\""

gnuPause = "pause -1 \"hit any key to continue..\""
gnuPic = "set term png size 1024,768\nset output \"#{$TICKER}-#{$maxDaysHistory}.png\"\nreplot\nset term x11"

`rm -rf #{gnuFilename} 1>>/dev/null 2>>/dev/null`
gnuFile = File.open(gnuFilename, "w")
gnuFile.puts "set title \"stock price vs. time\""
gnuFile.puts "set xlabel \"#{$maxDaysHistory} days since #{$start_day}\""
gnuFile.puts "set ylabel \"Stock price\""
gnuFile.puts "set key left"
gnuFile.puts gnuPlot
gnuFile.puts gnuPause
gnuFile.puts gnuPic
gnuFile.close

`gnuplot #{gnuFilename}`
`rm *.data; rm #{gnuFilename}`

KIXEYE DevOps Scripting Challenge

#!/usr/bin/ruby

require "rubygems"
require "json"
require "net/ssh"
require "net/scp"

# deploystuff.rb
# DevOps scripting exercise for Kixeye interview
#
# Matt Feenstra 916-914-4009
# matt.a.feenstra@gmail.com


# configuration parameters
#
# normally we wouldn't use the plaintext password, but would distribute a ssh key instead

artifact_ext = ".tar.gz"
username = "matt"
password = "abc"
source_dir = "/tmp"
tar = "tar -zxvf"
dest_dir = "/mnt/kixeye"

# get command line arg for build revision (should be a number)
# mild sanity check
buildvers = nil
if ARGV[0] =~ /\d+/ && ARGV[1] =~ /.+/ && ARGV[2] =~ /.+/ then
  buildvers = ARGV[0]
  configfile = ARGV[1]
  artifact_dir = ARGV[2]
  puts "build version is #{buildvers}\n"
else
  puts  "incorrect syntax: #{$0} #{ARGV[0]} #{ARGV[1]} #{ARGV[2]}\n"
  abort "correct syntax:   #{$0} <build #> <config file> <source folder>"
end

# read in the configuration json
file = open(configfile)
json = file.read
file.close
config = JSON.parse(json)

# loop through the config.json hash (config)
config.each do |key,value|

  ## artifact keyword loop (web, base, map)
  artifact_name = key

  config[artifact_name].each do |key2, value2|

    if key2 =~ /hosts/

      config[artifact_name][key2].each do |hostname|

        filename = "#{artifact_name}-#{buildvers}#{artifact_ext}"

        # commands:  1) move the tarball to the game directory
        #            2) cd to game directory and untar (has build # on extraction dir)
        #            3) remove old symbolic link and relink to the new files
        #               ^ this is to allow for easy rollback to previous version by changing symlink
        #
        command =  "mv #{source_dir}/#{filename} #{dest_dir}",
                   "cd #{dest_dir}; #{tar} #{filename}",
                   "cd #{dest_dir}; rm -rf #{artifact_name}; ln -s #{artifact_name}-#{buildvers} #{artifact_name}"

        puts "uploading #{filename} to #{hostname}.."

        # scp files and install packages
        ssh = Net::SSH.start(hostname, username, :password => password)

        scp = ssh.scp
        scp.upload!("#{artifact_dir}/#{filename}", "#{source_dir}")

        puts "extracting #{filename} to #{dest_dir}..."

        command.each do |cmd|
          result = ssh.exec!(cmd)
          puts result
        end

        ssh.close
        puts "done.\n\n"

      end # config[artifact_name][key2].each

    end # if key2 =~ /hosts/
  end # config[artifact_name].each
end # config.each