Add largest facts command

README.md

@@ -1,13 +1,32 @@
 # support-script-tooling
 
 ## Requirements
+* mlr
 * gawk
 * gnuplot
+* bash 4+
+
+Note for MacOS users: to get a recent enough version of `bash`, you will need to manually install it, e.g.
+
+```
+brew install bash
+```
+
+Then confirm the version with
+
+```
+/usr/bin/env bash --version
+```
+
+See issue #13 if you have suggetions to improve this.
 
 For convenience, you may want to add this repo to `$PATH` instead of using absolute paths.
 
 Below is a brief description of each job.
 
+### support-tooling.sh
+An interactive wrapper script for some of the more common/useful things you can do with this repo.  Run `support-tooling.sh` from within an extracted support script and a simple menu will guide the user.
+
 ### puppet-top-api-calls.sh
 
 Summarize the count, average duration, and 99th percentile duration of per endpoint in Puppet server and DB access logs per 30 minute interval.  Can also graph the count and average duration in `gnuplot`

console_largest_facts.awk

@@ -0,0 +1,7 @@
+#!/usr/bin/env -S gawk -f
+
+# Sample usage:
+# console_largest_facts.awk logs/puppetserver/puppetserver-access.log | sort -nr | cut -f2- -d ' ' | head
+
+# We'll just use $NF for now for speed.
+$7 ~ "classified/nodes" { print $NF " " $0 }

puppet-top-api-calls.sh

@@ -7,7 +7,7 @@
 
 cleanup() {
   for f in "${tmp_files[@]}"; do
-    rm -- "$f"
+    [[ -e $f ]] && rm -- "$f"
   done
 }
 trap cleanup EXIT
@@ -44,28 +44,27 @@ while getopts ":bgc" opt; do
 
 shift $((OPTIND-1))
 
-if [[ $graph && $borrow ]]; then
-  printf "%s\n" "Graphing output of borrow times currently not supported" >&2
-  unset graph
-fi
+
+perl_command=(perl -ne '/^\S+ \S+ \S+ \[[^\]]+\] "[A-Z]+ [^ "]+? HTTP\/[0-9.]+" [0-9]{3} [0-9]+|- "[^"]*" "[^"]*"/ && print')
 
 for file in "$@"; do
   printf 'Processing: %s\n' "$file" >&2
 
   case "${file##*.}" in
     gz)
-      read_cmd=(gunzip -c)
+      read_cmd=("gunzip" "-c" "--" "$file")
       ;;
     *)
-      read_cmd=(cat)
+      # UUOC, but it makes setting up read_cmd easier
+      read_cmd=("cat" "--" "$file")
       ;;
   esac
 
   case "${file##*/}" in
     puppetserver-access*)
       if [[ $borrow ]]; then duration_field=0; else duration_field=2; fi
 
-      "${read_cmd[@]}" "$file" \
+      "${read_cmd[@]}" | "${perl_command[@]}" \
         | gawk -v n="$duration_field" \
           'BEGIN { print "date,path,duration" }
                {
@@ -120,6 +119,15 @@ tmp_files+=("$plot_tmp")
 
 # TODO: remember how tf this works
 if [[ $count ]]; then mlr_field=3; else mlr_field=4; fi
+
+if [[ $borrow ]]; then
+  borrow_tmp="$(mktemp)"
+  tmp_files+=("$borrow_tmp")
+
+  sed '/-$/d' "$mlr_tmp" >"$borrow_tmp"
+  mv "$borrow_tmp" "$mlr_tmp"
+fi
+
 sed '/^date/d' "$mlr_tmp" | sort -k2,2 \
   | gawk -v n="$mlr_field" 'NR == 1 { printf "\"%s\"\n", $2 } NR >1 && prev != $2 { print ""; print ""; printf "\"%s\"\n", $2} { print $1 " " $n }  { prev = $2 }' >"$plot_tmp"
 

puppetserver_largest_reports.awk

@@ -0,0 +1,8 @@
+#!/usr/bin/env -S gawk -f
+
+# Sample usage:
+# puppetserver_largest_reports.awk logs/puppetserver/puppetserver-access.log | sort -nr | cut -f2- -d ' ' | head
+
+BEGIN { FPAT="([^ ]+)|(\"[^\"]+\")" }
+
+$6 ~ "/report" { print $12 " " $0 }

puppetserver_longest_borrows.awk

@@ -0,0 +1,8 @@
+#!/usr/bin/env -S gawk -f
+
+# Sample usage:
+# puppetserver_longest_borrows.awk logs/puppetserver/puppetserver-access.log | sort -nr | cut -f2- -d ' ' | head
+
+BEGIN { FPAT="([^ ]+)|(\"[^\"]+\")" }
+
+{ print $13 " " $0 }

puppetserver_longest_compiles.awk

@@ -3,10 +3,10 @@
 # Sample usage:
 # echo "Average Count Hostname" && puppetserver_longest_compiles.awk logs/puppetserver/puppetserver.log | sort -rn | head | column -t
 
-/Compiled static catalog/ {
+/Compiled (static )?catalog/ {
   # Store the cumulative total and count for each hostname in arrays
-  name[$10] += $15
-  count[$10] += 1
+  name[$(NF - 6)] += $(NF-1)
+  count[$(NF - 6)] += 1
 }
 
 END {

support-tooling.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+#TODO: common.sh
+warn() {
+  printf '%s\n' "$@"
+}
+
+latest_logs() {
+  # Absolutely terrible hack because the dev is too lazy to deal with non-GNU utilities
+  _latest=($(ls -t **/*"$1"*))
+}
+
+puppetserver_latest() {
+  latest_logs "puppetserver-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No puppetserver-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/puppet-top-api-calls.sh -g "${_latest[0]}"
+}
+
+puppetserver_longest_borrows() {
+  latest_logs "puppetserver-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No puppetserver-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/puppetserver_longest_borrows.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
+}
+
+puppetserver_largest_reports() {
+  latest_logs "puppetserver-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No puppetserver-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/puppetserver_largest_reports.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
+}
+
+console_largest_facts() {
+  latest_logs "console-services-api-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No console-services-api-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/console_largest_facts.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
+}
+
+confirm_opt() {
+  cur_opt=$1
+  cur_menu=("$@")
+  cur_menu=("${cur_menu[@]:1}")
+
+  [[ " ${cur_menu[@]} " =~ " $cur_opt " ]]
+}
+
+comm_cmds() {
+  declare -A local menu=(
+    ['Plot latest puppetserver-access.log']=puppetserver_latest
+    ['Longest JRuby borrow times']=puppetserver_longest_borrows
+    ['Largest report submissions']=puppetserver_largest_reports
+    ['Largest fact sizes']=console_largest_facts
+  )
+
+  select opt in "${!menu[@]}"; do
+    confirm_opt "$opt" "${!menu[@]}" || {
+      warn 'Please enter a valid number'
+      warn 'CTRL+C exits'
+      break
+    }
+    "${menu[$opt]}"
+    break
+  done
+}
+
+shopt -s globstar nullglob
+_tmp="$(mktemp)"
+
+base_dir="${BASH_SOURCE[0]%/*}"
+PS3="Select an option by number: "
+declare -A main_menu=(
+  ['Common commands']=comm_cmds
+)
+
+while :; do
+  select opt in "${!main_menu[@]}"; do
+    confirm_opt "$opt" "${!main_menu[@]}" || {
+      warn 'Please enter a valid number'
+      warn 'CTRL+C exits'
+      break
+    }
+    "${main_menu[$opt]}"
+    break
+  done
+done