From ef4306b84094a90b3c9318a2adbac4e2e90e8aca Mon Sep 17 00:00:00 2001
From: Adrian Parreiras Horta <sensei.loafage@gmail.com>
Date: Tue, 30 Mar 2021 12:28:04 -0700
Subject: [PATCH 01/13] Add mlr requirement

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index c1fb0ae..ec9145b 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
 # support-script-tooling
 
 ## Requirements
+* mlr
 * gawk
 * gnuplot
 

From eadeda3bf528931544c34bfc1706dcdb320fe815 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Wed, 7 Apr 2021 13:56:41 -0700
Subject: [PATCH 02/13] Fix longest compile times without static catalogs

---
 puppetserver_longest_compiles.awk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/puppetserver_longest_compiles.awk b/puppetserver_longest_compiles.awk
index e4a8c38..9d2903c 100755
--- a/puppetserver_longest_compiles.awk
+++ b/puppetserver_longest_compiles.awk
@@ -3,10 +3,10 @@
 # Sample usage:
 # echo "Average Count Hostname" && puppetserver_longest_compiles.awk logs/puppetserver/puppetserver.log | sort -rn | head | column -t
 
-/Compiled static catalog/ {
+/Compiled (static )?catalog/ {
   # Store the cumulative total and count for each hostname in arrays
-  name[$10] += $15
-  count[$10] += 1
+  name[$(NF - 6)] += $(NF-1)
+  count[$(NF - 6)] += 1
 }
 
 END {

From dc96590006436b54163bd2f6480cdd8d7ad64c71 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Tue, 13 Apr 2021 14:53:44 -0700
Subject: [PATCH 03/13] Add a PCRE regex for CLF files

Prior to this commit, some access log entries were not in the common log
format, e.g. missing the IP address.  This commit uses perl to only
process valid lines.
---
 puppet-top-api-calls.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/puppet-top-api-calls.sh b/puppet-top-api-calls.sh
index 8f612b8..96dd02d 100755
--- a/puppet-top-api-calls.sh
+++ b/puppet-top-api-calls.sh
@@ -49,15 +49,18 @@ if [[ $graph && $borrow ]]; then
   unset graph
 fi
 
+perl_command=(perl -ne '/^\S+ \S+ \S+ \[[^\]]+\] "[A-Z]+ [^ "]+? HTTP\/[0-9.]+" [0-9]{3} [0-9]+|- "[^"]*" "[^"]*"/ && print')
+
 for file in "$@"; do
   printf 'Processing: %s\n' "$file" >&2
 
   case "${file##*.}" in
     gz)
-      read_cmd=(gunzip -c)
+      read_cmd=("gunzip" "-c" "--" "$file")
       ;;
     *)
-      read_cmd=(cat)
+      # UUOC, but it makes setting up read_cmd easier
+      read_cmd=("cat" "--" "$file")
       ;;
   esac
 
@@ -65,7 +68,7 @@ for file in "$@"; do
     puppetserver-access*)
       if [[ $borrow ]]; then duration_field=0; else duration_field=2; fi
 
-      "${read_cmd[@]}" "$file" \
+      "${read_cmd[@]}" | "${perl_command[@]}" \
         | gawk -v n="$duration_field" \
           'BEGIN { print "date,path,duration" }
                {

From e552042ffb29d5f86667fab69c8311e7ca70aaba Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Fri, 23 Apr 2021 15:31:16 -0700
Subject: [PATCH 04/13] Add graphing of JRuby-only endpoints

---
 puppet-top-api-calls.sh | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/puppet-top-api-calls.sh b/puppet-top-api-calls.sh
index 96dd02d..55e7715 100755
--- a/puppet-top-api-calls.sh
+++ b/puppet-top-api-calls.sh
@@ -7,7 +7,7 @@
 
 cleanup() {
   for f in "${tmp_files[@]}"; do
-    rm -- "$f"
+    [[ -e $f ]] && rm -- "$f"
   done
 }
 trap cleanup EXIT
@@ -44,10 +44,6 @@ while getopts ":bgc" opt; do
 
 shift $((OPTIND-1))
 
-if [[ $graph && $borrow ]]; then
-  printf "%s\n" "Graphing output of borrow times currently not supported" >&2
-  unset graph
-fi
 
 perl_command=(perl -ne '/^\S+ \S+ \S+ \[[^\]]+\] "[A-Z]+ [^ "]+? HTTP\/[0-9.]+" [0-9]{3} [0-9]+|- "[^"]*" "[^"]*"/ && print')
 
@@ -123,6 +119,15 @@ tmp_files+=("$plot_tmp")
 
 # TODO: remember how tf this works
 if [[ $count ]]; then mlr_field=3; else mlr_field=4; fi
+
+if [[ $borrow ]]; then
+  borrow_tmp="$(mktemp)"
+  tmp_files+=("$borrow_tmp")
+
+  sed '/-$/d' "$mlr_tmp" >"$borrow_tmp"
+  mv "$borrow_tmp" "$mlr_tmp"
+fi
+
 sed '/^date/d' "$mlr_tmp" | sort -k2,2 \
   | gawk -v n="$mlr_field" 'NR == 1 { printf "\"%s\"\n", $2 } NR >1 && prev != $2 { print ""; print ""; printf "\"%s\"\n", $2} { print $1 " " $n }  { prev = $2 }' >"$plot_tmp"
 

From 4e3225f01bae825f9474f6dc9e3f051c5a5ac494 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Wed, 12 May 2021 14:05:55 -0700
Subject: [PATCH 05/13] Add awk script to show largest report submissions

---
 puppetserver_largest_reports.awk | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100755 puppetserver_largest_reports.awk

diff --git a/puppetserver_largest_reports.awk b/puppetserver_largest_reports.awk
new file mode 100755
index 0000000..3160812
--- /dev/null
+++ b/puppetserver_largest_reports.awk
@@ -0,0 +1,8 @@
+#!/usr/bin/env -S gawk -f
+
+# Sample usage:
+# puppetserver_largest_reports.awk logs/puppetserver/puppetserver-access.log | sort -nr | cut -f2- -d ' ' | head
+
+BEGIN { FPAT="([^ ]+)|(\"[^\"]+\")" }
+
+$6 ~ "/report" { print $12 " " $0 }

From 24ce5350609dbeec955b79226662b2f9a4f68288 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Wed, 12 May 2021 14:23:57 -0700
Subject: [PATCH 06/13] Add awk script for longest JRuby borrow times

---
 puppetserver_longest_borrows.awk | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100755 puppetserver_longest_borrows.awk

diff --git a/puppetserver_longest_borrows.awk b/puppetserver_longest_borrows.awk
new file mode 100755
index 0000000..0343f6e
--- /dev/null
+++ b/puppetserver_longest_borrows.awk
@@ -0,0 +1,8 @@
+#!/usr/bin/env -S gawk -f
+
+# Sample usage:
+# puppetserver_longest_borrows.awk logs/puppetserver/puppetserver-access.log | sort -nr | cut -f2- -d ' ' | head
+
+BEGIN { FPAT="([^ ]+)|(\"[^\"]+\")" }
+
+{ print $13 " " $0 }

From 565a3546f7d71b1d32061f064964b3b0cb0a9e13 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Wed, 12 May 2021 11:21:03 -0700
Subject: [PATCH 07/13] Initial commit for interactive wrapper

---
 support-tooling.sh | 66 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100755 support-tooling.sh

diff --git a/support-tooling.sh b/support-tooling.sh
new file mode 100755
index 0000000..40c07ab
--- /dev/null
+++ b/support-tooling.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+#TODO: common.sh
+warn() {
+  printf '%s\n' "$@"
+}
+
+latest_logs() {
+  while IFS= read -rd '' line; do
+    _latest+=("${line#* }")
+  done < <(find . -type f -name "$1*" -printf '%T@ %p\0' | sort -znr)
+
+}
+
+puppetserver_latest() {
+  latest_logs "puppetserver-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No puppetserver-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/puppet-top-api-calls.sh -g "${_latest[0]}"
+}
+
+confirm_opt() {
+  cur_opt=$1
+  cur_menu=("$@")
+  cur_menu=("${cur_menu[@]:1}")
+
+  [[ " ${cur_menu[@]} " =~ " $cur_opt " ]]
+}
+
+comm_cmds() {
+  declare -A local menu=(
+    ['Plot latest puppetserver-access.log']=puppetserver_latest
+  )
+
+  #TODO: function
+  select opt in "${!menu[@]}"; do
+    confirm_opt "$opt" "${!menu[@]}" || {
+      warn 'Please enter a valid number'
+      warn 'CTRL+C exits'
+      break
+    }
+    "${menu[$opt]}"
+    break
+  done
+}
+
+base_dir="${BASH_SOURCE[0]%/*}"
+PS3="Select an option by number: "
+declare -A main_menu=(
+  ['Common commands']=comm_cmds
+)
+
+while :; do
+  select opt in "${!main_menu[@]}"; do
+    confirm_opt "$opt" "${!main_menu[@]}" || {
+      warn 'Please enter a valid number'
+      warn 'CTRL+C exits'
+      break
+    }
+    "${main_menu[$opt]}"
+    break
+  done
+done

From b4e1a3262969017cbff8fc8b93d6841fab98de24 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Wed, 12 May 2021 15:14:22 -0700
Subject: [PATCH 08/13] Add largest reports and longest borrows to menu

---
 support-tooling.sh | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/support-tooling.sh b/support-tooling.sh
index 40c07ab..4c11fcd 100755
--- a/support-tooling.sh
+++ b/support-tooling.sh
@@ -22,6 +22,26 @@ puppetserver_latest() {
   "$base_dir"/puppet-top-api-calls.sh -g "${_latest[0]}"
 }
 
+puppetserver_longest_borrows() {
+  latest_logs "puppetserver-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No puppetserver-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/puppetserver_longest_borrows.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
+}
+
+puppetserver_largest_reports() {
+  latest_logs "puppetserver-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No puppetserver-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/puppetserver_largest_reports.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
+}
+
 confirm_opt() {
   cur_opt=$1
   cur_menu=("$@")
@@ -33,9 +53,10 @@ confirm_opt() {
 comm_cmds() {
   declare -A local menu=(
     ['Plot latest puppetserver-access.log']=puppetserver_latest
+    ['Longest JRuby borrow times']=puppetserver_longest_borrows
+    ['Largest report submissions']=puppetserver_largest_reports
   )
 
-  #TODO: function
   select opt in "${!menu[@]}"; do
     confirm_opt "$opt" "${!menu[@]}" || {
       warn 'Please enter a valid number'

From 1810ba74d407ca5667875af2f46b454574377ad3 Mon Sep 17 00:00:00 2001
From: Adrian Parreiras Horta <adrian.parreiras-horta@puppet.com>
Date: Wed, 12 May 2021 15:35:19 -0700
Subject: [PATCH 09/13] Update README.md

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index ec9145b..1154426 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,9 @@ For convenience, you may want to add this repo to `$PATH` instead of using absol
 
 Below is a brief description of each job.
 
+### support-tooling.sh
+An interactive wrapper script for some of the more common/useful things you can do with this repo.  Run `support-tooling.sh` from within an extracted support script and a simple menu will guide the user.
+
 ### puppet-top-api-calls.sh
 
 Summarize the count, average duration, and 99th percentile duration of per endpoint in Puppet server and DB access logs per 30 minute interval.  Can also graph the count and average duration in `gnuplot`

From d0469f59fdf39ce3e29079e9dd708bc7cc3bfb38 Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Thu, 13 May 2021 08:55:39 -0700
Subject: [PATCH 10/13] Remove find -printf

Instead, we use globstar and ls.  This is bad, but easier than figuring
out the differences in find, xargs, sort, etc.  Since we should only be
parsing our log files, the changes of spaces in filesnames is low
---
 support-tooling.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/support-tooling.sh b/support-tooling.sh
index 4c11fcd..adb80b0 100755
--- a/support-tooling.sh
+++ b/support-tooling.sh
@@ -6,10 +6,8 @@ warn() {
 }
 
 latest_logs() {
-  while IFS= read -rd '' line; do
-    _latest+=("${line#* }")
-  done < <(find . -type f -name "$1*" -printf '%T@ %p\0' | sort -znr)
-
+  # Absolutely terrible hack because the dev is too lazy to deal with non-GNU utilities
+  _latest=($(ls -t **/*"$1"*))
 }
 
 puppetserver_latest() {
@@ -68,6 +66,9 @@ comm_cmds() {
   done
 }
 
+shopt -s globstar nullglob
+_tmp="$(mktemp)"
+
 base_dir="${BASH_SOURCE[0]%/*}"
 PS3="Select an option by number: "
 declare -A main_menu=(

From c5276f4d27b625def3077faf80ec168bbfc70cbb Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Fri, 15 Oct 2021 08:54:28 -0700
Subject: [PATCH 11/13] Add command for largest facts

Also add it to the interactive component.
---
 support-tooling.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/support-tooling.sh b/support-tooling.sh
index adb80b0..f08ad8b 100755
--- a/support-tooling.sh
+++ b/support-tooling.sh
@@ -40,6 +40,16 @@ puppetserver_largest_reports() {
   "$base_dir"/puppetserver_largest_reports.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
 }
 
+console_largest_facts() {
+  latest_logs "console-services-api-access"
+  (( ${#_latest[@]} > 0 )) || {
+    warn 'No console-services-api-access.log found.' 'Please run me from the root of an extracted support script'
+    return
+  }
+
+  "$base_dir"/console_largest_facts.awk "${_latest[0]}" | sort -nr | cut -f2- -d ' ' | head
+}
+
 confirm_opt() {
   cur_opt=$1
   cur_menu=("$@")
@@ -53,6 +63,7 @@ comm_cmds() {
     ['Plot latest puppetserver-access.log']=puppetserver_latest
     ['Longest JRuby borrow times']=puppetserver_longest_borrows
     ['Largest report submissions']=puppetserver_largest_reports
+    ['Largest fact sizes']=console_largest_facts
   )
 
   select opt in "${!menu[@]}"; do

From e5bc9fbab00071c5da3af082744d882777841f19 Mon Sep 17 00:00:00 2001
From: Adrian Parreiras Horta <adrian.parreiras-horta@puppet.com>
Date: Tue, 19 Oct 2021 11:02:11 -0700
Subject: [PATCH 12/13] Update README.md

---
 README.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/README.md b/README.md
index 1154426..ba1345b 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,21 @@
 * mlr
 * gawk
 * gnuplot
+* bash 4+
+
+Note for MacOS users: to get a recent enough version of `bash`, you will need to manually install it, e.g.
+
+```
+brew install bash
+```
+
+Then confirm the version with
+
+```
+/usr/bin/env bash --version
+```
+
+See issue #13 if you have suggetions to improve this.
 
 For convenience, you may want to add this repo to `$PATH` instead of using absolute paths.
 

From 7398b2e5efc0f96fcca66e74b25430e928c7814c Mon Sep 17 00:00:00 2001
From: Adrian <sensei.loafage@gmail.com>
Date: Sun, 24 Oct 2021 10:23:03 -0700
Subject: [PATCH 13/13] Actually check in the file

---
 console_largest_facts.awk | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100755 console_largest_facts.awk

diff --git a/console_largest_facts.awk b/console_largest_facts.awk
new file mode 100755
index 0000000..f81495b
--- /dev/null
+++ b/console_largest_facts.awk
@@ -0,0 +1,7 @@
+#!/usr/bin/env -S gawk -f
+
+# Sample usage:
+# console_largest_facts.awk logs/puppetserver/puppetserver-access.log | sort -nr | cut -f2- -d ' ' | head
+
+# We'll just use $NF for now for speed.
+$7 ~ "classified/nodes" { print $NF " " $0 }