topuniq

#!/bin/bash
#
# topuniq - sort input by unique count, printing total and percentages
#
#    Copyright (C) 2012 Rodrigo Silva (MestreLion) <linux@rodrigosilva.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program. If not see <http://www.gnu.org/licenses/gpl.html>
#
# Think of it as a sort | uniq -c | sort -nr on steroids ;)

#Defaults:
mincount=0
minperc=0
top=0
topperc=100
precision=1
showtotal=1
showperc=1
showother=1
totallast=0
sortother=0
stoptop=0
stopcount=0
enhance=0
totallabel="Total (%d)"
otherlabel="Other (%d)"
debug=0
awk=awk
debugopts=(--lint --profile --dump-variables)
gawkopts=(--use-lc-numeric)

argerr()  { echo "$myname: ${1:-error}" >&2 ; usage 1 ; }
invalid() { argerr "invalid option: $1" ; }
missing() { argerr "missing ${2:+$2 }operand${1:+ from $1}." ; }
integer() { [[ "$1" ]] || missing "$2" "${3-NUM}"; [[ "$1" != *[!0-9]* ]] ||
            argerr "'$1'${2:+ in $2} is not an integer."; }
numeric() { [[ "$1" ]] || missing "$2" "${3-NUM}"; re='^[0-9]+([.][0-9]+)?$';
            [[ "$1" =~ $re ]] || argerr "'$1'${2:+ in $2} is not a number."; }


usage() {
	cat <<- USAGE
	Usage: $myname [options] [FILE...]
	USAGE
	if [[ "$1" ]] ; then
		cat >&2 <<- USAGE
		Try '$myname --help' for more information.
		USAGE
		exit 1
	fi
	cat <<-USAGE

	Sort input by count, printing totals and percentages. Think of it as
	sort | uniq -c | sort -nr on steroids.

	If FILE is not given, read from standard input. For numeric input
	options, NUM must be a positive integer (digits only). All options
	requiring arguments accept both --option=ARG or --option ARG forms
	Options not listed here, if any, are appended to uniq -c

	Options:
	  -h|--help              show this page.

	  --min-count=NUM        only print lines with count >= NUM
	  --min-perc=NUM         only print lines with count percent >= NUM%
	  --top=NUM              only print the top NUM lines. 0 = all lines
	  --top-perc=NUM         only print the top NUM% lines

	  All lines with count less than any of the above options will be
	  grouped together as a single <other> line, printed last by default.
	  Setting a minimum higher than total, either count or percentage,
	  will effectively disable printing the <total> line. For --top-*
	  options, NUM does not include the total. --*-perc options accepts
	  non-integers such as 0.5

	  --stop-after-top=NUM   stop reading after NUM top unique lines
	  --stop-after-count=NUM stop reading after lines with count < NUM

	  Unlike --min-* and --top-* options, the above will discard lines,
	  thus affecting <total>, <other> and all percentages.
	  --stop-after-top is equivalent to 'head -nNUM' after sort -nr and
	  before $myname's enhancements. For both, NUM=0 disables the option

	  --precision=NUM        use NUM decimal digits for the percentages,
	                         default $precision

	  --no-perc              do not print percentages
	  --no-total             do not print <total> line
	  --no-other             do not print <other> line

	  --total-last           print <total> line last instead of first
	  --sort-other           print <other> line in sorted position

	  --label-total=LABEL use LABEL for <total> line, default "$totallabel"
	  --label-other=LABEL use LABEL for <other> line, default "$otherlabel"

	  For the --label-* options, optional "%d" prints the number of unique
	  lines that <total> or <other> represents

	  --enhance-uniq       consider input as already processed by
	                       sort | uniq -c, skip it and process from there.
	                       Useful for enhancing previously saved data


	Environment Variables:

	  $myname uses sort and uniq, so the user locale, particularly
	  LC_COLLATE, affects ordering and unique matching, as well as sort
	  performance. LC_NUMERIC affects decimal separator when printing
	  percentages. Use LC_ALL=C for the fastest and locale-independent
	  results.


	Examples:

	# Ignore lines with count < 10%, using case-insensitive uniq
	$myname --min-perc=10 --no-other --ignore-case

	# Top 20, sorting <others> within the list, and customizing its label
	$myname --top=20 --sort-other --label-other="Other %d unique lines"

	# Enhance an existing input, discarding lines with count < 10
	$myname my_uniq_data.txt --enhance-uniq --stop-after-count=10

	# Behaves exactly like sort | uniq -c | sort -nr
	$myname --no-total --no-perc

	For input data, some examples you may pipe directly to $myname:

	# Words in Bash's manual page
	man bash | tr '[:punct:][:blank:]' '\n' | sed '/^$/d'

	# Icon types in /usr/share/icons
	find /usr/share/icons -type f -name "*.*" | awk -F. '{print \$NF}'

	# Shebangs from /usr/bin scripts
	for f in /usr/bin/*; do [ -f "$f" ] && head -n1 "$f" | grep ^#!; done

	Copyright (C) 2012 Rodrigo Silva (MestreLion) <linux@rodrigosilva.com>
	License: GPLv3 or later. See <http://www.gnu.org/licenses/gpl.html>
	USAGE
	exit 0
}

myname="${0##*/}"
files=()
uniqopts=()
for arg in "$@"; do [[ "$arg" == "-h" || "$arg" == "--help" ]] && usage ; done
while (( $# )); do
	case "$1" in
	--min-count=*        ) mincount="${1#*=}"      ;;
	--min-perc=*         ) minperc="${1#*=}"       ;;
	--top=*              ) top="${1#*=}"           ;;
	--top-perc=*         ) topperc="${1#*=}"       ;;
	--precision=*        ) precision="${1#*=}"     ;;
	--label-total=*      ) totallabel="${1#*=}"    ;;
	--label-other=*      ) otherlabel="${1#*=}"    ;;
	--stop-after-top=*   ) stoptop="${1#*=}"       ;;
	--stop-after-count=* ) stopcount="${1#*=}"     ;;
	--min-count          ) shift ; mincount="$1"   ;;
	--min-perc           ) shift ; minperc="$1"    ;;
	--top                ) shift ; top="$1"        ;;
	--top-perc           ) shift ; topperc="$1"    ;;
	--precision          ) shift ; precision="$1"  ;;
	--label-total        ) shift ; totallabel="$1" ;;
	--label-other        ) shift ; otherlabel="$1" ;;
	--stop-after-top     ) shift ; stoptop="$1"    ;;
	--stop-after-count   ) shift ; stopcount="$1"  ;;
	--no-total           ) showtotal=0             ;;
	--no-perc            ) showperc=0              ;;
	--no-other           ) showother=0             ;;
	--total-last         ) totallast=1             ;;
	--sort-other         ) sortother=1             ;;
	--enhance-uniq       ) enhance=1               ;;
	--debug              ) debug=1                 ;;
	--                   ) shift ; break           ;;
	-*                   ) uniqopts+=( "$1" )      ;;
	*                    ) files+=( "$1" )         ;;
	esac
	shift
done

if ((debug)) ; then
	exec 2>"$myname.log"
	set -x
	if type pgawk >/dev/null 2>&1 ; then
		awk=pgawk
	else
		debugopts=()
	fi
else
	debugopts=()
fi

# gawk test
$awk "${gawkopts[@]}" 2>/dev/null || gawkopts=()

files+=( "$@" )

[[ "$totallabel" ]] || missing "--label-total" "LABEL"
[[ "$otherlabel" ]] || missing "--label-other" "LABEL"

integer "$mincount"  "--min-count"
numeric "$minperc"   "--min-perc"
integer "$top"       "--top"
numeric "$topperc"   "--top-perc"
integer "$precision" "--precision"
integer "$stoptop"   "--stop-after-top"
integer "$stopcount" "--stop-after-count"

if ((enhance)); then
	cat "${files[@]}"
else
	sort "${files[@]}" | uniq -c "${uniqopts[@]}"
fi |
if ((stoptop)); then head -n${stoptop}; else cat; fi |
LC_ALL=C sort -nr |
$awk -v showtotal=$showtotal -v showperc=$showperc -v showother=$showother \
     -v mincount=$mincount   -v minperc=$minperc   -v precision=$precision \
     -v top=$top             -v topperc=$topperc   -v stopcount=$stopcount \
     -v totallast=$totallast -v sortother=$sortother                       \
     -v otherlabel="$otherlabel" -v totallabel="$totallabel"               \
     -F ' ' "${gawkopts[@]}" "${debugopts[@]}" '
function printline(item) {
	if (showperc)
		printf("%*d %*.*f%% %s\n", cpad, count[item],
		        ppad, precision, percs[item], label[item])
	else
		printf("%*d %s\n", cpad, count[item], label[item])
}
BEGIN {
	items = 0

	count["total"] = 0
	percs["total"] = 100
	lines["total"] = 0

	count["other"] = 0
	lines["other"] = 0
}
$1 < stopcount {
	exit
}
{
	# FIRST PASS: calculate total

	count["total"] += $1
	lines["total"]++

	# Filter on counter. Not needed here, but it keeps arrays small
	if ((!top || (items < top)) && ($1 >= mincount)) {
		count[items] = $1
		label[items] = $0
		sub("[ ]*" $1 " ", "", label[items])  # label = $2 onwards
		items++
	}
	else {
		count["other"] += $1
		lines["other"]++
	}
}
END {
	# Do some <total> handling
	if (count["total"] > 0)
		if ((count["total"] < mincount) || (percs["total"] < minperc))
			showtotal = 0
		else
			label["total"] = sprintf(totallabel, lines["total"])
	else
		exit

	# SECOND PASS: calculate percentages and <other>

	topitems = items * 100 / topperc  # trim based on topperc
	for (i=0; i < topitems; i++) {
		percent = count[i] * 100 / count["total"]
		if (percent >= minperc)
			percs[i] = percent
		else {
			topitems = i  # trim again based on minperc
			break
		}
	}
	# Put remaining items in <other>
	for (; i<items; i++) {
		count["other"] += count[i]
		lines["other"]++
	}
	if (count["other"] > 0) {
		percs["other"] = count["other"] * 100 / count["total"]
		label["other"] = sprintf(otherlabel, lines["other"])
	}
	else
		showother = 0

	# Calculate max lengths (for paddings)
	if (showtotal) {
		# padding taken from <total>
		maxcount = count["total"]
		maxpercs = percs["total"]  # == 100, duh
	}
	else {
		if (showother && (count["other"] > count[0])) {
			# padding taken from <other>
			maxcount = count["other"]
			maxpercs = percs["other"]
		}
		else {
			# padding taken from top line
			maxcount = count[0]
			maxpercs = percs[0]
		}
	}

	# percentage padding
	ppad = length(maxpercs "")
	if (precision > 0) ppad += precision+1  # account for decimal separator

	# count padding
	mincpad = 7  # seems to be uniq -c hardcoded min padding
	if (length(maxcount "") < mincpad)
		cpad = mincpad
	else
		cpad = length(maxcount "")

	# THIRD PASS: print results

	if (showtotal && !totallast) {
		printline("total")
		showtotal = 0
	}

	for (i=0; i<topitems; i++) {
		if (showother && sortother && (count["other"] > count[i])) {
			printline("other")
			showother = 0
		}
		if ((count[i] >= mincount) && (percs[i] >= minperc))
			printline(i)
	}

	if (showother) printline("other")
	if (showtotal) printline("total")
}
'