move purge settings to conf. set report file extension

Changelog

@@ -1,3 +1,9 @@
+* 1.17 2023-07-24 - robl
+  - ixp-watch-tidy: get config options from config file.
+  - ixp-watch-tidy: Option to purge reports as well as samples.
+  - install.sh: prompt for purge settings.
+  - Configurable report file extension. .TXT by default.
+
 * 1.16a 2023-05-09 - robl
   - Bugfix: Do not try to copy the report via slack on arp alert, only email.
   - Bugfix: RRD Graph PNGs not updated.

bin/ixp-watch

@@ -298,6 +298,8 @@ E_DATE=$(date +%s)  # Epoch UNIX date (for RRD graphs updates)
 FILEDATE=$(date "+%Y-%m-%d-%H-%M")  # Use as basis for file name. Do not change.
 STARTDATE=$(date "+%Y-%m-%d %H:%M") # Date used in reports. Adjust to taste.
 
+REP_EXTN=${REP_EXTN:=TXT}           # Report file extension "TXT" by default.
+
 # You may need to check SAMPLESIZE below in this script, if your ls -la
 # output is not the usual format. (Works on debian and other linux okay)
 
@@ -476,7 +478,7 @@ $TSHARK -T fields -e ipv6.src -e eth.src_resolved -r $SAMPLEDIR/$FILEDATE -Y "ic
 cat $TEMP_DIR/notip.$$ | awk '{print $3" -> "$5" "$6" "$7" "$8" "$9" "$10" "$11" "$12}' | sort | uniq -c | sort -rn > $LOG_ROOT/NOT_IP.LOG
 
 # Create alarms for bad traffic:
-cd  $LOG_ROOT/active/
+cd $LOG_ROOT/active/
 
 # Non IP Traffic (CDP etc.)
 cat $LOG_ROOT/NOT_IP.LOG | sed 's/[^a-zA-Z0-9\ \:\>\-]/_/g' | grep -v "STP " | awk '{print "echo \"Alarm for "$__"\" >"$2"__"$4}' | sh
@@ -531,12 +533,23 @@ if [ "$SPANNING_TREE_BAD" = "1" ] ; then
 
 fi
 
+# Clear old active:
+find $LOG_ROOT/active/ -type f -ctime +7 -exec rm {} \;
 
-for i in $( ls ); do
- if [ ! -f $LOG_ROOT/alarms/$i ] ; then
- cat $i >> $TEMP_DIR/alarms.tmp
- cp $i  $LOG_ROOT/alarms/
- fi
+# Remove alarm states for traffic no longer active:
+for file in ${LOG_ROOT}/alarms/* ; do
+   if [ ! -f "${LOG_ROOT}/active/${file##*/}" ] ; then
+     rm -f ${file}
+   fi
+done
+
+# Log new alarms, create state for things we just 
+# alerted about, so we don't alert again:
+for file in ${LOG_ROOT}/active/* ; do
+   if [ ! -f "${LOG_ROOT}/alarms/${file##*/}" ] ; then
+     cat ${file} >> $TEMP_DIR/alarms.tmp
+     cp ${file} $LOG_ROOT/alarms/
+   fi
 done
 
 # Take the ICMP output and munge it to report:
@@ -648,7 +661,7 @@ function create_report()
 
 
 # Create report
-create_report $LOGDIR/$FILEDATE.TXT "$NETWORK LAN Traffic Summary Report - $ISODATE"
+create_report $LOGDIR/$FILEDATE.$REP_EXTN "$NETWORK LAN Traffic Summary Report - $ISODATE"
 report "Analysis based on a sample of $NUM_MINUTES minutes."
 report "Started at $STARTDATE, ended at $STOPDATE"
 report "The entire session is saved in:  $SAMPLEDIR/$FILEDATE.gz"
@@ -712,7 +725,7 @@ if [ -n "$HTML_DIR" ] ; then
 		if [ -f "$HTML_DIR/report_2.html" ] ; then mv $HTML_DIR/report_2.html $HTML_DIR/report_3.html ; fi
 	  if [ -f "$HTML_DIR/report_1.html" ] ; then mv $HTML_DIR/report_1.html $HTML_DIR/report_2.html ; fi
 
-		cp $LOGDIR/$FILEDATE.TXT $HTML_DIR/report_1.html
+		cp $LOGDIR/$FILEDATE.$REP_EXTN $HTML_DIR/report_1.html
 
 fi
 
@@ -727,7 +740,7 @@ function do_graph()
   if [ ! -f "$RRD_DIR/${rrd}.rrd" ]  ; then $RRDTOOL create $RRD_DIR/${rrd}.rrd --step=$SAMPLE_TIME DS:${rrd}:GAUGE:1800:U:U RRA:AVERAGE:0.5:1:96 ; fi
 
   $RRDTOOL update $RRD_DIR/${rrd}.rrd $E_DATE:${val}
-  $RRDTOOL graph $GRAPH_DIR/${rrd}.png --vertical-label "$title" DEF:${rrd}=${RRD_DIR}/${rrd}.rrd:${rrd}:AVERAGE AREA:"${rrd}#99CCFF"
+  $RRDTOOL graph $GRAPH_DIR/${rrd}.png --vertical-label "$title" DEF:${rrd}=${RRD_DIR}/${rrd}.rrd:${rrd}:AVERAGE AREA:"${rrd}#99CCFF" > /dev/null
 
 }
 
@@ -769,7 +782,7 @@ echo "ARPS Per min is $NUM_ARPS_MIN" > $LOG_ROOT/alarms_arp.tmp
 
   if [ ! -f $LOG_ROOT/arpstorm.warn ] ; then
 
-   do_alert CRITICAL "High number of ARPS in progress - Please investigate" $LOG_ROOT/alarms_arp.tmp $LOGDIR/$FILEDATE.TXT
+   do_alert CRITICAL "High number of ARPS in progress - Please investigate" $LOG_ROOT/alarms_arp.tmp $LOGDIR/$FILEDATE.$REP_EXTN
    touch $LOG_ROOT/arpstorm.warn
 
   fi
@@ -811,7 +824,7 @@ fi
 # If you want to send full reports to e-mail:
 
 if [ -n "$REPORT_EMAIL" ] ; then
- $MAILPROG -s "[$NETWORK] Traffic Summary Report" $REPORT_EMAIL< $LOGDIR/$FILEDATE.TXT
+ $MAILPROG -s "[$NETWORK] Traffic Summary Report" $REPORT_EMAIL< $LOGDIR/$FILEDATE.$REP_EXTN
 fi
 
 # Clean up

bin/ixp-watch-tidy

@@ -45,38 +45,68 @@ while getopts ${optstring} arg; do
 done
 
 
-if [ ! -f $CONFIG ] ; then
- echo "Could not open config file $CONFIG!"
+if [ ! -f ${CONFIG} ] ; then
+ echo "Could not open config file ${CONFIG}!"
  exit 1
 fi
 
-source $CONFIG
+source ${CONFIG}
 
+if [ ! -d "${SAMPLE_ROOT}" ] ; then
+        echo "ERROR: SAMPLE_ROOT not found: ${SAMPLE_ROOT}!"
+        exit 1
+fi
 
-if [ ! -d "$SAMPLE_ROOT" ] ; then
+cd ${SAMPLE_ROOT}
 
-	echo "Unable to locate directory: $SAMPLE_ROOT"
-	exit 1
+# Check that we can cd in the right directory before we start
+# trying to delete stuff!
 
+if [ "${PWD}" != "${SAMPLE_ROOT}" ] ; then
+        echo "ERROR: Unable to cd to ${SAMPLE_ROOT}!"
+        exit 1
 fi
 
-cd $SAMPLE_ROOT
+# Remove samples files over 10, (or PURGE_SAMPLE_DAYS if set) days old:
+find ${SAMPLE_ROOT} -type f -ctime +${PURGE_SAMPLE_DAYS:-10} -exec rm {} \;
+#find ${SAMPLE_ROOT} -type f -ctime +${PURGE_SAMPLE_DAYS:-10} -exec ls {} \;
 
-# Check that we can cd in the right directory before we start
-# trying to delete stuff!
+# Remove empty directories:
+find ${SAMPLE_ROOT} -type d -empty -exec rmdir {} \;
+#find $SAMPLE_ROOT -type d -empty -exec ls {} \;
+
+# Compress reports over ZIP_REPORTS_DAYS old:
+if [ -n "${ZIP_REPORTS_DAYS}" ] ; then
 
-if [ "$PWD" != "$SAMPLE_ROOT" ] ; then
+   if [ ! -d "${LOG_ROOT}" ] ; then
+      echo "ERROR: LOG_ROOT not found: ${LOG_ROOT}!"
+      exit 1
+   fi
 
-	echo "ERROR: Unable to cd to $SAMPLE_ROOT!"
-	exit 1
+   REP_EXTN=${REP_EXTN:=TXT}
+   PURGE_REP_EXTN="${REP_EXTN}.gz"
+
+   find ${LOG_ROOT} -type f -ctime +${ZIP_REPORTS_DAYS} -name "*.${REP_EXTN}" -exec ${GZIP} -9 {} \;
 
 fi
 
-# Remove samples files over 7 days old:
-find $SAMPLE_ROOT -type f -ctime +7 -exec rm {} \;
-# find $SAMPLE_ROOT -type f -ctime +7 -exec ls {} \;
+# Delete reports over PURGE_REPORTS_DAYS old:
+if [ ${PURGE_REPORTS_DAYS:=0} -gt 0 ] ; then
 
-# Remove empty directories:
-find $SAMPLE_ROOT -type d -empty -exec rmdir {} \;
-# find $SAMPLE_ROOT -type d -empty -exec ls {} \;
+   if [ ! -d "${LOG_ROOT}" ] ; then
+      echo "ERROR: LOG_ROOT not found: ${LOG_ROOT}!"
+      exit 1
+   fi
+
+   # Safety check before doing mass-delete:
+   cd ${LOG_ROOT}
+   if [ "${PWD}" != "${LOG_ROOT}" ] ; then
+     echo "ERROR: Unable to cd to ${LOG_ROOT}!"
+     exit 1
+   fi
 
+   PURGE_REP_EXTN=${PURGE_REP_EXTN:=$REP_EXTN}
+
+   find ${LOG_ROOT} -type f -ctime +${PURGE_REPORTS_DAYS} -name "*.${PURGE_REP_EXTN}" -exec rm {} \;
+
+fi

conf/config.dist

@@ -101,8 +101,11 @@ DISK_PERCENT_MAX=95   # Max disk use before deleting samples.
 # the old format (no resolving), set BGPOPENS_OLD_FORMAT=1.
 BGPOPENS_OLD_FORMAT=1
 
+# Reports file extension:
+REP_EXTN='TXT'
+
 #
-# Optional tools settings (sponge, auto_sponge, update_ethers)
+# Optional tools settings (sponge, auto_sponge, update_ethers, ixpwatch-tidy)
 #
 MY_NET="5.57.80.0"  # network base address
 MASK="22"           # network CIDR mask

install/install.sh

@@ -24,6 +24,10 @@ DEF_LINK_DIR="/usr/local/bin"
 DEF_DATA_DIR="/var/ixpwatch"
 DEF_CONF_DIR="/etc/ixpwatch"
 
+DEF_PURGE_SAMPLE_DAYS=10
+DEF_ZIP_REPORTS_DAYS=30
+DEF_PURGE_REPORTS_DAYS=0
+
 DEF_LOGGER="/usr/bin/logger"
 DEF_LOGHOSTS="192.168.100.10 192.168.200.20"
 
@@ -135,7 +139,7 @@ INSTALLDIR=$( get_prompt "Script install directory" ${DEF_INSTALL_DIR} )
 
 echo ""
 echo "Data directory setup. Must be a location with sufficient space"
-echo "for example 5-10G for sample and report storage."
+echo "for example 15-20G for sample and report storage."
 echo ""
 
 DATA_DIR=$( get_prompt "Data directory"          "${DEF_DATA_DIR}" )
@@ -147,6 +151,9 @@ TEMP_DIR="${DATA_DIR}/tmp"
 
 DEF_HTML_DIR="${DATA_DIR}/www"
 
+PURGE_SAMPLE_DAYS=$( get_prompt "Days to keep capture sample files"   "${DEF_PURGE_SAMPLE_DAYS}" )
+ZIP_REPORTS_DAYS=$( get_prompt  "Compress reports older than (days)"  "${DEF_ZIP_REPORTS_DAYS}" )
+
 echo ""
 CAP_INTERFACE=$( get_prompt "Peering LAN capture interface"  ${DEF_CAP_INTERFACE} )
 PREFIX_IPV4=$( get_prompt "Peering LAN IPv4 prefix"  ${DEF_PREFIX_IPV4} )