Big refactor

- merges random subprojects into this repo
- move config handling into it's own module
- remove most placeholders
- interface based aggregators (easier to add more)
- interface based and context aware scheduling, needs more work
- update deps
- go 1.21

.github/ISSUE_TEMPLATE/download-or-update-issue.md

@@ -9,8 +9,8 @@ assignees: ''
 
 **What project were you trying to use? (e.g. Blender, Arch Linux, Gentoo):**
 
-**What failed? (e.g. ISO download failed, `apt upgrade` failed, project is out of sync, Blender download failed):**
+**What failed? (e.g. ISO download failed, `apt upgrade` failed, project is out of sync):**
 
-**When did you first experience the issue:**
+**Are you an official project maintainer?:**
 
-**Please provide any other information:**
+Feel free to add any other details you think are relevant :smile:

.github/ISSUE_TEMPLATE/mirror-request.md

@@ -7,10 +7,8 @@ assignees: ''
 
 ---
 
-## Include full project name, a short name, and the home page url
+## Include full project name, a short name, and the home page url.
 
-## Are you an official maintainer of this project? How would we become an "Official" mirror
+## Please provide documentation on how to mirror the project
 
-## Is there a link to documentation on how to mirror the project
-
-## (Optional) Around how much disk space is required to host the project
+## Are you an official maintainer of this project?

.github/workflows/go.yml

@@ -2,24 +2,58 @@ name: Go
 
 on:
   push:
-    branches: [ master ]
+    branches: [ "main" ]
   pull_request:
-    branches: [ master ]
+    branches: [ "main" ]
 
 jobs:
-
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v4
       with:
-        go-version: 1.18
+        go-version: '1.21'
 
     - name: Build
       run: go build -v ./...
 
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+
+    - name: Verify modfile
+      run: go mod verify
+
+    - name: Vet
+      run: go vet ./...
+
+    - name: Install golint
+      run: go install golang.org/x/lint/golint@latest
+
+    - name: Lint
+      run: golint -set_exit_status ./...
+
+    - name: Check code formatting using gofmt
+      uses: Jerome1337/[email protected]
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+
     - name: Test
-      run: go test -v ./...
+      run: go test -v ./...

README.md

@@ -1,12 +1,12 @@
 # Mirror
 
-Monolithic software for our [mirror](https://mirror.clarkson.edu) that handles the website, tracking, and scheduling systems. We use an influxdb time series database for storage. 
+Monolithic software for our [mirror](https://mirror.clarkson.edu) that handles the website, tracking, and scheduling systems. We use an influxdb time series database for storage.
 
 ![preview](./preview.png)
 
 ## Setup
 
-```
+```cli
 git clone --recurse_submodule [email protected]:COSI-Lab/Mirror.git
 ```
 
@@ -15,17 +15,8 @@ git clone --recurse_submodule [email protected]:COSI-Lab/Mirror.git
 Secrets and some configuration is managed through creating a `.env` file.
 
 ```text
-# "adm" group id. check with "getent group admin"
-# the user running this script should be in the "adm" group
-# so that they can read and write log files.
-ADM_GROUP=
-
-# Discord Webhook URL and id to ping when things panic
-# Omit either and the bot will not communicate with discord
-HOOK_URL=
-PING_ID=
-
-# Maxmind DB token to update the database, omit and we'll only use a local copy if it exists
+# Maxmind DB token to update the database. Omit and we'll only use a local copy if it exists
+# Note: The maxmind DB license requires we use an up-to-date copy
 MAXMIND_LICENSE_KEY=
 
 # InfluxDB Token
@@ -54,9 +45,6 @@ SYNC_DRY_RUN=true
 # Directory to store the rsync log files, if empty then we don't keep logs. It will be created if it doesn't exist.
 RSYNC_LOGS=
 
-# "true" if we should cache the result of executing templates
-WEB_SERVER_CACHE=false
-
 # Secret pull token
 PULL_TOKEN=token
 ```

_typos.toml

@@ -1,5 +1,6 @@
 [default.extend-words]
 templeos = "templeos"
+WRONLY = "WRONLY"
 
 [files]
 extend-exclude = ["scripts"]

aggregator.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"time"
+
+	"github.com/influxdata/influxdb-client-go/v2/api"
+)
+
+// Aggregator is an interface for aggregating a metric `T`
+type Aggregator[T any] interface {
+	// Initialize the aggregator with a starting value from influxdb
+	Init(reader api.QueryAPI) (lastUpdated time.Time, err error)
+
+	// Aggregate adds metric T into the aggregator
+	Aggregate(entry T)
+
+	// Send the aggregated statistics to influxdb
+	Send(writer api.WriteAPI)
+}
+
+// StartAggregator starts the aggregator with the given Aggregator implementation, channel of type T, influxdb QueryAPI and WriteAPI.
+// It returns the lastUpdated time and an error if any occurred during initialization.
+func StartAggregator[T any](aggregator Aggregator[T], c <-chan T, reader api.QueryAPI, writer api.WriteAPI) (lastUpdated time.Time, err error) {
+	lastUpdated, err = aggregator.Init(reader)
+	if err != nil {
+		return lastUpdated, err
+	}
+
+	go func() {
+		ticker := time.NewTicker(time.Minute)
+
+		for {
+			select {
+			case <-ticker.C:
+				aggregator.Send(writer)
+			case entry := <-c:
+				aggregator.Aggregate(entry)
+			}
+		}
+	}()
+
+	return lastUpdated, nil
+}
+
+// NetStat is a commonly used struct for aggregating network statistics
+type NetStat struct {
+	BytesSent int64
+	BytesRecv int64
+	Requests  int64
+}