-
Notifications
You must be signed in to change notification settings - Fork 2
/
maintenance.sh
150 lines (118 loc) · 6.41 KB
/
maintenance.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
# Commands useful for maintenance
ssh nixbitcoin.org
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
## Protocol for coordinating work on the server
# Before changing server state:
# 1. Show logged-in users
who
# 2. If there are other users logged in, notify them about the changes you're
# about to make.
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
# Systemd
systemctl status
systemctl list-jobs
systemctl list-units --failed
## Show messages messages with prio >= error since last boot
journalctl -b -p 3
# Exclude noisy services
journalctl -b -p 3 | egrep -v 'synapse|postfix|sshd'
## Show messages messages with prio >= warning since last boot
journalctl -b -p 4
journalctl -b -p 4 | egrep -v 'synapse|postfix|sshd'
journalctl -f
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
# ZFS
zpool status
zfs list
# show useful stats
zfs list -o space,used,compressratio -t all
zfs get compressratio
# Create manual snapshot.
# This is useful before deploying a major NixOS upgrade.
zfs snapshot rpool/root@pre-update
# Delete snapshot
zfs destroy rpool/root@pre-update
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
# Access webinterfaces of RTL and btcpayserver via SSH
# This can be faster than using onion services.
# Start shell with SSH tunnels to services on nixbitcoin.org.
# The tunnels are automatically closed on shell exit.
systemd-run --user -u ssh-tunnels-nborg -tGd --setenv=PATH="$PATH" --setenv=SHLVL="$SHLVL" bash -c '
ssh -N -o ControlPath=none -L 10000:169.254.1.29:3000 -L 10001:169.254.1.24:23000 nixbitcoin.org&
bash
'
# RTL
gpg --decrypt ../secrets/nix-bitcoin/rtl-password.gpg 2>/dev/null
xdg-open http://localhost:10000
# btcpayserver
gpg --decrypt ../secrets/client-side/btcpayserver-credentials.gpg 2>/dev/null
xdg-open http://localhost:10001/btcpayserver
# Show donations
xdg-open http://localhost:10001/btcpayserver/stores/3omKK79tAqxUSTHhAvXXcc2UocDjQTHXgdD71bzr93Ja/invoices?searchTerm=status%3Asettled
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
# Backups
# https://borgbackup.readthedocs.io/en/stable
# See also: ../deployment/deploy.sh (Restore backups)
journalctl -u borgbackup-job-main -n 50
# show repo stats
borg-job-main info
# list backups
borg-job-main list
## inspect backups
# show stats of last backup
borg-job-main info ::$(borg-job-main list --short | tail -1)
# show first 10 files of last backup
borg-job-main list ::$(borg-job-main list --short | tail -1) | head -10
# show specific paths
borg-job-main list ::$(borg-job-main list --short | tail -1) var/lib/clightning
# diff contents of penultimate backup with the last backup
backups=$(borg-job-main list --short); borg-job-main diff ::$(<<<"$backups" tail -2 | head -1) $(<<<"$backups" tail -1)
## restore files
# restore a path from last backup (dry-run)
borg-job-main extract --dry-run --progress --list ::$(borg-job-main list --short | tail -1) var/lib/clightning
# show specific file content from last backup
borg-job-main extract --stdout ::$(borg-job-main list --short | tail -1) var/lib/clightning/config
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
# Update postgresql database schema
# Required when the postgresql version is updated when changing `system.stateVersion`
# See also: https://nixos.org/manual/nixos/stable/index.html#module-services-postgres-upgrading
# The system state version that postgresql should be updated to
newSystemStateVersion=23.05
# Build script
drv=$(nix eval --raw ../deployment#lib --apply "lib: (lib.postgresql.updateSystem \"$newSystemStateVersion\").drvPath")
script=$(nix build --no-link --print-out-paths $drv)
# Check postgresql versions in script
# See also: https://www.postgresql.org/docs/current/pgupgrade.html
# By using option `--link` for `pg_upgrade`, db files are hardlinked when
# the datadir for the new schema is created
cat $script
# Run script
nix copy --to ssh://nixbitcoin.org $script
time ssh nixbitcoin.org $script
# Now change `system.stateVersion` to $newSystemStateVersion in ../base.nix and deploy
# Check postgresql
ssh nixbitcoin.org 'systemctl status postgresql'
# Run recommended analyzer script
schema=$(nix eval --raw ../deployment#lib.postgresql.systemPostgresqlSchema)
echo $schema
ssh nixbitcoin.org "doas -u postgres /var/lib/postgresql/$schema/analyze_new_cluster.sh"
# Delete old data dir
ssh nixbitcoin.org "doas -u postgres /var/lib/postgresql/$schema/delete_old_cluster.sh"
ssh nixbitcoin.org 'ls -al /var/lib/postgresql'
### Appendix
# The update step runs pretty fast (~10s)
# Example:
# Update the postgresql datadir from schema 11.1 to 14 (2022-07-02)
ssh nixbitcoin.org 'du -sh --apparent /var/lib/postgresql/11.1' # => 47G
# Duration:
# real 0m10.088s
ssh nixbitcoin.org 'ls -al /var/lib/postgresql'
# Undo migration (requires that postgresql was not run with the new datadir.)
ssh nixbitcoin.org 'mv /var/lib/postgresql/<schema>/global/pg_control{.old,}'
#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
# Securely erase hard disks before decommissioning a server
# 1. Run `deployInstallerSystem` (../deployment/deploy.sh) to kexec into the installer system
# 2. Run the following on the installer system:
lsblk # Show disks
shred -v --iterations=1 /dev/sda &
shred -v --iterations=1 /dev/sdb