Support dst_N_autocreation ZFS properties

.github/workflows/spelling/expect.txt

@@ -26,6 +26,7 @@ attr
 austingroupbugs
 autocommit
 autoconf
+autocreation
 autom
 automake
 automounting

CHANGES

@@ -11,6 +11,7 @@ znapzend (0.21.3) unstable; urgency=medium
   * Added rc-script and integration documentation for FreeBSD and similar platforms
   * Converted configure.ac and numerous Makefile.am to avoid GNU Make syntax in favor of portability: tested with Solaris/illumos Sun make and with FreeBSD make
   * Extended `--autoCreation` effect (or lack thereof) to newly appearing sub-datasets; added a `--noautoCreation` option to help override configuration file settings (where used)
+  * Introduced `dst_N_autocreation` setting via ZFS properties (per-destination, inheritable)
 
  -- Jim Klimov <[email protected]>  Tue, 12 Mar 2024 13:42:28 +0100
 

bin/znapzend

@@ -48,7 +48,7 @@
 
     if (defined($opts->{sendIntermediates})) {
         if ( (defined($opts->{skipIntermediates})) && ($opts->{skipIntermediates} != 0) ) {
             warn "Options skipIntermediates and sendIntermediates are exclusive; and sendIntermediates wins!";
         }
         # Note that the legacy default is to sendIntermediates.
         # Beware or benefit that this can deliver manually named
@@ -85,6 +85,9 @@
         $opts->{forbidDestRollback} = 0;
     }
 
+    # Note: default is "undef" to use a ZFS property dst_N_autocreation
+    # (lower-case "c" in the name) if present; finally assumes 0 (false)
+    # if not set in any configuration source for a particular dataset.
     if (defined($opts->{noautoCreation})) {
         $opts->{autoCreation} = 0;
         delete $opts->{noautoCreation};
@@ -111,7 +114,7 @@
     } else {
         if (defined($opts->{since})) {
             $opts->{forbidDestRollback} = 1;
             warn "The --since option is used and will try to ensure that the snapshot exists in history of destinations only if an older snapshot is the latest common (would not delete and rewrite subsequent snapshots to make way)"
         }
     }
 

bin/znapzendzetup

@@ -402,6 +402,54 @@ sub main {
 
         last;
     };
+    /^enable-dst-autocreation$/ && do {
+        $opts->{dst} = pop @ARGV;
+        $opts->{src} = pop @ARGV;
+        if (!defined $opts->{src}) {
+            pod2usage(-exitval => 'NOEXIT');
+            die ("ERROR: source argument for option $mainOpt was not provided\n");
+        }
+        if (!defined $opts->{dst}) {
+            pod2usage(-exitval => 'NOEXIT');
+            die ("ERROR: destination argument for option $mainOpt was not provided\n");
+        }
+        $zConfig->enableBackupSetDstAutoCreation($opts->{src}, $opts->{dst})
+            or die "ERROR: cannot enable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n";
+
+        last;
+    };
+    /^disable-dst-autocreation$/ && do {
+        $opts->{dst} = pop @ARGV;
+        $opts->{src} = pop @ARGV;
+        if (!defined $opts->{src}) {
+            pod2usage(-exitval => 'NOEXIT');
+            die ("ERROR: source argument for option $mainOpt was not provided\n");
+        }
+        if (!defined $opts->{dst}) {
+            pod2usage(-exitval => 'NOEXIT');
+            die ("ERROR: destination argument for option $mainOpt was not provided\n");
+        }
+        $zConfig->disableBackupSetDstAutoCreation($opts->{src}, $opts->{dst})
+            or die "ERROR: cannot disable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n";
+
+        last;
+    };
+    /^inherit-dst-autocreation$/ && do {
+        $opts->{dst} = pop @ARGV;
+        $opts->{src} = pop @ARGV;
+        if (!defined $opts->{src}) {
+            pod2usage(-exitval => 'NOEXIT');
+            die ("ERROR: source argument for option $mainOpt was not provided\n");
+        }
+        if (!defined $opts->{dst}) {
+            pod2usage(-exitval => 'NOEXIT');
+            die ("ERROR: destination argument for option $mainOpt was not provided\n");
+        }
+        $zConfig->inheritBackupSetDstAutoCreation($opts->{src}, $opts->{dst})
+            or die "ERROR: cannot disable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n";
+
+        last;
+    };
     /^list$/ && do {
         GetOptions($opts, (@ROOT_EXEC_OPTIONS, qw(recursive|r inherited))) or exit 1;
 
@@ -591,6 +639,12 @@ and where 'command' and its unique options is one of the following:
 
     disable-dst <src_dataset> <DST_key>
 
+    enable-dst-autocreation  <src_dataset> <DST_key>
+
+    disable-dst-autocreation <src_dataset> <DST_key>
+
+    inherit-dst-autocreation  <src_dataset> <DST_key>
+
     list    [--recursive] [--inherited] [src_dataset...]
 
     export  <src_dataset>

lib/ZnapZend.pm

@@ -49,7 +49,7 @@ has pidfile                 => sub { q{} };
 has forcedSnapshotSuffix    => sub { q{} };
 has defaultPidFile          => sub { q{/var/run/znapzend.pid} };
 has terminate               => sub { 0 };
-has autoCreation            => sub { 0 };
+has autoCreation            => sub { undef };
 has timeWarp                => sub { undef };
 has nodelay                 => sub { 0 };
 has skipOnPreSnapCmdFail    => sub { 0 };
@@ -404,14 +404,23 @@ my $refreshBackupPlans = sub {
         #create backup hashes for all destinations
         for (keys %$backupSet){
             my ($key) = /^dst_([^_]+)_plan$/ or next;
+            my $autoCreation = $self->autoCreation;
+            if (!defined($autoCreation)) {
+                # Caller did not require any particular behavior, so
+                # check the ZFS property name (note lower-case "c"):
+                $autoCreation = (exists $backupSet->{"dst_$key" . '_autocreation'} ? $backupSet->{"dst_$key" . '_autocreation'} : undef);
+            }
+            if (!defined($autoCreation)) {
+                $autoCreation = 0;
+            }
 
             #check if destination exists (i.e. is valid) otherwise recheck as dst might be online, now
             if (!$backupSet->{"dst_$key" . '_valid'}){
 
                 $backupSet->{"dst_$key" . '_valid'} =
                     $self->zZfs->dataSetExists($backupSet->{"dst_$key"}) or do {
 
-                    if ($self->autoCreation && !$self->sendRaw) {
+                    if ($autoCreation && !$self->sendRaw) {
                         my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//;
 
                         # check if we can access destination zpool, if so create parent dataset
@@ -429,7 +438,7 @@ my $refreshBackupPlans = sub {
                     $backupSet->{"dst_$key" . '_valid'} or
                         $self->zLog->warn("destination '" . $backupSet->{"dst_$key"}
                             . "' does not exist or is offline. will be rechecked every run..."
-                            . ( $self->autoCreation ? "" : " Consider running znapzend --autoCreation" ) );
+                            . ( $autoCreation ? "" : " Consider running znapzend --autoCreation" ) );
                 };
 
                 $self->zLog->debug('refreshBackupPlans(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
@@ -563,10 +572,23 @@ my $sendRecvCleanup = sub {
         #recheck non valid dst as it might be online, now
         if (!$backupSet->{"dst_$key" . '_valid'}) {
 
+            my $autoCreation = $self->autoCreation;
+            if (!defined($autoCreation)) {
+                # Caller did not require any particular behavior, so
+                # check the ZFS property name (note lower-case "c").
+                # Note we are looking at "root" datasets with a backup
+                # schedule here (enumerated earlier); children if any
+                # would be checked below:
+                $autoCreation = (exists $backupSet->{"dst_$key" . '_autocreation'} ? $backupSet->{"dst_$key" . '_autocreation'} : undef);
+            }
+            if (!defined($autoCreation)) {
+                $autoCreation = 0;
+            }
+
             $backupSet->{"dst_$key" . '_valid'} =
                 $self->zZfs->dataSetExists($backupSet->{"dst_$key"}) or do {
 
-                if ($self->autoCreation && !$self->sendRaw) {
+                if ($autoCreation && !$self->sendRaw) {
                     my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//;
 
                     # check if we can access destination zpool, if so -
@@ -584,12 +606,18 @@ my $sendRecvCleanup = sub {
                         }
                     };
                 }
-                ( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $self->autoCreation) ) or do {
+                ( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $autoCreation) ) or do {
                     my $errmsg = "destination '" . $backupSet->{"dst_$key"}
                         . "' does not exist or is offline; ignoring it for this round...";
-                    $self->zLog->warn($errmsg);
-                    push (@sendFailed, $errmsg);
-                    $thisSendFailed = 1;
+                    # Avoid spamming for every loop cycle, if we do not have
+                    # the dataset and know we do not intend to auto-create it
+                    $self->zLog->warn($errmsg) if ($autoCreation or $self->debug);
+                    if (!$autoCreation) {
+                        $self->zLog->warn("Autocreation is disabled for this dataset or whole run, so skipping without error") if ($self->debug);
+                    } else {
+                        push (@sendFailed, $errmsg);
+                        $thisSendFailed = 1;
+                    }
                     next;
                 };
             };
@@ -607,6 +635,26 @@ my $sendRecvCleanup = sub {
             my $dstDataSet = $srcDataSet;
             $dstDataSet =~ s/^\Q$backupSet->{src}\E/$backupSet->{$dst}/;
 
+            my $autoCreation = $self->autoCreation;
+            if (!defined($autoCreation)) {
+                # Caller did not require any particular behavior, so
+                # check the ZFS property name (note lower-case "c").
+                # Look at properties of this dataset, allow inherited
+                # values. TOTHINK: Get properties once for all tree?
+                my $properties = $self->zZfs->getDataSetProperties($srcDataSet, 0, 1);
+                if ($properties->[0]) {
+                    for my $prop (keys %{$properties->[0]}) {
+                        if ($prop eq "dst_$key" . '_autocreation') {
+                            $autoCreation = (%{$properties->[0]}{$prop} eq "on" ? 1 : 0);
+                            last;
+                        }
+                    }
+                }
+            }
+            if (!defined($autoCreation)) {
+                $autoCreation = 0;
+            }
+
             my $srcDataSetDisabled = (grep (/^\Q$srcDataSet\E$/, @dataSetsExplicitlyDisabled));
             $self->zLog->debug('sending snapshots from ' . $srcDataSet . ' to ' . $dstDataSet .
                 ($srcDataSetDisabled ? ": not enabled, skipped" : ""));
@@ -616,14 +664,20 @@ my $sendRecvCleanup = sub {
 
             # Time to check if the target sub-dataset exists
             # at all (unless we would auto-create one anyway).
-            if (!$self->autoCreation && !$self->sendRaw && !$self->zZfs->dataSetExists($dstDataSet)) {
+            if ((!$autoCreation || !$self->sendRaw) && !($self->zZfs->dataSetExists($dstDataSet))) {
                 my $errmsg = "sub-destination '" . $dstDataSet
                     . "' does not exist or is offline; ignoring it for this round... Consider "
-                    . ( $self->autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " )
+                    . ( $autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " )
                     . "disabling this dataset from znapzend handling.";
-                $self->zLog->warn($errmsg);
-                push (@sendFailed, $errmsg);
-                $thisSendFailed = 1;
+                # Avoid spamming for every loop cycle, if we do not have
+                # the dataset and know we do not intend to auto-create it
+                $self->zLog->warn($errmsg) if ($autoCreation or $self->debug);
+                if (!$autoCreation) {
+                    $self->zLog->warn("Autocreation is disabled for this dataset or whole run, so skipping without error") if ($self->debug);
+                } else {
+                    push (@sendFailed, $errmsg);
+                    $thisSendFailed = 1;
+                }
                 next;
             }
 
@@ -635,7 +689,7 @@ my $sendRecvCleanup = sub {
                         'since=="' . $self->since . '"'.
                         ', skipIntermediates=="' . $self->skipIntermediates . '"' .
                         ', forbidDestRollback=="' . $self->forbidDestRollback . '"' .
-                        ', autoCreation=="' . $self->autoCreation . '"' .
+                        ', autoCreation=="' . ( $autoCreation ? "true" : "false" ) . '"' .
                         ', sendRaw=="' . $self->sendRaw . '"' .
                         ', valid=="' . ( $backupSet->{"dst_$key" . '_valid'} ? "true" : "false" ) . '"' .
                         ', justCreated=="' . ( $backupSet->{"dst_$key" . '_justCreated'} ? "true" : "false" ) . '"'