Merge pull request #381 from foundriesio/detsch-user-rollback

Add user-initiated rollback

docker-e2e-test/e2e-test.py

@@ -284,13 +284,14 @@ def cleanup_tuf_metadata():
 def cleanup_installed_data():
     os.system("""sqlite3 /var/sota/sql.db  "delete from installed_versions;" ".exit" """)
 
-def install_with_separate_steps(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True):
+def install_with_separate_steps(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True, do_reboot=True, do_finalize=True):
     cp = invoke_aklite(['check', '--json', '1'])
     assert cp.returncode == ReturnCodes.CheckinUpdateNewVersion
     verify_callback([("check-for-update-pre", ""), ("check-for-update-post", "OK")])
 
+    previous_version = aklite_current_version()
     if install_rollback or run_rollback:
-        final_version = aklite_current_version()
+        final_version = previous_version
     else:
         final_version = version
 
@@ -333,18 +334,21 @@ def install_with_separate_steps(version, requires_reboot=False, install_rollback
     elif requires_reboot:
         assert cp.returncode == ReturnCodes.InstallNeedsReboot
         verify_callback([("install-pre", ""), ("install-post", "NEEDS_COMPLETION")])
-        sys_reboot()
-        verify_events(version, {
-                ('EcuInstallationStarted', None),
-                ('EcuInstallationApplied', None),
-            })
-        cp = invoke_aklite(['run'])
-        verify_callback([("install-final-pre", ""), ("install-post", "OK")])
         verify_events(version, {
                 ('EcuInstallationStarted', None),
                 ('EcuInstallationApplied', None),
-                ('EcuInstallationCompleted', True),
             })
+        if do_reboot:
+            sys_reboot()
+
+        if do_finalize:
+            cp = invoke_aklite(['run'])
+            verify_callback([("install-final-pre", ""), ("install-post", "OK")])
+            verify_events(version, {
+                    ('EcuInstallationStarted', None),
+                    ('EcuInstallationApplied', None),
+                    ('EcuInstallationCompleted', True),
+                })
     else:
         if run_rollback:
             assert cp.returncode == ReturnCodes.InstallRollbackOk
@@ -370,16 +374,21 @@ def install_with_separate_steps(version, requires_reboot=False, install_rollback
                 ('EcuInstallationCompleted', True),
             })
 
-    assert aklite_current_version() == final_version
+    if (requires_reboot and not do_reboot):
+        assert aklite_current_version() == previous_version, cp.stdout.decode("utf-8")
+    else:
+        assert aklite_current_version() == final_version, cp.stdout.decode("utf-8")
+
     if not explicit_version:
         # Make sure we would not try a new install, after trying to install the latest one
         cp = invoke_aklite(['check', '--json', '1'])
         assert cp.returncode == ReturnCodes.Ok
         verify_callback([("check-for-update-pre", ""), ("check-for-update-post", "OK")])
 
-def install_with_single_step(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True):
+def install_with_single_step(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True, do_reboot=True, do_finalize=True):
+    previous_version = aklite_current_version()
     if install_rollback or run_rollback:
-        final_version = aklite_current_version()
+        final_version = previous_version
     else:
         final_version = version
 
@@ -413,23 +422,25 @@ def install_with_single_step(version, requires_reboot=False, install_rollback=Fa
                 ("download-pre", ""), ("download-post", "OK"),
                 ("install-pre", ""), ("install-post", "NEEDS_COMPLETION"),
                 ])
-        sys_reboot()
-        verify_events(version, {
-                ('EcuDownloadStarted', None),
-                ('EcuDownloadCompleted', True),
-                ('EcuInstallationStarted', None),
-                ('EcuInstallationApplied', None),
-            })
-        cp = invoke_aklite(['run'])
-        # TODO: handle run_rollback
-        verify_callback([("install-final-pre", ""), ("install-post", "OK")])
         verify_events(version, {
                 ('EcuDownloadStarted', None),
                 ('EcuDownloadCompleted', True),
                 ('EcuInstallationStarted', None),
                 ('EcuInstallationApplied', None),
-                ('EcuInstallationCompleted', True),
             })
+        if do_reboot:
+            sys_reboot()
+        if do_finalize:
+            cp = invoke_aklite(['run'])
+            # TODO: handle run_rollback
+            verify_callback([("install-final-pre", ""), ("install-post", "OK")])
+            verify_events(version, {
+                    ('EcuDownloadStarted', None),
+                    ('EcuDownloadCompleted', True),
+                    ('EcuInstallationStarted', None),
+                    ('EcuInstallationApplied', None),
+                    ('EcuInstallationCompleted', True),
+                })
     else:
         if run_rollback:
             assert cp.returncode == ReturnCodes.InstallRollbackOk
@@ -464,19 +475,98 @@ def install_with_single_step(version, requires_reboot=False, install_rollback=Fa
                 ('EcuInstallationStarted', None),
                 ('EcuInstallationCompleted', True),
             })
-    assert aklite_current_version() == final_version
+    if (requires_reboot and not do_reboot):
+        assert aklite_current_version() == previous_version, cp.stdout.decode("utf-8")
+    else:
+        assert aklite_current_version() == final_version, cp.stdout.decode("utf-8")
 
     if not explicit_version:
         # Make sure we would not try a new install, after trying to install the latest one
         cp = invoke_aklite(['check', '--json', '1'])
         assert cp.returncode == ReturnCodes.Ok
         verify_callback([("check-for-update-pre", ""), ("check-for-update-post", "OK")])
 
-def install_version(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True):
+def install_version(version, requires_reboot=False, install_rollback=False, run_rollback=False, explicit_version=True, do_reboot=True, do_finalize=True):
     if single_step:
-        install_with_single_step(version, requires_reboot, install_rollback, run_rollback, explicit_version)
+        install_with_single_step(version, requires_reboot, install_rollback, run_rollback, explicit_version, do_reboot, do_finalize)
+    else:
+        install_with_separate_steps(version, requires_reboot, install_rollback, run_rollback, explicit_version, do_reboot, do_finalize)
+
+
+def do_rollback(version, requires_reboot, installation_in_progress):
+    cp = invoke_aklite(['rollback'])
+
+    # TODO: Check events for previous version when `requires_reboot and installation_in_progress`
+    if requires_reboot:
+        # allowing ReturnCodes.InstallAppsNeedFinalization to workaround an test environment limitation related to getCurrent target
+        assert cp.returncode in [ReturnCodes.InstallNeedsReboot, ReturnCodes.InstallAppsNeedFinalization] , cp.stdout.decode("utf-8")
+        if installation_in_progress:
+            verify_callback([
+                    ("install-post", "FAILED"),
+                    ("install-pre", ""), ("install-post", "NEEDS_COMPLETION"),
+                    ])
+        else:
+            verify_callback([
+                    # ("check-for-update-pre", ""), ("check-for-update-post", "OK"),
+                    ("download-pre", ""), ("download-post", "OK"),
+                    ("install-pre", ""), ("install-post", "NEEDS_COMPLETION"),
+                    ])
+
+        sys_reboot()
+        if installation_in_progress:
+            verify_events(version, {
+                    ('EcuInstallationStarted', None),
+                    ('EcuInstallationApplied', None),
+                })
+        else:
+            verify_events(version, {
+                    ('EcuDownloadStarted', None),
+                    ('EcuDownloadCompleted', True),
+                    ('EcuInstallationStarted', None),
+                    ('EcuInstallationApplied', None),
+                })
+        cp = invoke_aklite(['run'])
+        # TODO: handle run_rollback
+        verify_callback([("install-final-pre", ""), ("install-post", "OK")])
+
+        if installation_in_progress:
+            verify_events(version, {
+                    ('EcuInstallationStarted', None),
+                    ('EcuInstallationApplied', None),
+                    ('EcuInstallationCompleted', True),
+                })
+        else:
+            verify_events(version, {
+                    ('EcuDownloadStarted', None),
+                    ('EcuDownloadCompleted', True),
+                    ('EcuInstallationStarted', None),
+                    ('EcuInstallationApplied', None),
+                    ('EcuInstallationCompleted', True),
+                })
     else:
-        install_with_separate_steps(version, requires_reboot, install_rollback, run_rollback, explicit_version)
+        assert cp.returncode == ReturnCodes.Ok, cp.stdout.decode("utf-8")
+        if installation_in_progress:
+            verify_callback([
+                ("install-post", "FAILED"),
+                ("install-pre", ""), ("install-post", "OK")
+                ])
+            verify_events(version, {
+                ('EcuInstallationStarted', None),
+                ('EcuInstallationCompleted', True),
+            })
+        else:
+            verify_callback([
+                # ("check-for-update-pre", ""), ("check-for-update-post", "OK"),
+                ("download-pre", ""), ("download-post", "OK"),
+                ("install-pre", ""), ("install-post", "OK")
+                ])
+            verify_events(version, {
+                ('EcuDownloadStarted', None),
+                ('EcuDownloadCompleted', True),
+                ('EcuInstallationStarted', None),
+                ('EcuInstallationCompleted', True),
+            })
+    assert aklite_current_version() == version, cp.stdout.decode("utf-8")
 
 def restore_system_state():
     # Get to the starting point
@@ -597,3 +687,42 @@ def test_update_to_latest(offline_, single_step_):
     offline = offline_
     single_step = single_step_
     run_test_sequence_update_to_latest()
+
+def run_test_rollback(do_reboot, do_finalize):
+    restore_system_state()
+    apps = None # All apps, for now
+    write_settings(apps, prune)
+
+    # Install
+    logger.info("Installing base target for rollback operations")
+    target = all_targets[Targets.WorkingOstree]
+    install_version(get_target_version(target.version_offset), True, target.install_rollback, target.run_rollback)
+
+    # Test a rollback that does *not* require a reboot
+    logger.info("Testing rollback not requiring reboot")
+    target = all_targets[Targets.AddFirstApp]
+    install_version(get_target_version(target.version_offset), False, target.install_rollback, target.run_rollback, True, do_reboot, do_finalize)
+    do_rollback(get_target_version(Targets.WorkingOstree), False, False)
+
+    # Test a rollback that *does* require a reboot
+    logger.info("Testing rollback requiring reboot")
+    target = all_targets[Targets.UpdateOstreeWithApps]
+    install_version(get_target_version(target.version_offset), True, target.install_rollback, target.run_rollback, True, do_reboot, do_finalize)
+    do_rollback(get_target_version(Targets.WorkingOstree), do_reboot, not do_reboot or not do_finalize)
+
+    # Do a new rollback, go back to "First" target, from the original system stare
+    logger.info("Performing additional rollback operation, on already rolled back environment")
+    do_rollback(get_target_version(Targets.First), True, False)
+
+@pytest.mark.parametrize('offline_', [True, False])
+@pytest.mark.parametrize('single_step_', [True, False])
+@pytest.mark.parametrize('do_reboot', [True, False])
+@pytest.mark.parametrize('do_finalize', [True, False])
+def test_rollback(do_reboot, do_finalize, offline_, single_step_):
+    if not do_reboot and do_finalize:
+        return
+    global offline, single_step
+    offline = offline_
+    single_step = single_step_
+    logger.info(f"Testing rollback {do_reboot=} {do_finalize=}")
+    run_test_rollback(do_reboot, do_finalize)

include/aktualizr-lite/aklite_client_ext.h

@@ -85,6 +85,8 @@ class AkliteClientExt : public AkliteClient {
                                const LocalUpdateSource *local_update_source = nullptr, bool do_download = true,
                                bool do_install = true, bool require_target_in_tuf = true);
 
+  InstallResult Rollback(const LocalUpdateSource *local_update_source);
+
   bool RebootIfRequired();
 
  private:

include/aktualizr-lite/api.h

@@ -301,7 +301,7 @@ class AkliteClient {
    * able to start it's Apps after rebooting from an ostree change. This
    * situation is only possible when `pacman.create_containers_before_reboot = 0`.
    */
-  TufTarget GetRollbackTarget() const;
+  TufTarget GetRollbackTarget(bool ignore_ostree_hash = false, bool allow_current = true) const;
 
   /**
    * Check in with device-gateway to get server managed information about

include/aktualizr-lite/cli/cli.h

@@ -86,6 +86,8 @@ StatusCode Install(AkliteClientExt &client, int version = -1, const std::string
 
 StatusCode CompleteInstall(AkliteClient &client);
 
+StatusCode Rollback(AkliteClientExt &client, const LocalUpdateSource *local_update_source);
+
 bool IsSuccessCode(StatusCode status);
 
 std::string StatusCodeDescription(StatusCode status);

src/aklite_client_ext.cc

@@ -8,6 +8,7 @@
 #include <boost/uuid/uuid_generators.hpp>
 #include <boost/uuid/uuid_io.hpp>
 
+#include "aktualizr-lite/api.h"
 #include "http/httpclient.h"
 #include "libaktualizr/config.h"
 #include "liteclient.h"
@@ -282,3 +283,49 @@ bool AkliteClientExt::RebootIfRequired() {
 
   return false;
 }
+
+InstallResult AkliteClientExt::Rollback(const LocalUpdateSource* local_update_source) {
+  const auto current = GetCurrent();
+  // Getting Uptane::Target instance in order to use correlation_id, which is not available in TufTarget class
+  auto pending_target = client_->getPendingTarget();
+  auto installation_in_progress = pending_target.IsValid();
+  const auto& bad_target = installation_in_progress ? Target::toTufTarget(pending_target) : current;
+
+  LOG_DEBUG << "User initiated rollback. Current Target is " << current.Name();
+  if (installation_in_progress) {
+    LOG_DEBUG << "Target installation is in progress:  " << pending_target.filename();
+  }
+
+  auto storage = INvStorage::newStorage(client_->config.storage, false);
+  LOG_INFO << "Marking target " << bad_target.Name() << " as a failing target";
+  storage->saveInstalledVersion("", Target::fromTufTarget(bad_target), InstalledVersionUpdateMode::kBadTarget);
+
+  // Get rollback target
+  auto rollback_target = GetRollbackTarget(true, installation_in_progress);
+  if (rollback_target.IsUnknown()) {
+    LOG_ERROR << "Failed to find Target to rollback to";
+    return {InstallResult::Status::Failed};
+  }
+
+  if (installation_in_progress) {
+    // Previous installation was not finalized
+    LOG_INFO << "Creating new installation log entry for " << pending_target.filename()
+             << ", as we try to rollback to it";
+    storage->saveInstalledVersion("", pending_target, InstalledVersionUpdateMode::kNone);
+  }
+
+  std::string reason = "User initiated rollback. Marked " + bad_target.Name() +
+                       " as a failing target, and rolling back to " + rollback_target.Name();
+  LOG_INFO << reason;
+
+  if (installation_in_progress) {
+    LOG_INFO << "Generating installation failed event / callback for Target " << pending_target.filename();
+    data::InstallationResult result(data::ResultCode::Numeric::kInstallFailed, reason);
+    client_->notifyInstallFinished(pending_target, result);
+  }
+
+  // If there is an installation in progress, do not perform download, and don't require target to be in TUF targets
+  auto ret = PullAndInstall(rollback_target, reason, "", InstallMode::All, local_update_source,
+                            !installation_in_progress, true, !installation_in_progress);
+  return ret;
+}

src/api.cc

@@ -1113,7 +1113,9 @@ InstallResult AkliteClient::CompleteInstallation() {
   return complete_install_res;
 }
 
-TufTarget AkliteClient::GetRollbackTarget() const { return Target::toTufTarget(client_->getRollbackTarget()); }
+TufTarget AkliteClient::GetRollbackTarget(bool ignore_ostree_hash, bool allow_current) const {
+  return Target::toTufTarget(client_->getRollbackTarget(ignore_ostree_hash, allow_current));
+}
 
 bool AkliteClient::IsRollback(const TufTarget& t) const {
   Json::Value target_json;

src/cli/cli.cc

@@ -384,4 +384,9 @@ StatusCode CompleteInstall(AkliteClient &client) {
   return res2StatusCode<InstallResult::Status>(i2s, ir.status);
 }
 
+StatusCode Rollback(AkliteClientExt &client, const LocalUpdateSource *local_update_source) {
+  auto install_result = client.Rollback(local_update_source);
+  return res2StatusCode<InstallResult::Status>(i2s, install_result.status);
+}
+
 }  // namespace aklite::cli

src/liteclient.cc

@@ -243,18 +243,19 @@ bool LiteClient::finalizeInstall(data::InstallationResult* ir) {
   return ret.result_code.num_code == data::ResultCode::Numeric::kOk;
 }
 
-Uptane::Target LiteClient::getRollbackTarget() {
-  const auto rollback_hash{sysroot_->getDeploymentHash(OSTree::Sysroot::Deployment::kRollback)};
+Uptane::Target LiteClient::getRollbackTarget(bool ignore_ostree_hash, bool allow_current) {
+  const auto rollback_hash{ignore_ostree_hash ? ""
+                                              : sysroot_->getDeploymentHash(OSTree::Sysroot::Deployment::kRollback)};
 
   Uptane::Target rollback_target{Uptane::Target::Unknown()};
   {
     std::vector<Uptane::Target> installed_versions;
-    storage->loadPrimaryInstallationLog(&installed_versions,
-                                        true /* make sure that Target has been successfully installed */);
+    storage->loadPrimaryInstallationLog(
+        &installed_versions, true /* make sure that Target has been successfully installed */, allow_current);
 
     std::vector<Uptane::Target>::reverse_iterator it;
     for (it = installed_versions.rbegin(); it != installed_versions.rend(); it++) {
-      if (it->sha256Hash() == rollback_hash) {
+      if (ignore_ostree_hash || it->sha256Hash() == rollback_hash) {
         rollback_target = *it;
         break;
       }

src/liteclient.h

@@ -49,7 +49,7 @@ class LiteClient {
   void checkForUpdatesEnd(const Uptane::Target& target);
   void checkForUpdatesEndWithFailure(const std::string& err);
   bool finalizeInstall(data::InstallationResult* ir = nullptr);
-  Uptane::Target getRollbackTarget();
+  Uptane::Target getRollbackTarget(bool ignore_ostree_hash = false, bool allow_current = true);
   DownloadResult download(const Uptane::Target& target, const std::string& reason);
   data::ResultCode::Numeric install(const Uptane::Target& target, InstallMode install_mode = InstallMode::All);
   void notifyInstallFinished(const Uptane::Target& t, data::InstallationResult& ir);