feat: Add ability to list remote available extensions (#3896)

* Split the extension list function into a separate file.

* Add support for listing remote extensions.

* Format files.

* Add 'ListRemoteExtensionsError' error type.

* Address the review comments, mainly keep the behavior consistent with other commands.

* Add e2e tests for listing available extensions.

* Address review comments.

* Update changelog.

CHANGELOG.md

@@ -36,6 +36,12 @@ This applies to the following:
  - tech stack value computation
  - packtool (vessel, mops etc)
 
+### feat: `dfx extension list` supports listing available extensions
+
+`dfx extension list` now support `--available` flag to list available extensions from the
+[extension catalog](https://github.com/dfinity/dfx-extensions/blob/main/catalog.json).
+The extension catalog can be overridden with the `--catalog-url` parameter.
+
 ## Dependencies
 
 ### Frontend canister

e2e/tests-dfx/extension.bash

@@ -767,3 +767,28 @@ EOF
   assert_command dfx extension run test_extension abc --the-another-param 464646 --the-param 123 456 789
   assert_eq "abc --the-another-param 464646 --the-param 123 456 789 --dfx-cache-path $CACHE_DIR"
 }
+
+@test "list available extensions from official catalog" {
+  assert_command dfx extension list --available
+  assert_contains "sns"
+  assert_contains "nns"
+}
+
+@test "list available extensions from customized catalog" {
+  start_webserver --directory www
+  CATALOG_URL_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/catalog.json"
+  mkdir -p  www/arbitrary
+
+    cat > www/arbitrary/catalog.json <<EOF
+{
+  "nns": "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/nns/extension.json",
+  "sns": "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/sns/extension.json",
+  "test": "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/sns/extension.json"
+}
+EOF
+
+  assert_command dfx extension list --catalog-url="$CATALOG_URL_URL"
+  assert_contains "sns"
+  assert_contains "nns"
+  assert_contains "test"
+}

src/dfx-core/src/error/extension.rs

@@ -33,6 +33,12 @@ pub enum ConvertExtensionSubcommandIntoClapCommandError {
     ConvertExtensionSubcommandIntoClapArgError(#[from] ConvertExtensionSubcommandIntoClapArgError),
 }
 
+#[derive(Error, Debug)]
+pub enum ListAvailableExtensionsError {
+    #[error(transparent)]
+    FetchCatalog(#[from] FetchCatalogError),
+}
+
 #[derive(Error, Debug)]
 pub enum ListInstalledExtensionsError {
     #[error(transparent)]

src/dfx-core/src/extension/manager/list.rs

@@ -0,0 +1,47 @@
+use super::ExtensionManager;
+use crate::error::extension::{ListAvailableExtensionsError, ListInstalledExtensionsError};
+use crate::extension::catalog::ExtensionCatalog;
+use crate::extension::installed::InstalledExtensionList;
+use crate::extension::ExtensionName;
+use std::vec;
+use url::Url;
+
+pub type AvailableExtensionList = Vec<ExtensionName>;
+
+impl ExtensionManager {
+    pub fn list_installed_extensions(
+        &self,
+    ) -> Result<InstalledExtensionList, ListInstalledExtensionsError> {
+        if !self.dir.exists() {
+            return Ok(vec![]);
+        }
+        let dir_content = crate::fs::read_dir(&self.dir)?;
+
+        let extensions = dir_content
+            .filter_map(|v| {
+                let dir_entry = v.ok()?;
+                if dir_entry.file_type().map_or(false, |e| e.is_dir())
+                    && !dir_entry.file_name().to_str()?.starts_with(".tmp")
+                {
+                    let name = dir_entry.file_name().to_string_lossy().to_string();
+                    Some(name)
+                } else {
+                    None
+                }
+            })
+            .collect();
+        Ok(extensions)
+    }
+
+    pub async fn list_available_extensions(
+        &self,
+        catalog_url: Option<&Url>,
+    ) -> Result<AvailableExtensionList, ListAvailableExtensionsError> {
+        let catalog = ExtensionCatalog::fetch(catalog_url)
+            .await
+            .map_err(ListAvailableExtensionsError::FetchCatalog)?;
+        let extensions: Vec<String> = catalog.0.into_keys().collect();
+
+        Ok(extensions)
+    }
+}

src/dfx-core/src/extension/manager/mod.rs

@@ -1,19 +1,16 @@
 use crate::config::cache::get_cache_path_for_version;
 use crate::error::extension::{
-    GetExtensionBinaryError, ListInstalledExtensionsError, LoadExtensionManifestsError,
-    NewExtensionManagerError,
-};
-use crate::extension::{
-    installed::{InstalledExtensionList, InstalledExtensionManifests},
-    manifest::ExtensionManifest,
+    GetExtensionBinaryError, LoadExtensionManifestsError, NewExtensionManagerError,
 };
+use crate::extension::{installed::InstalledExtensionManifests, manifest::ExtensionManifest};
 pub use install::InstallOutcome;
 use semver::Version;
 use std::collections::HashMap;
 use std::path::PathBuf;
 
 mod execute;
 mod install;
+mod list;
 mod uninstall;
 
 pub struct ExtensionManager {
@@ -59,30 +56,6 @@ impl ExtensionManager {
         self.get_extension_directory(extension_name).exists()
     }
 
-    pub fn list_installed_extensions(
-        &self,
-    ) -> Result<InstalledExtensionList, ListInstalledExtensionsError> {
-        if !self.dir.exists() {
-            return Ok(vec![]);
-        }
-        let dir_content = crate::fs::read_dir(&self.dir)?;
-
-        let extensions = dir_content
-            .filter_map(|v| {
-                let dir_entry = v.ok()?;
-                if dir_entry.file_type().map_or(false, |e| e.is_dir())
-                    && !dir_entry.file_name().to_str()?.starts_with(".tmp")
-                {
-                    let name = dir_entry.file_name().to_string_lossy().to_string();
-                    Some(name)
-                } else {
-                    None
-                }
-            })
-            .collect();
-        Ok(extensions)
-    }
-
     pub fn load_installed_extension_manifests(
         &self,
     ) -> Result<InstalledExtensionManifests, LoadExtensionManifestsError> {

src/dfx/src/commands/extension/list.rs

@@ -1,22 +1,63 @@
 use crate::lib::environment::Environment;
 use crate::lib::error::DfxResult;
+use clap::Parser;
 use std::io::Write;
+use tokio::runtime::Runtime;
+use url::Url;
 
-pub fn exec(env: &dyn Environment) -> DfxResult<()> {
+#[derive(Parser)]
+pub struct ListOpts {
+    /// Specifies to list the available remote extensions.
+    #[arg(long)]
+    available: bool,
+    /// Specifies the URL of the catalog to use to find the extension.
+    #[clap(long)]
+    catalog_url: Option<Url>,
+}
+
+pub fn exec(env: &dyn Environment, opts: ListOpts) -> DfxResult<()> {
     let mgr = env.get_extension_manager();
-    let extensions = mgr.list_installed_extensions()?;
 
+    if opts.available || opts.catalog_url.is_some() {
+        let runtime = Runtime::new().expect("Unable to create a runtime");
+        let extensions = runtime.block_on(async {
+            mgr.list_available_extensions(opts.catalog_url.as_ref())
+                .await
+        })?;
+
+        display_extension_list(
+            &extensions,
+            "No extensions available.",
+            "Available extensions:",
+        )
+    } else {
+        let extensions = mgr.list_installed_extensions()?;
+
+        display_extension_list(
+            &extensions,
+            "No extensions installed.",
+            "Installed extensions:",
+        )
+    }
+}
+
+fn display_extension_list(
+    extensions: &Vec<String>,
+    empty_msg: &str,
+    header_msg: &str,
+) -> DfxResult<()> {
     if extensions.is_empty() {
-        eprintln!("No extensions installed.");
+        eprintln!("{}", empty_msg);
         return Ok(());
     }
 
-    eprintln!("Installed extensions:");
+    eprintln!("{}", header_msg);
     for extension in extensions {
         eprint!("  ");
         std::io::stderr().flush()?;
         println!("{}", extension);
         std::io::stdout().flush()?;
     }
+
     Ok(())
 }

src/dfx/src/commands/extension/mod.rs

@@ -24,15 +24,15 @@ pub enum SubCommand {
     Uninstall(uninstall::UninstallOpts),
     /// Execute an extension.
     Run(run::RunOpts),
-    /// List installed extensions.
-    List,
+    /// List installed or available extensions.
+    List(list::ListOpts),
 }
 
 pub fn exec(env: &dyn Environment, opts: ExtensionOpts) -> DfxResult {
     match opts.subcmd {
         SubCommand::Install(v) => install::exec(env, v),
         SubCommand::Uninstall(v) => uninstall::exec(env, v),
         SubCommand::Run(v) => run::exec(env, v),
-        SubCommand::List => list::exec(env),
+        SubCommand::List(v) => list::exec(env, v),
     }
 }