fix(agents/sealevel): handle non configured igp account payments (#5075)

### Description

- Handles the case where igp fees are paid to a non-configured igp
account in a backward compatible way.
- Zero out payment for non-matching igp accounts so that the db stays
continuous to prevent an infinite loop in the sequencer aware parser but
at the same time, enforce payments to the right igp account was made
before processing the message.

Note: If we want to display these neutralized payments (in say, the
Explorer), we'll need to think of a more breaking solution, which isn't
necessary today.

### Drive-by changes

None

### Related issues

- fixes hyperlane-xyz/issues#1392

### Backward compatibility

Yes

### Testing

- integration test paying to a different newly deployed account

rust/main/agents/relayer/src/settings/mod.rs

@@ -125,7 +125,7 @@ impl FromRawConf<RawRelayerSettings> for RelayerSettings {
             .parse_from_str("Expected database path")
             .unwrap_or_else(|| std::env::current_dir().unwrap().join("hyperlane_db"));
 
-        // is_gas_payment_enforcement_set determines if we should be checking if the correct gas payment enforcement policy has been provided with "gasPaymentEnforcement" key
+        // is_gas_payment_enforcement_set determines if we should be checking for the correct gas payment enforcement policy has been provided with "gasPaymentEnforcement" key
         let (
             raw_gas_payment_enforcement_path,
             raw_gas_payment_enforcement,

rust/main/chains/hyperlane-sealevel/src/interchain_gas.rs

@@ -248,11 +248,17 @@ impl Indexer<InterchainGasPayment> for SealevelInterchainGasPaymasterIndexer {
         for nonce in range {
             if let Ok(sealevel_payment) = self.get_payment_with_sequence(nonce.into()).await {
                 let igp_account_filter = self.igp.igp_account;
-                if igp_account_filter == sealevel_payment.igp_account_pubkey {
-                    payments.push((sealevel_payment.payment, sealevel_payment.log_meta));
-                } else {
-                    tracing::debug!(sealevel_payment=?sealevel_payment, igp_account_filter=?igp_account_filter, "Found interchain gas payment for a different IGP account, skipping");
+                let mut payment = *sealevel_payment.payment.inner();
+                // If fees is paid to a different IGP account, we zero out the payment to make sure the db entries are contiguous, but at the same time, gasEnforcer will reject the message (if not set to none policy)
+                if igp_account_filter != sealevel_payment.igp_account_pubkey {
+                    tracing::debug!(sealevel_payment=?sealevel_payment, igp_account_filter=?igp_account_filter, "Found interchain gas payment for a different IGP account, neutralizing payment");
+
+                    payment.payment = U256::from(0);
                 }
+                payments.push((
+                    Indexed::new(payment).with_sequence(nonce),
+                    sealevel_payment.log_meta,
+                ));
             }
         }
         Ok(payments)

rust/main/utils/run-locally/src/invariants/common.rs

@@ -1,3 +1,4 @@
 // This number should be even, so the messages can be split into two equal halves
 // sent before and after the relayer spins up, to avoid rounding errors.
 pub const SOL_MESSAGES_EXPECTED: u32 = 20;
+pub const SOL_MESSAGES_WITH_NON_MATCHING_IGP: u32 = 1;

rust/main/utils/run-locally/src/invariants/termination_invariants.rs

@@ -7,7 +7,7 @@ use crate::utils::get_matching_lines;
 use maplit::hashmap;
 use relayer::GAS_EXPENDITURE_LOG_MESSAGE;
 
-use crate::invariants::SOL_MESSAGES_EXPECTED;
+use crate::invariants::common::{SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP};
 use crate::logging::log;
 use crate::solana::solana_termination_invariants_met;
 use crate::{
@@ -30,15 +30,25 @@ pub fn termination_invariants_met(
     } else {
         0
     };
+    let sol_messages_with_non_matching_igp = if config.sealevel_enabled {
+        SOL_MESSAGES_WITH_NON_MATCHING_IGP
+    } else {
+        0
+    };
+
+    // this is total messages expected to be delivered
     let total_messages_expected = eth_messages_expected + sol_messages_expected;
+    let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp;
 
     let lengths = fetch_metric(
         RELAYER_METRICS_PORT,
         "hyperlane_submitter_queue_length",
         &hashmap! {},
     )?;
     assert!(!lengths.is_empty(), "Could not find queue length metric");
-    if lengths.iter().sum::<u32>() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
+    if lengths.iter().sum::<u32>()
+        != ZERO_MERKLE_INSERTION_KATHY_MESSAGES + sol_messages_with_non_matching_igp
+    {
         log!(
             "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}",
             lengths
@@ -131,11 +141,11 @@ pub fn termination_invariants_met(
 
     // TestSendReceiver randomly breaks gas payments up into
     // two. So we expect at least as many gas payments as messages.
-    if gas_payment_events_count < total_messages_expected {
+    if gas_payment_events_count < total_messages_dispatched {
         log!(
             "Relayer has {} gas payment events, expected at least {}",
             gas_payment_events_count,
-            total_messages_expected
+            total_messages_dispatched
         );
         return Ok(false);
     }
@@ -156,12 +166,13 @@ pub fn termination_invariants_met(
     )?
     .iter()
     .sum::<u32>();
-    if dispatched_messages_scraped != total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES
+    if dispatched_messages_scraped
+        != total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES
     {
         log!(
             "Scraper has scraped {} dispatched messages, expected {}",
             dispatched_messages_scraped,
-            total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES,
+            total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES,
         );
         return Ok(false);
     }
@@ -193,7 +204,7 @@ pub fn termination_invariants_met(
         log!(
             "Scraper has scraped {} delivered messages, expected {}",
             delivered_messages_scraped,
-            total_messages_expected
+            total_messages_expected + sol_messages_with_non_matching_igp
         );
         return Ok(false);
     }

rust/main/utils/run-locally/src/main.rs

@@ -465,6 +465,11 @@ fn main() -> ExitCode {
             initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone())
                 .join();
         }
+        initiate_solana_non_matching_igp_paying_transfer(
+            solana_path.clone(),
+            solana_config_path.clone(),
+        )
+        .join();
     }
 
     log!("Setup complete! Agents running in background...");

rust/main/utils/run-locally/src/solana.rs

@@ -73,6 +73,8 @@ const SOLANA_REMOTE_CHAIN_ID: &str = "13376";
 pub const SOLANA_CHECKPOINT_LOCATION: &str =
     "/tmp/test_sealevel_checkpoints_0x70997970c51812dc3a010c7d01b50e0d17dc79c8";
 
+const SOLANA_GAS_ORACLE_CONFIG_FILE: &str =
+    "../sealevel/environments/local-e2e/gas-oracle-configs.json";
 const SOLANA_OVERHEAD_CONFIG_FILE: &str = "../sealevel/environments/local-e2e/overheads.json";
 
 // Install the CLI tools and return the path to the bin dir.
@@ -280,6 +282,7 @@ pub fn start_solana_test_validator(
         .join();
 
     sealevel_client
+        .clone()
         .cmd("validator-announce")
         .cmd("announce")
         .arg("validator", "0x70997970c51812dc3a010c7d01b50e0d17dc79c8")
@@ -291,6 +294,43 @@ pub fn start_solana_test_validator(
         .run()
         .join();
 
+    sealevel_client
+        .clone()
+        .cmd("igp")
+        .cmd("init-igp-account")
+        .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U")
+        .arg("environment", SOLANA_ENV_NAME)
+        .arg("environments-dir", SOLANA_ENVS_DIR)
+        .arg("chain", "sealeveltest1")
+        .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE)
+        .arg("gas-oracle-config-file", SOLANA_GAS_ORACLE_CONFIG_FILE)
+        .arg(
+            "account-salt",
+            "0x0000000000000000000000000000000000000000000000000000000000000001",
+        )
+        .run()
+        .join();
+
+    sealevel_client
+        .cmd("igp")
+        .cmd("init-overhead-igp-account")
+        .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U")
+        .arg("environment", SOLANA_ENV_NAME)
+        .arg("environments-dir", SOLANA_ENVS_DIR)
+        .arg("chain", "sealeveltest1")
+        .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE)
+        .arg("overhead-config-file", SOLANA_OVERHEAD_CONFIG_FILE)
+        .arg(
+            "inner-igp-account",
+            "8EniU8dQaGQ3HWWtT77V7hrksheygvEu6TtzJ3pX1nKM",
+        )
+        .arg(
+            "account-salt",
+            "0x0000000000000000000000000000000000000000000000000000000000000001",
+        )
+        .run()
+        .join();
+
     log!("Local Solana chain started and hyperlane programs deployed and initialized successfully");
 
     (solana_config_path, validator)
@@ -341,6 +381,57 @@ pub fn initiate_solana_hyperlane_transfer(
     message_id
 }
 
+#[apply(as_task)]
+#[allow(clippy::get_first)]
+pub fn initiate_solana_non_matching_igp_paying_transfer(
+    solana_cli_tools_path: PathBuf,
+    solana_config_path: PathBuf,
+) -> String {
+    let sender = Program::new(concat_path(&solana_cli_tools_path, "solana"))
+        .arg("config", solana_config_path.to_str().unwrap())
+        .arg("keypair", SOLANA_KEYPAIR)
+        .cmd("address")
+        .run_with_output()
+        .join()
+        .get(0)
+        .expect("failed to get sender address")
+        .trim()
+        .to_owned();
+
+    let output = sealevel_client(&solana_cli_tools_path, &solana_config_path)
+        .cmd("token")
+        .cmd("transfer-remote")
+        .cmd(SOLANA_KEYPAIR)
+        .cmd("10000000000")
+        .cmd(SOLANA_REMOTE_CHAIN_ID)
+        .cmd(sender) // send to self
+        .cmd("native")
+        .arg("program-id", "CGn8yNtSD3aTTqJfYhUb6s1aVTN75NzwtsFKo1e83aga")
+        .run_with_output()
+        .join();
+    let non_matching_igp_message_id = get_message_id_from_logs(output.clone())
+        .unwrap_or_else(|| panic!("failed to get message id from logs: {:?}", output));
+
+    log!(
+        "paying gas to a different IGP account for message id: {}",
+        non_matching_igp_message_id
+    );
+    sealevel_client(&solana_cli_tools_path, &solana_config_path)
+        .cmd("igp")
+        .cmd("pay-for-gas")
+        .arg("program-id", "GwHaw8ewMyzZn9vvrZEnTEAAYpLdkGYs195XWcLDCN4U")
+        .arg("message-id", non_matching_igp_message_id.clone())
+        .arg("destination-domain", SOLANA_REMOTE_CHAIN_ID)
+        .arg("gas", "100000")
+        .arg(
+            "account-salt",
+            "0x0000000000000000000000000000000000000000000000000000000000000001",
+        )
+        .run()
+        .join();
+    non_matching_igp_message_id
+}
+
 fn get_message_id_from_logs(logs: Vec<String>) -> Option<String> {
     let message_id_regex = Regex::new(r"Dispatched message to \d+, ID 0x([0-9a-fA-F]+)").unwrap();
     for log in logs {

rust/sealevel/client/src/igp.rs

@@ -90,11 +90,24 @@ pub(crate) fn process_igp_cmd(mut ctx: Context, cmd: IgpCmd) {
             let context_dir =
                 create_new_directory(&chain_dir, get_context_dir_name(init.context.as_ref()));
 
-            let artifacts_path = context_dir.join("igp-accounts.json");
+            let artifacts_path = if init.account_salt.is_some() {
+                context_dir.join(format!(
+                    "igp-accounts-{}.json",
+                    init.account_salt.clone().unwrap()
+                ))
+            } else {
+                context_dir.join("igp-accounts.json")
+            };
 
             let existing_artifacts = try_read_json::<IgpAccountsArtifacts>(&artifacts_path).ok();
 
-            let salt = get_context_salt(init.context.as_ref());
+            let salt = init
+                .account_salt
+                .map(|s| {
+                    let salt_str = s.trim_start_matches("0x");
+                    H256::from_str(salt_str).expect("Invalid salt format")
+                })
+                .unwrap_or_else(|| get_context_salt(init.context.as_ref()));
 
             let chain_configs =
                 read_json::<HashMap<String, ChainMetadata>>(&init.chain_config_file);
@@ -123,11 +136,24 @@ pub(crate) fn process_igp_cmd(mut ctx: Context, cmd: IgpCmd) {
             let context_dir =
                 create_new_directory(&chain_dir, get_context_dir_name(init.context.as_ref()));
 
-            let artifacts_path = context_dir.join("igp-accounts.json");
+            let artifacts_path = if init.account_salt.is_some() {
+                context_dir.join(format!(
+                    "igp-accounts-{}.json",
+                    init.account_salt.clone().unwrap()
+                ))
+            } else {
+                context_dir.join("igp-accounts.json")
+            };
 
             let existing_artifacts = try_read_json::<IgpAccountsArtifacts>(&artifacts_path).ok();
 
-            let salt = get_context_salt(init.context.as_ref());
+            let salt = init
+                .account_salt
+                .map(|s| {
+                    let salt_str = s.trim_start_matches("0x");
+                    H256::from_str(salt_str).expect("Invalid salt format")
+                })
+                .unwrap_or_else(|| get_context_salt(init.context.as_ref()));
 
             let chain_configs =
                 read_json::<HashMap<String, ChainMetadata>>(&init.chain_config_file);
@@ -190,7 +216,15 @@ pub(crate) fn process_igp_cmd(mut ctx: Context, cmd: IgpCmd) {
         }
         IgpSubCmd::PayForGas(payment_details) => {
             let unique_gas_payment_keypair = Keypair::new();
-            let salt = H256::zero();
+
+            let salt = payment_details
+                .account_salt
+                .map(|s| {
+                    let salt_str = s.trim_start_matches("0x");
+                    H256::from_str(salt_str).expect("Invalid salt format")
+                })
+                .unwrap_or_else(H256::zero);
+
             let (igp_account, _igp_account_bump) = Pubkey::find_program_address(
                 hyperlane_sealevel_igp::igp_pda_seeds!(salt),
                 &payment_details.program_id,

rust/sealevel/client/src/main.rs

@@ -430,6 +430,8 @@ struct InitIgpAccountArgs {
     context: Option<String>,
     #[arg(long)]
     gas_oracle_config_file: Option<PathBuf>,
+    #[arg(long)]
+    account_salt: Option<String>, // optional salt for deterministic account creation
 }
 
 #[derive(Args)]
@@ -448,6 +450,8 @@ struct InitOverheadIgpAccountArgs {
     context: Option<String>,
     #[arg(long)]
     overhead_config_file: Option<PathBuf>,
+    #[arg(long)]
+    account_salt: Option<String>, // optional salt for deterministic account creation
 }
 
 #[derive(Args)]
@@ -481,6 +485,8 @@ struct PayForGasArgs {
     destination_domain: u32,
     #[arg(long)]
     gas: u64,
+    #[arg(long)]
+    account_salt: Option<String>, // optional salt for paying gas to a deterministically derived account
 }
 
 #[derive(Args)]

rust/sealevel/environments/local-e2e/gas-oracle-configs.json

@@ -0,0 +1,20 @@
+[
+  {
+    "domain": 13375,
+    "gasOracle": {
+      "type": "remoteGasData",
+      "tokenExchangeRate": "10000000000000000000",
+      "gasPrice": "0",
+      "tokenDecimals": 18
+    }
+  },
+  {
+    "domain": 13376,
+    "gasOracle": {
+      "type": "remoteGasData",
+      "tokenExchangeRate": "10000000000000000000",
+      "gasPrice": "0",
+      "tokenDecimals": 18
+    }
+  }
+]