interdomain-k8s: unexpected healing every minute

[glazychev-art]

Description

If we run an interdomain scenario, wait a while and look at the logs on the client, we will see the healing work:

...
Jan 23 11:18:02.499�[36m [INFO] [id:alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0] [heal:eventLoop] [type:networkService] �[0m(7.5)         Heal success
Jan 23 11:19:00.685�[37m [DEBU] [id:alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0] [heal:eventLoop] [type:networkService] �[0m(7.2)         received eventIn: type:UPDATE  connections:{key:"bdce686c-3652-406f-a264-9fb3ef84d355"  value:{id:"bdce686c-3652-406f-a264-9fb3ef84d355"  network_service:"[email protected]"  mechanism:{cls:"LOCAL"  type:"KERNEL"  parameters:{key:"inodeURL"  value:"inode://4/4026532963"}  parameters:{key:"name"  value:"nsm-1"}}  context:{ip_context:{src_ip_addrs:"172.16.1.3/32"  dst_ip_addrs:"172.16.1.2/32"  src_routes:{prefix:"172.16.1.2/32"}  dst_routes:{prefix:"172.16.1.3/32"}  excluded_prefixes:"10.28.0.0/23"  excluded_prefixes:"10.32.0.1/32"  excluded_prefixes:"10.32.0.10/32"  excluded_prefixes:"10.32.1.81/32"  excluded_prefixes:"10.32.1.103/32"  excluded_prefixes:"10.32.1.222/32"  excluded_prefixes:"10.32.2.178/32"  excluded_prefixes:"10.32.7.29/32"  excluded_prefixes:"10.32.8.37/32"  excluded_prefixes:"10.32.11.164/32"  excluded_prefixes:"10.32.13.66/32"  excluded_prefixes:"10.32.13.226/32"  excluded_prefixes:"10.32.14.53/32"  excluded_prefixes:"10.100.0.1/32"  excluded_prefixes:"10.100.0.10/32"  excluded_prefixes:"10.100.62.231/32"  excluded_prefixes:"10.100.83.14/32"  excluded_prefixes:"10.100.93.207/32"  excluded_prefixes:"10.100.123.114/32"  excluded_prefixes:"10.100.158.4/32"  excluded_prefixes:"10.100.163.229/32"  excluded_prefixes:"10.100.207.92/32"}  dns_context:{}  MTU:1406}  labels:{key:"clusterName"  value:"cluster2"}  labels:{key:"nodeName"  value:"gke-nsm-1-ag-default-pool-27c785aa-f45h"}  labels:{key:"podName"  value:"alpine"}  path:{index:1  path_segments:{name:"alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af"  id:"alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnMtaW50ZXJkb21haW4ta2VybmVsMmV0aGVybmV0Mmtlcm5lbC9wb2QvYWxwaW5lIiwiYXVkIjpbInNwaWZmZTovL25zbS5jbHVzdGVyMS9ucy9uc20tc3lzdGVtL3BvZC9uc21nci1nNWpkNyJdLCJleHAiOjE3MDYwMDkyNzh9.CCfx7WPv2saSzN5LiMdy3C5Snn8RitTFv4eIrefLsYwIu1Ui_jAd7eDNqHC-3_iskPorYxwISuR8YcxQe3GlVQ"  expires:{seconds:1706009278  nanos:808397821}  metrics:{key:"client_interface"  value:"KERNEL/nsm-1"}}  path_segments:{name:"nsmgr-g5jd7"  id:"bdce686c-3652-406f-a264-9fb3ef84d355"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItZzVqZDciLCJhdWQiOlsic3BpZmZlOi8vbnNtLmNsdXN0ZXIxL25zL25zbS1zeXN0ZW0vcG9kL2ZvcndhcmRlci12cHAtOXc1YjciXSwiZXhwIjoxNzA2MDA5Mjc5fQ.fOeTXbgsu8NhXldfXN71W2G1JADpyGX8PUToqzeKCw5ODglBnvtVySga5y-of0KAzEoY8QfVk5NOXzZWLNj4bQ"  expires:{seconds:1706009279  nanos:112378532}}  path_segments:{name:"forwarder-vpp-9w5b7"  id:"e830862c-b8db-475b-b5d6-9b224193f17c"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvZm9yd2FyZGVyLXZwcC05dzViNyIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktNTliNGI1NjliZi1sN21qMiJdLCJleHAiOjE3MDYwMDkyNzl9.vSAUHI-tCV-rEyC1RBYNRtc1B-QWI5V4AYASp8YZF1Rk2qNvaRdU5qCQ5V3Pa_oXTEmHhoTX4oRa8dfGPsKsQQ"  expires:{seconds:1706009279  nanos:837421117}  metrics:{key:"client_drops"  value:"0"}  metrics:{key:"client_interface"  value:"VXLAN/vxlan_tunnel0"}  metrics:{key:"client_rx_bytes"  value:"0"}  metrics:{key:"client_rx_packets"  value:"0"}  metrics:{key:"client_tx_bytes"  value:"0"}  metrics:{key:"client_tx_packets"  value:"0"}  metrics:{key:"server_drops"  value:"0"}  metrics:{key:"server_interface"  value:"AF-PACKET/host-server-bdce686c"}  metrics:{key:"server_rx_bytes"  value:"442"}  metrics:{key:"server_rx_packets"  value:"5"}  metrics:{key:"server_tx_bytes"  value:"0"}  metrics:{key:"server_tx_packets"  value:"0"}}  path_segments:{name:"nsmgr-proxy-59b4b569bf-l7mj2"  id:"85d2153d-9745-4eb0-98ad-183cd2011dbf"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktNTliNGI1NjliZi1sN21qMiIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktN2M2Nzk4ZGRmZC1kNXZwbSJdLCJleHAiOjE3MDYwMDkyNzl9.lV7hWdJEjiPRvj0gPUpewzYkXdBNyNqWGsVf-IpWQMfWrDWQKHXUW0y3uYY4aHR-CpxzPUI2l1UFmL0aeO7TQg"  expires:{seconds:1706009279  nanos:937127505}}  path_segments:{name:"nsmgr-proxy-7c6798ddfd-d5vpm"  id:"e9560470-5026-4d91-886a-3ea6f8aa02ac"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktN2M2Nzk4ZGRmZC1kNXZwbSIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItbGQ3enYiXSwiZXhwIjoxNzA2MDA5Mjc5fQ.nt7b2Ye2f4J4N3-iDFsLXLW5gE8T6jOIyFWPyldGBwlOpIJLgR5oNdXXyT-7xxc3aPu-7UWYYnKBg3Fkfpiglw"  expires:{seconds:1706009279  nanos:976197633}}  path_segments:{name:"nsmgr-ld7zv"  id:"e25f5b05-2d13-40bd-90a0-254bb9ed86a5"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItbGQ3enYiLCJhdWQiOlsic3BpZmZlOi8vbnNtLmNsdXN0ZXIyL25zL25zbS1zeXN0ZW0vcG9kL2ZvcndhcmRlci12cHAta212c2giXSwiZXhwIjoxNzA2MDA5MjgwfQ.-UBzHBS8N3IjEGeL84V_AzIoyFFjkW_-B08g_aQyNyv0yEpyNp-Mrp9X1FIK6AmsMVmPHMRGsfo8LweJcmQvbg"  expires:{seconds:1706009280  nanos:21546695}}  path_segments:{name:"forwarder-vpp-kmvsh"  id:"ee792f4e-9e45-4efb-9fe3-9c8dcfa14be3"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvZm9yd2FyZGVyLXZwcC1rbXZzaCIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnMtaW50ZXJkb21haW4ta2VybmVsMmV0aGVybmV0Mmtlcm5lbC9wb2QvbnNlLWtlcm5lbC1iYmQ3OTdmNWItcjlid3giXSwiZXhwIjoxNzA2MDA5MjgwfQ.Wyr43nzr49edYFIAW0-QoEEYAN9i5RraWNI5DKhzGoI29s9hk90XZaCYHP8_3zkIHaEiD1TpMfLujkvsgWjFSw"  expires:{seconds:1706009280  nanos:93767548}  metrics:{key:"client_drops"  value:"0"}  metrics:{key:"client_interface"  value:"VIRTIO/tap0"}  metrics:{key:"client_rx_bytes"  value:"0"}  metrics:{key:"client_rx_packets"  value:"0"}  metrics:{key:"client_tx_bytes"  value:"0"}  metrics:{key:"client_tx_packets"  value:"0"}  metrics:{key:"server_drops"  value:"0"}  metrics:{key:"server_interface"  value:"VXLAN/vxlan_tunnel0"}  metrics:{key:"server_rx_bytes"  value:"0"}  metrics:{key:"server_rx_packets"  value:"0"}  metrics:{key:"server_tx_bytes"  value:"0"}  metrics:{key:"server_tx_packets"  value:"0"}}  path_segments:{name:"nse-kernel-bbd797f5b-r9bwx"  id:"47253e2d-13c1-4a6e-b447-39089bcab84d"  token:"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnMtaW50ZXJkb21haW4ta2VybmVsMmV0aGVybmV0Mmtlcm5lbC9wb2QvbnNlLWtlcm5lbC1iYmQ3OTdmNWItcjlid3giLCJhdWQiOlsic3BpZmZlOi8vbnNtLmNsdXN0ZXIyL25zL25zbS1zeXN0ZW0vcG9kL2ZvcndhcmRlci12cHAta212c2giXSwiZXhwIjoxNzA2MDA5MjgwfQ.Zfbz6abMRpdf4BLSHZwTmC64GgJ6mUfeuH-65qX1ymgn0PiqoNhw46CUNuQSFz1f-XH_DZtyCQeRdMHCM3SA6Q"  expires:{seconds:1706009280  nanos:96057500}  metrics:{key:"server_interface"  value:"KERNEL/interdomai-4725"}}}  network_service_endpoint_name:"[email protected]"  payload:"ETHERNET"  state:DOWN}}, err: <nil>
Jan 23 11:19:00.685�[33m [WARN] [id:alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0] [heal:eventLoop] [type:networkService] �[0m(7.3)         Control plane is down
Jan 23 11:19:00.686�[37m [DEBU] [id:alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0] [heal:eventLoop] [type:networkService] �[0m(7.4)         Reconnect with reselect
Jan 23 11:19:00.686�[37m [TRAC] [id:alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0] [type:networkService] �[0m(2.39)    close={"id":"alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0","network_service":"[email protected]","mechanism":{"cls":"LOCAL","type":"KERNEL","parameters":{"inodeURL":"file:///proc/thread-self/ns/net","name":"nsm-1"}},"context":{"ip_context":{"src_ip_addrs":["172.16.1.3/32"],"dst_ip_addrs":["172.16.1.2/32"],"src_routes":[{"prefix":"172.16.1.2/32"}],"dst_routes":[{"prefix":"172.16.1.3/32"}],"excluded_prefixes":["10.28.0.0/23","10.32.0.1/32","10.32.0.10/32","10.32.1.81/32","10.32.1.103/32","10.32.1.222/32","10.32.2.178/32","10.32.7.29/32","10.32.8.37/32","10.32.11.164/32","10.32.13.66/32","10.32.13.226/32","10.32.14.53/32","10.100.0.1/32","10.100.0.10/32","10.100.62.231/32","10.100.83.14/32","10.100.93.207/32","10.100.123.114/32","10.100.158.4/32","10.100.163.229/32","10.100.207.92/32"]},"dns_context":{},"MTU":1406},"labels":{"clusterName":"cluster2","nodeName":"gke-nsm-1-ag-default-pool-27c785aa-f45h","podName":"alpine"},"path":{"path_segments":[{"name":"alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af","id":"alpine-b1e010fd-006f-418c-ac6f-c54635c1f9af-0","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnMtaW50ZXJkb21haW4ta2VybmVsMmV0aGVybmV0Mmtlcm5lbC9wb2QvYWxwaW5lIiwiYXVkIjpbInNwaWZmZTovL25zbS5jbHVzdGVyMS9ucy9uc20tc3lzdGVtL3BvZC9uc21nci1nNWpkNyJdLCJleHAiOjE3MDYwMDkyNzh9.CCfx7WPv2saSzN5LiMdy3C5Snn8RitTFv4eIrefLsYwIu1Ui_jAd7eDNqHC-3_iskPorYxwISuR8YcxQe3GlVQ","expires":{"seconds":1706009278,"nanos":808397821},"metrics":{"client_interface":"KERNEL/nsm-1"}},{"name":"nsmgr-g5jd7","id":"bdce686c-3652-406f-a264-9fb3ef84d355","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItZzVqZDciLCJhdWQiOlsic3BpZmZlOi8vbnNtLmNsdXN0ZXIxL25zL25zbS1zeXN0ZW0vcG9kL2ZvcndhcmRlci12cHAtOXc1YjciXSwiZXhwIjoxNzA2MDA5Mjc5fQ.fOeTXbgsu8NhXldfXN71W2G1JADpyGX8PUToqzeKCw5ODglBnvtVySga5y-of0KAzEoY8QfVk5NOXzZWLNj4bQ","expires":{"seconds":1706009279,"nanos":112378532}},{"name":"forwarder-vpp-9w5b7","id":"e830862c-b8db-475b-b5d6-9b224193f17c","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvZm9yd2FyZGVyLXZwcC05dzViNyIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktNTliNGI1NjliZi1sN21qMiJdLCJleHAiOjE3MDYwMDkyNzl9.vSAUHI-tCV-rEyC1RBYNRtc1B-QWI5V4AYASp8YZF1Rk2qNvaRdU5qCQ5V3Pa_oXTEmHhoTX4oRa8dfGPsKsQQ","expires":{"seconds":1706009279,"nanos":837421117},"metrics":{"client_drops":"0","client_interface":"VXLAN/vxlan_tunnel0","client_rx_bytes":"0","client_rx_packets":"0","client_tx_bytes":"0","client_tx_packets":"0","server_drops":"0","server_interface":"AF-PACKET/host-server-bdce686c","server_rx_bytes":"532","server_rx_packets":"6","server_tx_bytes":"0","server_tx_packets":"0"}},{"name":"nsmgr-proxy-59b4b569bf-l7mj2","id":"85d2153d-9745-4eb0-98ad-183cd2011dbf","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjEvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktNTliNGI1NjliZi1sN21qMiIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktN2M2Nzk4ZGRmZC1kNXZwbSJdLCJleHAiOjE3MDYwMDkyNzl9.lV7hWdJEjiPRvj0gPUpewzYkXdBNyNqWGsVf-IpWQMfWrDWQKHXUW0y3uYY4aHR-CpxzPUI2l1UFmL0aeO7TQg","expires":{"seconds":1706009279,"nanos":937127505}},{"name":"nsmgr-proxy-7c6798ddfd-d5vpm","id":"e9560470-5026-4d91-886a-3ea6f8aa02ac","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItcHJveHktN2M2Nzk4ZGRmZC1kNXZwbSIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItbGQ3enYiXSwiZXhwIjoxNzA2MDA5Mjc5fQ.nt7b2Ye2f4J4N3-iDFsLXLW5gE8T6jOIyFWPyldGBwlOpIJLgR5oNdXXyT-7xxc3aPu-7UWYYnKBg3Fkfpiglw","expires":{"seconds":1706009279,"nanos":976197633}},{"name":"nsmgr-ld7zv","id":"e25f5b05-2d13-40bd-90a0-254bb9ed86a5","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvbnNtZ3ItbGQ3enYiLCJhdWQiOlsic3BpZmZlOi8vbnNtLmNsdXN0ZXIyL25zL25zbS1zeXN0ZW0vcG9kL2ZvcndhcmRlci12cHAta212c2giXSwiZXhwIjoxNzA2MDA5MjgwfQ.-UBzHBS8N3IjEGeL84V_AzIoyFFjkW_-B08g_aQyNyv0yEpyNp-Mrp9X1FIK6AmsMVmPHMRGsfo8LweJcmQvbg","expires":{"seconds":1706009280,"nanos":21546695}},{"name":"forwarder-vpp-kmvsh","id":"ee792f4e-9e45-4efb-9fe3-9c8dcfa14be3","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnNtLXN5c3RlbS9wb2QvZm9yd2FyZGVyLXZwcC1rbXZzaCIsImF1ZCI6WyJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnMtaW50ZXJkb21haW4ta2VybmVsMmV0aGVybmV0Mmtlcm5lbC9wb2QvbnNlLWtlcm5lbC1iYmQ3OTdmNWItcjlid3giXSwiZXhwIjoxNzA2MDA5MjgwfQ.Wyr43nzr49edYFIAW0-QoEEYAN9i5RraWNI5DKhzGoI29s9hk90XZaCYHP8_3zkIHaEiD1TpMfLujkvsgWjFSw","expires":{"seconds":1706009280,"nanos":93767548},"metrics":{"client_drops":"0","client_interface":"VIRTIO/tap0","client_rx_bytes":"0","client_rx_packets":"0","client_tx_bytes":"0","client_tx_packets":"0","server_drops":"0","server_interface":"VXLAN/vxlan_tunnel0","server_rx_bytes":"0","server_rx_packets":"0","server_tx_bytes":"0","server_tx_packets":"0"}},{"name":"nse-kernel-bbd797f5b-r9bwx","id":"47253e2d-13c1-4a6e-b447-39089bcab84d","token":"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzcGlmZmU6Ly9uc20uY2x1c3RlcjIvbnMvbnMtaW50ZXJkb21haW4ta2VybmVsMmV0aGVybmV0Mmtlcm5lbC9wb2QvbnNlLWtlcm5lbC1iYmQ3OTdmNWItcjlid3giLCJhdWQiOlsic3BpZmZlOi8vbnNtLmNsdXN0ZXIyL25zL25zbS1zeXN0ZW0vcG9kL2ZvcndhcmRlci12cHAta212c2giXSwiZXhwIjoxNzA2MDA5MjgwfQ.Zfbz6abMRpdf4BLSHZwTmC64GgJ6mUfeuH-65qX1ymgn0PiqoNhw46CUNuQSFz1f-XH_DZtyCQeRdMHCM3SA6Q","expires":{"seconds":1706009280,"nanos":96057500},"metrics":{"server_interface":"KERNEL/interdomai-4725"}}]},"network_service_endpoint_name":"[email protected]","payload":"ETHERNET"}
...

As you can see, the monitor event comes every minute, which causes healing.
Note: in the example above, datapath healing was disabled.

This behavior was not observed on kind clusters.