diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go
index abf7ab496e..c3008b75d3 100644
--- a/pkg/bindings/connection.go
+++ b/pkg/bindings/connection.go
@@ -19,6 +19,7 @@ import (
 	"github.com/blang/semver/v4"
 	"github.com/containers/common/pkg/ssh"
 	"github.com/containers/podman/v5/version"
+	"github.com/containers/storage/pkg/fileutils"
 	"github.com/kevinburke/ssh_config"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/proxy"
@@ -92,9 +93,7 @@ func NewConnection(ctx context.Context, uri string) (context.Context, error) {
 // or unix:///run/podman/podman.sock
 // or ssh://<user>@<host>[:port]/run/podman/podman.sock
 func NewConnectionWithIdentity(ctx context.Context, uri string, identity string, machine bool) (context.Context, error) {
-	var (
-		err error
-	)
+	var err error
 	if v, found := os.LookupEnv("CONTAINER_HOST"); found && uri == "" {
 		uri = v
 	}
@@ -210,15 +209,27 @@ func sshClient(_url *url.URL, uri string, identity string, machine bool) (Connec
 
 		if identity == "" {
 			if val := cfg.Get(alias, "IdentityFile"); val != "" {
+				// we get default IdentityFile value (~/.ssh/identity) every time
+				// checking if we got default
+				defaultIdentityPath := val == ssh_config.Default("IdentityFile")
+
 				identity = strings.Trim(val, "\"")
+
 				if strings.HasPrefix(identity, "~/") {
 					homedir, err := os.UserHomeDir()
 					if err != nil {
 						return connection, fmt.Errorf("failed to find home dir: %w", err)
 					}
+
 					identity = filepath.Join(homedir, identity[2:])
 				}
-				found = true
+
+				// if we have default value but no file exists ignoring identity
+				if err := fileutils.Exists(identity); err != nil && defaultIdentityPath {
+					identity = ""
+				} else {
+					found = true
+				}
 			}
 		}
 
@@ -262,7 +273,8 @@ func sshClient(_url *url.URL, uri string, identity string, machine bool) (Connec
 	connection.Client = &http.Client{
 		Transport: &http.Transport{
 			DialContext: dialContext,
-		}}
+		},
+	}
 	return connection, nil
 }