From 8b9385eebb4149cdb247b37459c4082882102832 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Mon, 6 Jan 2025 05:39:15 +0330 Subject: [PATCH 01/19] chore(): add check functions --- internal/check/fucntion_test.go | 73 +++++++++++++++++++ internal/check/function.go | 124 ++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 internal/check/fucntion_test.go create mode 100644 internal/check/function.go diff --git a/internal/check/fucntion_test.go b/internal/check/fucntion_test.go new file mode 100644 index 0000000..59128dc --- /dev/null +++ b/internal/check/fucntion_test.go @@ -0,0 +1,73 @@ +package check + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDomainValidator(t *testing.T) { + tests := []struct { + name string + domain string + expected bool + }{ + { + name: "Valid domain", + domain: "example.com", + expected: true, + }, + { + name: "Valid subdomain", + domain: "sub.example.com", + expected: true, + }, + { + name: "Invalid domain - hyphen at start", + domain: "-invalid.com", + expected: false, + }, + { + name: "Invalid domain - hyphen at end", + domain: "invalid-.com", + expected: false, + }, + { + name: "Invalid domain - missing top-level domain", + domain: "example", + expected: false, + }, + { + name: "Invalid domain - double dots", + domain: "invalid..com", + expected: false, + }, + { + name: "Valid domain with hyphens", + domain: "valid-domain.org", + expected: true, + }, + { + name: "Invalid domain - too long", + domain: "toolongdomainnamethatiswaylongerthanthemaximumallowedlengthof253charactersandshouldfailvalidationbecauseitistoolongandexceedsthelimit.toolongdomainnamethatiswaylongerthanthemaximumallowedlengthof253charactersandshouldfailvalidationbecauseitistoolongandexceedsthelimit.toolongdomainnamethatiswaylongerthanthemaximumallowedlengthof253charactersandshouldfailvalidationbecauseitistoolongandexceedsthelimit.toolongdomainnamethatiswaylongerthanthemaximumallowedlengthof253charactersandshouldfailvalidationbecauseitistoolongandexceedsthelimit", + expected: false, + }, + { + name: "Invalid domain - starts with dot", + domain: ".invalid", + expected: false, + }, + { + name: "Invalid domain - ends with dot", + domain: "invalid.", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := DomainValidator(tt.domain) + assert.Equal(t, tt.expected, result, "Test case: %s", tt.name) + }) + } +} diff --git a/internal/check/function.go b/internal/check/function.go new file mode 100644 index 0000000..692106b --- /dev/null +++ b/internal/check/function.go @@ -0,0 +1,124 @@ +package check + +import ( + "context" + "fmt" + "log" + "net" + "net/http" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/urfave/cli/v2" +) + +func ChangeDNS(dns string) *http.Client { + dialer := &net.Dialer{} + customResolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + dnsServer := fmt.Sprintf("%s:53", dns) + log.Printf("Using DNS server: %s\n", dnsServer) + return dialer.DialContext(ctx, "udp", dnsServer) + }, + } + customDialer := &net.Dialer{ + Resolver: customResolver, + } + transport := &http.Transport{ + DialContext: customDialer.DialContext, + } + client := &http.Client{ + Transport: transport, + } + return client +} + +func CheckWithDNS(c *cli.Context) error { + url := c.Args().First() + dnsList, err := ReadDNSFromFile("config/dns.conf") + if err != nil { + fmt.Println(err) + return err + } + + var wg sync.WaitGroup + for _, dns := range dnsList { + wg.Add(1) + go func(dns string) { + defer wg.Done() + + client := ChangeDNS(dns) + + hostname := strings.TrimPrefix(url, "https://") + hostname = strings.TrimPrefix(hostname, "http://") + hostname = strings.Split(hostname, "/")[0] + + startTime := time.Now() + ips, err := net.LookupIP(hostname) + if err != nil { + log.Printf("Failed to resolve hostname %s with DNS %s: %v\n", hostname, dns, err) + return + } + resolutionTime := time.Since(startTime) + + log.Printf("Resolved IPs for %s: %v (DNS: %s)\n", hostname, ips, dns) + log.Printf("DNS resolution took: %v\n", resolutionTime) + + resp, err := client.Get(url) + if err != nil { + log.Printf("Failed to fetch URL %s with DNS %s: %v\n", url, dns, err) + return + } + defer resp.Body.Close() + + log.Printf("Response status for %s (DNS: %s): %s\n", url, dns, resp.Status) + }(dns) + } + + wg.Wait() + return nil +} + +func ReadDNSFromFile(filename string) ([]string, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + dnsServers := strings.Fields(string(data)) + return dnsServers, nil +} +func DomainValidator(domain string) bool { + + // Regular expression to validate domain names + // This regex ensures: + // - The domain contains only alphanumeric characters, hyphens, and dots. + // - It does not start or end with a hyphen or dot. + // - It has at least one dot. + domainRegex := `^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$` + + // Match the domain against the regex + match, _ := regexp.MatchString(domainRegex, domain) + if !match { + return false + } + + // Additional checks: + // 1. The total length of the domain should not exceed 253 characters. + if len(domain) > 253 { + return false + } + + // 2. Each segment between dots should be between 1 and 63 characters long. + segments := strings.Split(domain, ".") + for _, segment := range segments { + if len(segment) < 1 || len(segment) > 63 { + return false + } + } + + return true +} From b040a07a9a62af16cca101323ebff76b4a44bd85 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Mon, 6 Jan 2025 05:39:40 +0330 Subject: [PATCH 02/19] chore(): add dns functions --- internal/dns/function.go | 28 +++++++++++++++ internal/dns/function_test.go | 64 +++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 internal/dns/function.go create mode 100644 internal/dns/function_test.go diff --git a/internal/dns/function.go b/internal/dns/function.go new file mode 100644 index 0000000..e6650bf --- /dev/null +++ b/internal/dns/function.go @@ -0,0 +1,28 @@ +package dns + +import ( + "net/url" + + "github.com/urfave/cli/v2" +) + +func URLValidator(URL string) bool { + // Parse the URL + u, err := url.Parse(URL) + if err != nil { + return false + } + // Check if the scheme is either "http" or "https" + if u.Scheme != "http" && u.Scheme != "https" { + return false + } + // Check if the host is present + if u.Host == "" { + return false + } + return true +} + +func CheckWithURL(c *cli.Context) error { + return nil +} diff --git a/internal/dns/function_test.go b/internal/dns/function_test.go new file mode 100644 index 0000000..d2bfc6a --- /dev/null +++ b/internal/dns/function_test.go @@ -0,0 +1,64 @@ +package dns + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func TestIsValidHTTPURL(t *testing.T) { + tests := []struct { + name string + url string + expected bool + }{ + { + name: "Valid HTTPS URL", + url: "https://www.example.com", + expected: true, + }, + { + name: "Valid HTTP URL", + url: "http://example.com", + expected: true, + }, + { + name: "Valid HTTP URL with IP", + url: "http://192.168.1.1", + expected: true, + }, + { + name: "Valid HTTP URL with port", + url: "http://localhost:8080", + expected: true, + }, + { + name: "Invalid URL - FTP scheme", + url: "ftp://example.com", + expected: false, + }, + { + name: "Invalid URL - Missing scheme", + url: "www.example.com", + expected: false, + }, + { + name: "Invalid URL - Missing host", + url: "https://", + expected: false, + }, + { + name: "Invalid URL - Empty string", + url: "", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + result := URLValidator(tt.url) + assert.Equal(t, tt.expected, result, "Test case: %s", tt.name) + }) + } +} From 44e92d8cfae61b4319156ac0607e1f54bbfc1247 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Mon, 6 Jan 2025 05:39:56 +0330 Subject: [PATCH 03/19] chore(): add docker functions --- internal/docker/fucntion_test.go | 37 ++++++++++++++++++ internal/docker/function.go | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 internal/docker/fucntion_test.go create mode 100644 internal/docker/function.go diff --git a/internal/docker/fucntion_test.go b/internal/docker/fucntion_test.go new file mode 100644 index 0000000..5945e32 --- /dev/null +++ b/internal/docker/fucntion_test.go @@ -0,0 +1,37 @@ +package docker + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateDockerImageName(t *testing.T) { + tests := []struct { + name string + image string + expected bool + }{ + // Valid image names + {"Valid image name without registry or tag", "ubuntu", true}, + {"Valid image name with namespace", "library/ubuntu", true}, + {"Valid image name with registry", "docker.io/library/ubuntu", true}, + {"Valid image name with custom registry and port", "localhost:5000/myproject/ubuntu", true}, + {"Valid image name with tag", "myregistry/myproject/ubuntu:latest", true}, + {"Valid image name with digest", "myregistry/myproject/ubuntu@sha256:abc123", true}, + + // Invalid image names + {"Invalid image name with uppercase letters in repository", "MyRegistry/MyProject/Ubuntu", false}, + {"Invalid image name with invalid characters", "invalid!@#/image/name", false}, + {"Invalid image name with empty repository", "", false}, + {"Invalid image name with only slashes", "///", false}, + {"Invalid image name with multiple @ symbols", "invalid@image@name", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := DockerImageValidator(tt.image) + assert.Equal(t, tt.expected, result, "Test case: %s", tt.name) + }) + } +} diff --git a/internal/docker/function.go b/internal/docker/function.go new file mode 100644 index 0000000..888db7b --- /dev/null +++ b/internal/docker/function.go @@ -0,0 +1,65 @@ +package docker + +import ( + "context" + "fmt" + "log" + "regexp" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/urfave/cli/v2" +) + +func DockerImageValidator(imageName string) bool { + // Regular expression to match a valid Docker image name + // This pattern allows for optional registry, namespace, and tag/digest + pattern := `^(?:[a-zA-Z0-9\-._]+(?::[0-9]+)?/)?` + // Optional registry (e.g., docker.io, localhost:5000) + `(?:[a-z0-9\-._]+/)?` + // Optional namespace (e.g., library, user) + `[a-z0-9\-._]+` + // Repository name (required) + `(?::[a-zA-Z0-9\-._]+)?` + // Optional tag (e.g., latest, v1.0) + `(?:@[a-zA-Z0-9\-._:]+)?$` // Optional digest (e.g., sha256:...) + + // Compile the regular expression + regex := regexp.MustCompile(pattern) + + // Check if the image name matches the pattern + return regex.MatchString(imageName) && !strings.Contains(imageName, "@@") +} + +// downloadDockerImage downloads a Docker image from a registry and saves it as a tarball. +func DownloadDockerImage(imageName, registry, outputPath string, timeoutSeconds int) error { + imageName = registry + "/" + imageName + outputPath = outputPath + "/" + imageName + // Parse the image reference (e.g., "ubuntu:latest") + ref, err := name.ParseReference(imageName) + if err != nil { + return fmt.Errorf("failed to parse image reference: %v", err) + } + // Authenticate with the registry (defaults to anonymous auth) + auth := authn.DefaultKeychain + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second) + defer cancel() + img, err := remote.Image(ref, remote.WithAuthFromKeychain(auth), remote.WithContext(ctx)) + if err != nil { + if ctx.Err() == context.DeadlineExceeded { + log.Printf("Download timed out after %d seconds, saving partially downloaded image...\n", timeoutSeconds) + } else { + return fmt.Errorf("failed to fetch image: %v", err) + } + } + // Save the image as a tarball + err = tarball.WriteToFile(outputPath, ref, img) + if err != nil { + return fmt.Errorf("failed to save image as tarball: %v", err) + } + fmt.Printf("Image successfully downloaded and saved to %s\n", outputPath) + return nil +} +func CheckWithDockerImage(c *cli.Context) error { + return nil +} From 8782942b343a2653514bf7249b8f926d7008f24c Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Mon, 6 Jan 2025 05:41:36 +0330 Subject: [PATCH 04/19] chore(): refactor unlocker cli --- go.mod | 37 ++++++++---- go.sum | 90 +++++++++++++++++++++------- internal/unlockercli/cli.go | 115 +++--------------------------------- 3 files changed, 102 insertions(+), 140 deletions(-) diff --git a/go.mod b/go.mod index 6c469c9..28d493f 100644 --- a/go.mod +++ b/go.mod @@ -2,21 +2,34 @@ module github.com/salehborhani/403Unlocker-cli go 1.23.1 -require github.com/knadh/koanf/v2 v2.1.1 +require ( + github.com/google/go-containerregistry v0.20.2 + github.com/stretchr/testify v1.8.4 + github.com/urfave/cli/v2 v2.27.5 + gotest.tools/v3 v3.0.3 +) require ( - github.com/cavaliergopher/grab/v3 v3.0.1 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/knadh/koanf/maps v0.1.1 // indirect - github.com/knadh/koanf/providers/env v1.0.0 // indirect - github.com/knadh/koanf/providers/file v1.1.2 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/cli v27.1.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/urfave/cli/v2 v2.27.5 // indirect + github.com/sirupsen/logrus v1.9.1 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.15.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4e09dcb..1b87b89 100644 --- a/go.sum +++ b/go.sum @@ -1,38 +1,84 @@ -github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= -github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= -github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= -github.com/knadh/koanf/providers/env v1.0.0 h1:ufePaI9BnWH+ajuxGGiJ8pdTG0uLEUWC7/HDDPGLah0= -github.com/knadh/koanf/providers/env v1.0.0/go.mod h1:mzFyRZueYhb37oPmC1HAv/oGEEuyvJDA98r3XAa8Gak= -github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUkxDOe+2nQY3w= -github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI= -github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= -github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= +github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= +github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= +github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/internal/unlockercli/cli.go b/internal/unlockercli/cli.go index c4af33f..42635f3 100644 --- a/internal/unlockercli/cli.go +++ b/internal/unlockercli/cli.go @@ -1,16 +1,13 @@ package unlockercli import ( - "context" "fmt" "log" - "net" - "net/http" "os" - "strings" - "sync" - "time" + "github.com/salehborhani/403Unlocker-cli/internal/check" + "github.com/salehborhani/403Unlocker-cli/internal/dns" + "github.com/salehborhani/403Unlocker-cli/internal/docker" "github.com/urfave/cli/v2" ) @@ -25,8 +22,8 @@ func Run() { Aliases: []string{"c"}, Usage: "Checks if the DNS SNI-Proxy can bypass 403 error for an specific domain", Action: func(cCtx *cli.Context) error { - if URLValidator(cCtx.Args().First()) { - return CheckWithDNS(cCtx) + if check.DomainValidator(cCtx.Args().First()) { + return check.CheckWithDNS(cCtx) } else { fmt.Println("need a valid domain example: https://pkg.go.dev") } @@ -38,8 +35,8 @@ func Run() { Aliases: []string{"d"}, Usage: "Finds the fastest docker registries for an specific docker image", Action: func(cCtx *cli.Context) error { - if DockerImageValidator(cCtx.Args().First()) { - return CheckWithDockerImage(cCtx) + if docker.DockerImageValidator(cCtx.Args().First()) { + return docker.CheckWithDockerImage(cCtx) } else { fmt.Println("need a valid docker image example: gitlab/gitlab-ce:17.0.0-ce.0") } @@ -50,8 +47,8 @@ func Run() { Name: "dns", Usage: "Finds the fastest DNS SNI-Proxy for downloading an specific URL", Action: func(cCtx *cli.Context) error { - if URLValidator(cCtx.Args().First()) { - return CheckWithURL(cCtx) + if dns.URLValidator(cCtx.Args().First()) { + return dns.CheckWithURL(cCtx) } else { fmt.Println("need a valid URL example: \"https://packages.gitlab.com/gitlab/gitlab-ce/packages/el/7/gitlab-ce-16.8.0-ce.0.el7.x86_64.rpm/download.rpm\"") } @@ -64,97 +61,3 @@ func Run() { log.Fatal(err) } } - -func ChangeDNS(dns string) *http.Client { - dialer := &net.Dialer{} - customResolver := &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - dnsServer := fmt.Sprintf("%s:53", dns) - log.Printf("Using DNS server: %s\n", dnsServer) - return dialer.DialContext(ctx, "udp", dnsServer) - }, - } - customDialer := &net.Dialer{ - Resolver: customResolver, - } - transport := &http.Transport{ - DialContext: customDialer.DialContext, - } - client := &http.Client{ - Transport: transport, - } - return client -} - -func CheckWithDNS(c *cli.Context) error { - url := c.Args().First() - dnsList, err := ReadDNSFromFile("config/dns.conf") - if err != nil { - fmt.Println(err) - return err - } - - var wg sync.WaitGroup - for _, dns := range dnsList { - wg.Add(1) - go func(dns string) { - defer wg.Done() - - client := ChangeDNS(dns) - - hostname := strings.TrimPrefix(url, "https://") - hostname = strings.TrimPrefix(hostname, "http://") - hostname = strings.Split(hostname, "/")[0] - - startTime := time.Now() - ips, err := net.LookupIP(hostname) - if err != nil { - log.Printf("Failed to resolve hostname %s with DNS %s: %v\n", hostname, dns, err) - return - } - resolutionTime := time.Since(startTime) - - log.Printf("Resolved IPs for %s: %v (DNS: %s)\n", hostname, ips, dns) - log.Printf("DNS resolution took: %v\n", resolutionTime) - - resp, err := client.Get(url) - if err != nil { - log.Printf("Failed to fetch URL %s with DNS %s: %v\n", url, dns, err) - return - } - defer resp.Body.Close() - - log.Printf("Response status for %s (DNS: %s): %s\n", url, dns, resp.Status) - }(dns) - } - - wg.Wait() - return nil -} - -func ReadDNSFromFile(filename string) ([]string, error) { - data, err := os.ReadFile(filename) - if err != nil { - return nil, err - } - dnsServers := strings.Fields(string(data)) - return dnsServers, nil -} - -// ################### need to be completed ######################## -func URLValidator(URL string) bool { - return false -} - -func DockerImageValidator(URL string) bool { - return false -} - -func CheckWithURL(c *cli.Context) error { - return nil -} - -func CheckWithDockerImage(c *cli.Context) error { - return nil -} From 94b4399c4138e797d1acd2e96e12b0b280f64fbb Mon Sep 17 00:00:00 2001 From: borhanisaleh6 Date: Thu, 9 Jan 2025 12:04:19 +0330 Subject: [PATCH 05/19] feat(): download with custom dns func --- go.mod | 1 + go.sum | 2 ++ internal/dns/function.go | 76 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 28d493f..97518a4 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( ) require ( + github.com/cavaliergopher/grab/v3 v3.0.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 1b87b89..00328de 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= +github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/internal/dns/function.go b/internal/dns/function.go index e6650bf..528428b 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -1,9 +1,15 @@ package dns import ( - "net/url" - + "context" + "fmt" + "github.com/cavaliergopher/grab/v3" + "github.com/salehborhani/403Unlocker-cli/internal/check" "github.com/urfave/cli/v2" + "net/url" + "os" + "sync" + "time" ) func URLValidator(URL string) bool { @@ -24,5 +30,71 @@ func URLValidator(URL string) bool { } func CheckWithURL(c *cli.Context) error { + fileToDownload := c.Args().First() + dnsList, err := check.ReadDNSFromFile("config/dns.conf") + if err != nil { + fmt.Println("Error reading DNS list:", err) + return err + } + + // Map to store the total size downloaded by each DNS + dnsSizeMap := make(map[string]int64) + var mutex sync.Mutex + + // Create a context with a timeout of 15 seconds + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + var wg sync.WaitGroup + for _, dns := range dnsList { + wg.Add(1) + go func(dns string) { + defer wg.Done() + + // Create a custom HTTP client with the specified DNS + clientWithCustomDNS := check.ChangeDNS(dns) + client := grab.NewClient() + client.HTTPClient = clientWithCustomDNS + + // Create a new download request + req, err := grab.NewRequest(".", fileToDownload) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating request for DNS %s: %v\n", dns, err) + } + req = req.WithContext(ctx) + + // Start the download + resp := client.Do(req) + if err := resp.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Download failed for DNS %s: %v\n", dns, err) + } + + // Update the total size downloaded by this DNS + mutex.Lock() + dnsSizeMap[dns] += resp.BytesComplete() // Use BytesComplete() for partial downloads + mutex.Unlock() + + fmt.Printf("Downloaded %d bytes using DNS %s\n", resp.BytesComplete(), dns) + }(dns) + } + + wg.Wait() + + // Determine which DNS downloaded the most data + var maxDNS string + var maxSize int64 + for dns, size := range dnsSizeMap { + if size > maxSize { + maxDNS = dns + maxSize = size + } + } + + if maxDNS != "" { + fmt.Printf("DNS %s downloaded the most data: %d bytes\n", maxDNS, maxSize) + } else { + fmt.Println("No DNS server was able to download any data.") + } + return nil } From 27d1a99ea6a6fdcf37c3ce51940dfbdf7d98c8bb Mon Sep 17 00:00:00 2001 From: borhanisaleh6 Date: Thu, 9 Jan 2025 12:27:40 +0330 Subject: [PATCH 06/19] fix(): handle golang lint --- internal/dns/function.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/dns/function.go b/internal/dns/function.go index 528428b..5d572dc 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -3,13 +3,14 @@ package dns import ( "context" "fmt" - "github.com/cavaliergopher/grab/v3" - "github.com/salehborhani/403Unlocker-cli/internal/check" - "github.com/urfave/cli/v2" "net/url" "os" "sync" "time" + + "github.com/cavaliergopher/grab/v3" + "github.com/salehborhani/403Unlocker-cli/internal/check" + "github.com/urfave/cli/v2" ) func URLValidator(URL string) bool { From ec8922d61367973b74c76a7454bc8688577fac14 Mon Sep 17 00:00:00 2001 From: borhanisaleh6 Date: Fri, 10 Jan 2025 19:37:05 +0330 Subject: [PATCH 07/19] fix(): change the logic of url validator --- go.mod | 1 + go.sum | 2 ++ internal/check/fucntion_test.go | 16 +++++++++++++--- internal/check/function.go | 27 +++++++++++++++++++++++++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 97518a4..161a502 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/klauspost/compress v1.16.5 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/logrusorgru/aurora/v3 v3.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect diff --git a/go.sum b/go.sum index 00328de..731c927 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= +github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= diff --git a/internal/check/fucntion_test.go b/internal/check/fucntion_test.go index 59128dc..695db30 100644 --- a/internal/check/fucntion_test.go +++ b/internal/check/fucntion_test.go @@ -12,14 +12,19 @@ func TestDomainValidator(t *testing.T) { domain string expected bool }{ + { + name: "Valid URL", + domain: "https://pkg.go.dev", + expected: true, + }, { name: "Valid domain", - domain: "example.com", + domain: "http://example.com", expected: true, }, { name: "Valid subdomain", - domain: "sub.example.com", + domain: "https://sub.example.com", expected: true, }, { @@ -44,7 +49,7 @@ func TestDomainValidator(t *testing.T) { }, { name: "Valid domain with hyphens", - domain: "valid-domain.org", + domain: "https://valid-domain.org", expected: true, }, { @@ -62,6 +67,11 @@ func TestDomainValidator(t *testing.T) { domain: "invalid.", expected: false, }, + { + name: "Invalid domain without scheme", + domain: "pkg.go.dev", + expected: false, + }, } for _, tt := range tests { diff --git a/internal/check/function.go b/internal/check/function.go index 692106b..a6ba911 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -45,6 +45,9 @@ func CheckWithDNS(c *cli.Context) error { return err } + dnsResolution := make(map[string]time.Duration) + var mutex sync.Mutex + var wg sync.WaitGroup for _, dns := range dnsList { wg.Add(1) @@ -73,13 +76,33 @@ func CheckWithDNS(c *cli.Context) error { log.Printf("Failed to fetch URL %s with DNS %s: %v\n", url, dns, err) return } + // Store the time which has been taken to resolve + + mutex.Lock() + dnsResolution[dns] += resolutionTime // Use BytesComplete() for partial downloads + mutex.Unlock() + defer resp.Body.Close() log.Printf("Response status for %s (DNS: %s): %s\n", url, dns, resp.Status) }(dns) } - wg.Wait() + + var dns string + var minTime time.Duration = 1<<63 - 1 + for d, t := range dnsResolution { + if t < minTime { + dns = d + minTime = t + } + } + + if dns != "" { + fmt.Printf("DNS %s resolved faster: %s \n", dns, minTime) + } else { + fmt.Println("No DNS server was able to download any data.") + } return nil } @@ -98,7 +121,7 @@ func DomainValidator(domain string) bool { // - The domain contains only alphanumeric characters, hyphens, and dots. // - It does not start or end with a hyphen or dot. // - It has at least one dot. - domainRegex := `^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$` + domainRegex := `^(https?:\/\/)([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$` // Match the domain against the regex match, _ := regexp.MatchString(domainRegex, domain) From 0b53fc0384c11e86dbdb598cb3a50169c589a6de Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 11:45:20 -0600 Subject: [PATCH 08/19] chore(): fix std output of dns sub command --- internal/dns/function.go | 56 ++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/internal/dns/function.go b/internal/dns/function.go index 5d572dc..572c894 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -5,7 +5,6 @@ import ( "fmt" "net/url" "os" - "sync" "time" "github.com/cavaliergopher/grab/v3" @@ -40,47 +39,37 @@ func CheckWithURL(c *cli.Context) error { // Map to store the total size downloaded by each DNS dnsSizeMap := make(map[string]int64) - var mutex sync.Mutex - // Create a context with a timeout of 15 seconds - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - var wg sync.WaitGroup for _, dns := range dnsList { - wg.Add(1) - go func(dns string) { - defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // Create a custom HTTP client with the specified DNS + clientWithCustomDNS := check.ChangeDNS(dns) + client := grab.NewClient() + client.HTTPClient = clientWithCustomDNS + + // Create a new download request + req, err := grab.NewRequest(".", fileToDownload) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating request for DNS %s: %v\n", dns, err) + } + req = req.WithContext(ctx) - // Create a custom HTTP client with the specified DNS - clientWithCustomDNS := check.ChangeDNS(dns) - client := grab.NewClient() - client.HTTPClient = clientWithCustomDNS + // Start the download + resp := client.Do(req) + // if err := resp.Err(); err != nil { - // Create a new download request - req, err := grab.NewRequest(".", fileToDownload) - if err != nil { - fmt.Fprintf(os.Stderr, "Error creating request for DNS %s: %v\n", dns, err) - } - req = req.WithContext(ctx) + // // fmt.Fprintf(os.Stderr, "Download failed for DNS %s: %v\n", dns, err) + // } - // Start the download - resp := client.Do(req) - if err := resp.Err(); err != nil { - fmt.Fprintf(os.Stderr, "Download failed for DNS %s: %v\n", dns, err) - } + // Update the total size downloaded by this DNS + dnsSizeMap[dns] += resp.BytesComplete() // Use BytesComplete() for partial downloads - // Update the total size downloaded by this DNS - mutex.Lock() - dnsSizeMap[dns] += resp.BytesComplete() // Use BytesComplete() for partial downloads - mutex.Unlock() + fmt.Printf("Downloaded %d KB using DNS %s\n", resp.BytesComplete()/8_000, dns) - fmt.Printf("Downloaded %d bytes using DNS %s\n", resp.BytesComplete(), dns) - }(dns) } - wg.Wait() - // Determine which DNS downloaded the most data var maxDNS string var maxSize int64 @@ -90,9 +79,8 @@ func CheckWithURL(c *cli.Context) error { maxSize = size } } - if maxDNS != "" { - fmt.Printf("DNS %s downloaded the most data: %d bytes\n", maxDNS, maxSize) + fmt.Printf("DNS %s downloaded the most data: %d KB\n", maxDNS, maxSize/8_000) } else { fmt.Println("No DNS server was able to download any data.") } From a9c2439a41aafb372b0139984307e494857ad3bf Mon Sep 17 00:00:00 2001 From: borhanisaleh6 Date: Fri, 10 Jan 2025 21:29:25 +0330 Subject: [PATCH 09/19] chore(): remove unnecessary log --- internal/check/function.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/check/function.go b/internal/check/function.go index a6ba911..275dc6f 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -21,7 +21,6 @@ func ChangeDNS(dns string) *http.Client { PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) { dnsServer := fmt.Sprintf("%s:53", dns) - log.Printf("Using DNS server: %s\n", dnsServer) return dialer.DialContext(ctx, "udp", dnsServer) }, } From 080642d8f997eb809f137a92f679392269c0386b Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 12:23:53 -0600 Subject: [PATCH 10/19] fix(): change stdout for check subcommand --- internal/check/function.go | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/internal/check/function.go b/internal/check/function.go index 275dc6f..a9a2d9b 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -60,19 +60,14 @@ func CheckWithDNS(c *cli.Context) error { hostname = strings.Split(hostname, "/")[0] startTime := time.Now() - ips, err := net.LookupIP(hostname) + _, err := net.LookupIP(hostname) if err != nil { log.Printf("Failed to resolve hostname %s with DNS %s: %v\n", hostname, dns, err) return } resolutionTime := time.Since(startTime) - - log.Printf("Resolved IPs for %s: %v (DNS: %s)\n", hostname, ips, dns) - log.Printf("DNS resolution took: %v\n", resolutionTime) - resp, err := client.Get(url) if err != nil { - log.Printf("Failed to fetch URL %s with DNS %s: %v\n", url, dns, err) return } // Store the time which has been taken to resolve @@ -83,25 +78,10 @@ func CheckWithDNS(c *cli.Context) error { defer resp.Body.Close() - log.Printf("Response status for %s (DNS: %s): %s\n", url, dns, resp.Status) + fmt.Printf("Response status for %s (DNS: %s): %s\n", url, dns, resp.Status) }(dns) } wg.Wait() - - var dns string - var minTime time.Duration = 1<<63 - 1 - for d, t := range dnsResolution { - if t < minTime { - dns = d - minTime = t - } - } - - if dns != "" { - fmt.Printf("DNS %s resolved faster: %s \n", dns, minTime) - } else { - fmt.Println("No DNS server was able to download any data.") - } return nil } @@ -114,7 +94,6 @@ func ReadDNSFromFile(filename string) ([]string, error) { return dnsServers, nil } func DomainValidator(domain string) bool { - // Regular expression to validate domain names // This regex ensures: // - The domain contains only alphanumeric characters, hyphens, and dots. @@ -127,7 +106,6 @@ func DomainValidator(domain string) bool { if !match { return false } - // Additional checks: // 1. The total length of the domain should not exceed 253 characters. if len(domain) > 253 { From 98b06c1f02920d2b1009fa906b8c55fed2cf5ca5 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 12:48:39 -0600 Subject: [PATCH 11/19] chore(): minimal check output subcommand --- internal/check/function.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/check/function.go b/internal/check/function.go index a9a2d9b..691ab76 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -77,8 +77,8 @@ func CheckWithDNS(c *cli.Context) error { mutex.Unlock() defer resp.Body.Close() - - fmt.Printf("Response status for %s (DNS: %s): %s\n", url, dns, resp.Status) + code := strings.Split(resp.Status, " ") + fmt.Printf("DNS: %s %s\n", dns, code[1]) }(dns) } wg.Wait() @@ -99,8 +99,7 @@ func DomainValidator(domain string) bool { // - The domain contains only alphanumeric characters, hyphens, and dots. // - It does not start or end with a hyphen or dot. // - It has at least one dot. - domainRegex := `^(https?:\/\/)([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$` - + domainRegex := `^(https?:\/\/)([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}\/.*$` // Match the domain against the regex match, _ := regexp.MatchString(domainRegex, domain) if !match { From ea4330dc2fc92b6ddbb9f95c208052ec8bbcd575 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 14:35:11 -0600 Subject: [PATCH 12/19] chore(): change dns output --- internal/dns/function.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/internal/dns/function.go b/internal/dns/function.go index 572c894..58014e9 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -55,18 +55,11 @@ func CheckWithURL(c *cli.Context) error { fmt.Fprintf(os.Stderr, "Error creating request for DNS %s: %v\n", dns, err) } req = req.WithContext(ctx) - // Start the download resp := client.Do(req) - // if err := resp.Err(); err != nil { - - // // fmt.Fprintf(os.Stderr, "Download failed for DNS %s: %v\n", dns, err) - // } - // Update the total size downloaded by this DNS dnsSizeMap[dns] += resp.BytesComplete() // Use BytesComplete() for partial downloads - - fmt.Printf("Downloaded %d KB using DNS %s\n", resp.BytesComplete()/8_000, dns) + fmt.Printf("Downloaded %d KB using DNS %s\n", resp.BytesComplete()/1_000, dns) } @@ -80,7 +73,7 @@ func CheckWithURL(c *cli.Context) error { } } if maxDNS != "" { - fmt.Printf("DNS %s downloaded the most data: %d KB\n", maxDNS, maxSize/8_000) + fmt.Printf("%s downloaded the most data: %d KB\n", maxDNS, maxSize/1_000) } else { fmt.Println("No DNS server was able to download any data.") } From af49131e0c01e6514bbb0a5cff7dacfcf1a95c12 Mon Sep 17 00:00:00 2001 From: borhanisaleh6 Date: Sat, 11 Jan 2025 01:32:26 +0330 Subject: [PATCH 13/19] fix(): fix schema --- internal/check/fucntion_test.go | 2 +- internal/check/function.go | 51 +++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/internal/check/fucntion_test.go b/internal/check/fucntion_test.go index 695db30..6305164 100644 --- a/internal/check/fucntion_test.go +++ b/internal/check/fucntion_test.go @@ -70,7 +70,7 @@ func TestDomainValidator(t *testing.T) { { name: "Invalid domain without scheme", domain: "pkg.go.dev", - expected: false, + expected: true, }, } diff --git a/internal/check/function.go b/internal/check/function.go index 691ab76..b15018f 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -3,14 +3,12 @@ package check import ( "context" "fmt" - "log" "net" "net/http" "os" "regexp" "strings" "sync" - "time" "github.com/urfave/cli/v2" ) @@ -44,8 +42,7 @@ func CheckWithDNS(c *cli.Context) error { return err } - dnsResolution := make(map[string]time.Duration) - var mutex sync.Mutex + url = ensureHTTPS(url) var wg sync.WaitGroup for _, dns := range dnsList { @@ -55,26 +52,10 @@ func CheckWithDNS(c *cli.Context) error { client := ChangeDNS(dns) - hostname := strings.TrimPrefix(url, "https://") - hostname = strings.TrimPrefix(hostname, "http://") - hostname = strings.Split(hostname, "/")[0] - - startTime := time.Now() - _, err := net.LookupIP(hostname) - if err != nil { - log.Printf("Failed to resolve hostname %s with DNS %s: %v\n", hostname, dns, err) - return - } - resolutionTime := time.Since(startTime) resp, err := client.Get(url) if err != nil { return } - // Store the time which has been taken to resolve - - mutex.Lock() - dnsResolution[dns] += resolutionTime // Use BytesComplete() for partial downloads - mutex.Unlock() defer resp.Body.Close() code := strings.Split(resp.Status, " ") @@ -99,7 +80,7 @@ func DomainValidator(domain string) bool { // - The domain contains only alphanumeric characters, hyphens, and dots. // - It does not start or end with a hyphen or dot. // - It has at least one dot. - domainRegex := `^(https?:\/\/)([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}\/.*$` + domainRegex := `^(http[s]?:\/\/)?([a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}).*?$` // Match the domain against the regex match, _ := regexp.MatchString(domainRegex, domain) if !match { @@ -121,3 +102,31 @@ func DomainValidator(domain string) bool { return true } + +func ensureHTTPS(url string) string { + // Regex to check if the URL starts with https:// + regex := `^(https)://` + re, err := regexp.Compile(regex) + if err != nil { + fmt.Println("Error compiling regex:", err) + return url + } + regexHTTP := `^(http)://` + + reHTTP, err := regexp.Compile(regexHTTP) + if err != nil { + fmt.Println("Error compiling regex:", err) + return url + } + + if reHTTP.MatchString(url) { + url = strings.TrimPrefix(url, "http://") + } + + // If the URL doesn't start with http:// or https://, prepend https:// + if !re.MatchString(url) { + url = "https://" + url + } + + return url +} From f70cb838ed33ab24e7d855fb69b1ffe3e2b33695 Mon Sep 17 00:00:00 2001 From: borhanisaleh6 Date: Sat, 11 Jan 2025 01:33:35 +0330 Subject: [PATCH 14/19] feat(): add timeout flag --- internal/dns/function.go | 12 +++++++++--- internal/unlockercli/cli.go | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/internal/dns/function.go b/internal/dns/function.go index 58014e9..f65502a 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -31,6 +31,8 @@ func URLValidator(URL string) bool { func CheckWithURL(c *cli.Context) error { fileToDownload := c.Args().First() + timeout := c.Int("timeout") + dnsList, err := check.ReadDNSFromFile("config/dns.conf") if err != nil { fmt.Println("Error reading DNS list:", err) @@ -39,10 +41,14 @@ func CheckWithURL(c *cli.Context) error { // Map to store the total size downloaded by each DNS dnsSizeMap := make(map[string]int64) - // Create a context with a timeout of 15 seconds + + fmt.Println("Timeout:", timeout) + fmt.Println("URL: ", fileToDownload) + + tempDir := time.Now().UnixMilli() for _, dns := range dnsList { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() // Create a custom HTTP client with the specified DNS clientWithCustomDNS := check.ChangeDNS(dns) @@ -50,7 +56,7 @@ func CheckWithURL(c *cli.Context) error { client.HTTPClient = clientWithCustomDNS // Create a new download request - req, err := grab.NewRequest(".", fileToDownload) + req, err := grab.NewRequest(fmt.Sprintf("/tmp/%v", tempDir), fileToDownload) if err != nil { fmt.Fprintf(os.Stderr, "Error creating request for DNS %s: %v\n", dns, err) } diff --git a/internal/unlockercli/cli.go b/internal/unlockercli/cli.go index 42635f3..673e2a5 100644 --- a/internal/unlockercli/cli.go +++ b/internal/unlockercli/cli.go @@ -34,6 +34,14 @@ func Run() { Name: "docker", Aliases: []string{"d"}, Usage: "Finds the fastest docker registries for an specific docker image", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "timeout", + Usage: "Sets timeout", + Value: 10, + Aliases: []string{"t"}, + }, + }, Action: func(cCtx *cli.Context) error { if docker.DockerImageValidator(cCtx.Args().First()) { return docker.CheckWithDockerImage(cCtx) @@ -46,6 +54,14 @@ func Run() { { Name: "dns", Usage: "Finds the fastest DNS SNI-Proxy for downloading an specific URL", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "timeout", + Usage: "Sets timeout", + Value: 10, + Aliases: []string{"t"}, + }, + }, Action: func(cCtx *cli.Context) error { if dns.URLValidator(cCtx.Args().First()) { return dns.CheckWithURL(cCtx) From b9bab417f9fd8adcc18ef7828628fa0b4eb08d68 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 17:27:43 -0600 Subject: [PATCH 15/19] feat(): change sub command name --- internal/unlockercli/cli.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/unlockercli/cli.go b/internal/unlockercli/cli.go index 673e2a5..60ee447 100644 --- a/internal/unlockercli/cli.go +++ b/internal/unlockercli/cli.go @@ -31,8 +31,8 @@ func Run() { }, }, { - Name: "docker", - Aliases: []string{"d"}, + Name: "fastdocker", + Aliases: []string{"docker"}, Usage: "Finds the fastest docker registries for an specific docker image", Flags: []cli.Flag{ &cli.IntFlag{ @@ -52,8 +52,9 @@ func Run() { }, }, { - Name: "dns", - Usage: "Finds the fastest DNS SNI-Proxy for downloading an specific URL", + Name: "bestdns", + Aliases: []string{"dns"}, + Usage: "Finds the fastest DNS SNI-Proxy for downloading an specific URL", Flags: []cli.Flag{ &cli.IntFlag{ Name: "timeout", From 12df65a3b78a6c4912eb1ed1b42108834de2b402 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 17:28:33 -0600 Subject: [PATCH 16/19] chore(): detele temp dir --- internal/dns/function.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/dns/function.go b/internal/dns/function.go index f65502a..8c20130 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -83,6 +83,6 @@ func CheckWithURL(c *cli.Context) error { } else { fmt.Println("No DNS server was able to download any data.") } - + os.RemoveAll(fmt.Sprintf("/tmp/%v", tempDir)) return nil } From dbc8b5316329d4a8a350d79897972dc9e1013d77 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 22:29:07 -0600 Subject: [PATCH 17/19] feat(): Add Docker subcommand --- config/dockerRegistry.conf | 1 + go.mod | 3 +- go.sum | 2 - internal/check/function.go | 17 ----- internal/common/function.go | 23 ++++++ internal/dns/function.go | 14 +--- internal/docker/function.go | 148 ++++++++++++++++++++++++++++-------- 7 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 config/dockerRegistry.conf create mode 100644 internal/common/function.go diff --git a/config/dockerRegistry.conf b/config/dockerRegistry.conf new file mode 100644 index 0000000..9c05284 --- /dev/null +++ b/config/dockerRegistry.conf @@ -0,0 +1 @@ +docker.arvancloud.ir focker.ir registry.docker.ir docker.host:5000 docker.iranserver.com docker.haiocloud.com registry.registryhub.ir \ No newline at end of file diff --git a/go.mod b/go.mod index 161a502..b9450b6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/salehborhani/403Unlocker-cli go 1.23.1 require ( + github.com/cavaliergopher/grab/v3 v3.0.1 github.com/google/go-containerregistry v0.20.2 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.27.5 @@ -10,7 +11,6 @@ require ( ) require ( - github.com/cavaliergopher/grab/v3 v3.0.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -21,7 +21,6 @@ require ( github.com/klauspost/compress v1.16.5 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/logrusorgru/aurora/v3 v3.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect diff --git a/go.sum b/go.sum index 731c927..00328de 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= -github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= diff --git a/internal/check/function.go b/internal/check/function.go index b15018f..e901f84 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -41,22 +41,17 @@ func CheckWithDNS(c *cli.Context) error { fmt.Println(err) return err } - url = ensureHTTPS(url) - var wg sync.WaitGroup for _, dns := range dnsList { wg.Add(1) go func(dns string) { defer wg.Done() - client := ChangeDNS(dns) - resp, err := client.Get(url) if err != nil { return } - defer resp.Body.Close() code := strings.Split(resp.Status, " ") fmt.Printf("DNS: %s %s\n", dns, code[1]) @@ -75,13 +70,7 @@ func ReadDNSFromFile(filename string) ([]string, error) { return dnsServers, nil } func DomainValidator(domain string) bool { - // Regular expression to validate domain names - // This regex ensures: - // - The domain contains only alphanumeric characters, hyphens, and dots. - // - It does not start or end with a hyphen or dot. - // - It has at least one dot. domainRegex := `^(http[s]?:\/\/)?([a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}).*?$` - // Match the domain against the regex match, _ := regexp.MatchString(domainRegex, domain) if !match { return false @@ -91,7 +80,6 @@ func DomainValidator(domain string) bool { if len(domain) > 253 { return false } - // 2. Each segment between dots should be between 1 and 63 characters long. segments := strings.Split(domain, ".") for _, segment := range segments { @@ -99,7 +87,6 @@ func DomainValidator(domain string) bool { return false } } - return true } @@ -112,21 +99,17 @@ func ensureHTTPS(url string) string { return url } regexHTTP := `^(http)://` - reHTTP, err := regexp.Compile(regexHTTP) if err != nil { fmt.Println("Error compiling regex:", err) return url } - if reHTTP.MatchString(url) { url = strings.TrimPrefix(url, "http://") } - // If the URL doesn't start with http:// or https://, prepend https:// if !re.MatchString(url) { url = "https://" + url } - return url } diff --git a/internal/common/function.go b/internal/common/function.go new file mode 100644 index 0000000..539e7a0 --- /dev/null +++ b/internal/common/function.go @@ -0,0 +1,23 @@ +package common + +import "fmt" + +// FormatDataSize converts the size in bytes to a human-readable string in KB, MB, or GB. +func FormatDataSize(bytes int64) string { + const ( + kb = 1024 + mb = kb * 1024 + gb = mb * 1024 + ) + + switch { + case bytes >= gb: + return fmt.Sprintf("%.2f GB", float64(bytes)/float64(gb)) + case bytes >= mb: + return fmt.Sprintf("%.2f MB", float64(bytes)/float64(mb)) + case bytes >= kb: + return fmt.Sprintf("%.2f KB", float64(bytes)/float64(kb)) + default: + return fmt.Sprintf("%d Bytes", bytes) + } +} diff --git a/internal/dns/function.go b/internal/dns/function.go index 8c20130..ac50df6 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -9,6 +9,7 @@ import ( "github.com/cavaliergopher/grab/v3" "github.com/salehborhani/403Unlocker-cli/internal/check" + "github.com/salehborhani/403Unlocker-cli/internal/common" "github.com/urfave/cli/v2" ) @@ -28,25 +29,19 @@ func URLValidator(URL string) bool { } return true } - func CheckWithURL(c *cli.Context) error { fileToDownload := c.Args().First() timeout := c.Int("timeout") - dnsList, err := check.ReadDNSFromFile("config/dns.conf") if err != nil { fmt.Println("Error reading DNS list:", err) return err } - // Map to store the total size downloaded by each DNS dnsSizeMap := make(map[string]int64) - fmt.Println("Timeout:", timeout) fmt.Println("URL: ", fileToDownload) - tempDir := time.Now().UnixMilli() - for _, dns := range dnsList { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() @@ -54,7 +49,6 @@ func CheckWithURL(c *cli.Context) error { clientWithCustomDNS := check.ChangeDNS(dns) client := grab.NewClient() client.HTTPClient = clientWithCustomDNS - // Create a new download request req, err := grab.NewRequest(fmt.Sprintf("/tmp/%v", tempDir), fileToDownload) if err != nil { @@ -65,10 +59,8 @@ func CheckWithURL(c *cli.Context) error { resp := client.Do(req) // Update the total size downloaded by this DNS dnsSizeMap[dns] += resp.BytesComplete() // Use BytesComplete() for partial downloads - fmt.Printf("Downloaded %d KB using DNS %s\n", resp.BytesComplete()/1_000, dns) - + fmt.Printf("%v\tDNS: %s\n", common.FormatDataSize(resp.BytesComplete()), dns) } - // Determine which DNS downloaded the most data var maxDNS string var maxSize int64 @@ -79,7 +71,7 @@ func CheckWithURL(c *cli.Context) error { } } if maxDNS != "" { - fmt.Printf("%s downloaded the most data: %d KB\n", maxDNS, maxSize/1_000) + fmt.Printf("best DNS is %s and downloaded the most data: %v\n", maxDNS, common.FormatDataSize(maxSize)) } else { fmt.Println("No DNS server was able to download any data.") } diff --git a/internal/docker/function.go b/internal/docker/function.go index 888db7b..6dfe633 100644 --- a/internal/docker/function.go +++ b/internal/docker/function.go @@ -3,63 +3,145 @@ package docker import ( "context" "fmt" + "io" "log" + "net/http" + "os" + "path/filepath" "regexp" "strings" + "sync/atomic" "time" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/salehborhani/403Unlocker-cli/internal/check" + "github.com/salehborhani/403Unlocker-cli/internal/common" "github.com/urfave/cli/v2" ) +// DockerImageValidator validates a Docker image name using a regular expression. func DockerImageValidator(imageName string) bool { - // Regular expression to match a valid Docker image name - // This pattern allows for optional registry, namespace, and tag/digest - pattern := `^(?:[a-zA-Z0-9\-._]+(?::[0-9]+)?/)?` + // Optional registry (e.g., docker.io, localhost:5000) - `(?:[a-z0-9\-._]+/)?` + // Optional namespace (e.g., library, user) - `[a-z0-9\-._]+` + // Repository name (required) - `(?::[a-zA-Z0-9\-._]+)?` + // Optional tag (e.g., latest, v1.0) - `(?:@[a-zA-Z0-9\-._:]+)?$` // Optional digest (e.g., sha256:...) - - // Compile the regular expression + pattern := `^(?:[a-zA-Z0-9\-._]+(?::[0-9]+)?/)?` + + `(?:[a-z0-9\-._]+/)?` + + `[a-z0-9\-._]+` + + `(?::[a-zA-Z0-9\-._]+)?` + + `(?:@[a-zA-Z0-9\-._:]+)?$` regex := regexp.MustCompile(pattern) - - // Check if the image name matches the pattern return regex.MatchString(imageName) && !strings.Contains(imageName, "@@") } -// downloadDockerImage downloads a Docker image from a registry and saves it as a tarball. -func DownloadDockerImage(imageName, registry, outputPath string, timeoutSeconds int) error { - imageName = registry + "/" + imageName - outputPath = outputPath + "/" + imageName - // Parse the image reference (e.g., "ubuntu:latest") - ref, err := name.ParseReference(imageName) +// customTransport tracks the number of bytes transferred during HTTP requests. +type customTransport struct { + Transport http.RoundTripper + Bytes int64 +} + +// RoundTrip implements the http.RoundTripper interface and wraps the response body to count bytes read. +func (c *customTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := c.Transport.RoundTrip(req) if err != nil { - return fmt.Errorf("failed to parse image reference: %v", err) + return nil, err } - // Authenticate with the registry (defaults to anonymous auth) - auth := authn.DefaultKeychain - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second) - defer cancel() - img, err := remote.Image(ref, remote.WithAuthFromKeychain(auth), remote.WithContext(ctx)) + resp.Body = &countingReader{inner: resp.Body, Bytes: &c.Bytes} + return resp, nil +} + +// countingReader wraps an io.ReadCloser and counts the bytes read. +type countingReader struct { + inner io.ReadCloser + Bytes *int64 +} + +func (cr *countingReader) Read(p []byte) (int, error) { + n, err := cr.inner.Read(p) + atomic.AddInt64(cr.Bytes, int64(n)) + return n, err +} + +func (cr *countingReader) Close() error { + return cr.inner.Close() +} + +// DownloadDockerImage downloads a Docker image from a registry and tracks the bytes downloaded. +func DownloadDockerImage(ctx context.Context, imageName, registry, outputPath string) (int64, error) { + + fullImageName := registry + "/" + imageName + + // Parse the image reference. + ref, err := name.ParseReference(fullImageName) if err != nil { - if ctx.Err() == context.DeadlineExceeded { - log.Printf("Download timed out after %d seconds, saving partially downloaded image...\n", timeoutSeconds) - } else { - return fmt.Errorf("failed to fetch image: %v", err) - } + return 0, fmt.Errorf("failed to parse image reference: %v", err) } - // Save the image as a tarball - err = tarball.WriteToFile(outputPath, ref, img) + + auth := authn.DefaultKeychain + transport := &customTransport{Transport: http.DefaultTransport} + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(auth), remote.WithContext(ctx), remote.WithTransport(transport)) if err != nil { - return fmt.Errorf("failed to save image as tarball: %v", err) + return transport.Bytes, fmt.Errorf("failed to download image: %v", err) } - fmt.Printf("Image successfully downloaded and saved to %s\n", outputPath) - return nil + + // Ensure output directory exists. + if err := os.MkdirAll(outputPath, 0755); err != nil { + return transport.Bytes, fmt.Errorf("failed to create output directory: %v", err) + } + + // Save the image as a tarball. + tarballPath := filepath.Join(outputPath, filepath.Base(imageName)+".tar") + if err := tarball.WriteToFile(tarballPath, ref, img); err != nil { + return transport.Bytes, nil + } + + return transport.Bytes, nil } + +// CheckWithDockerImage downloads the image from multiple registries and reports the downloaded data size. func CheckWithDockerImage(c *cli.Context) error { + registrySizeMap := make(map[string]int64) + timeout := c.Int("timeout") + imageName := c.Args().First() + tempDir := time.Now().UnixMilli() + fmt.Println("Timeout:", timeout) + fmt.Println("Docker Image: ", imageName) + + if imageName == "" { + return fmt.Errorf("image name cannot be empty") + } + + registryList, err := check.ReadDNSFromFile("config/dockerRegistry.conf") + if err != nil { + log.Printf("Error reading registry list: %v", err) + return err + } + + for _, registry := range registryList { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + size, err := DownloadDockerImage(ctx, imageName, registry, fmt.Sprintf("/tmp/%v", tempDir)) + if err != nil { + fmt.Printf("%s: %v\n", registry, "failed") + continue + } + registrySizeMap[registry] += size + fmt.Printf("%s downloaded : %v\n", registry, common.FormatDataSize(size)) + } + // Determine which DNS downloaded the most data + var maxRegistry string + var maxSize int64 + for dns, size := range registrySizeMap { + if size > maxSize { + maxRegistry = dns + maxSize = size + } + } + if maxRegistry != "" { + fmt.Printf("best Registry is %s and downloaded the most data: %v\n", maxRegistry, common.FormatDataSize(maxSize)) + } else { + fmt.Println("No DNS server was able to download any data.") + } + os.RemoveAll(fmt.Sprintf("/tmp/%v", tempDir)) return nil } From 82f05a8aae5dd6704ff6c82a088db4ace171a013 Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Fri, 10 Jan 2025 23:41:32 -0600 Subject: [PATCH 18/19] chore(): Add help to subcommands --- internal/unlockercli/cli.go | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/internal/unlockercli/cli.go b/internal/unlockercli/cli.go index 60ee447..d5c2b91 100644 --- a/internal/unlockercli/cli.go +++ b/internal/unlockercli/cli.go @@ -20,12 +20,17 @@ func Run() { { Name: "check", Aliases: []string{"c"}, - Usage: "Checks if the DNS SNI-Proxy can bypass 403 error for an specific domain", + Usage: "Checks if the DNS SNI-Proxy can bypass 403 error for a specific domain", + Description: `Examples: + 403unlocker check https://pkg.go.dev`, Action: func(cCtx *cli.Context) error { if check.DomainValidator(cCtx.Args().First()) { return check.CheckWithDNS(cCtx) } else { - fmt.Println("need a valid domain example: https://pkg.go.dev") + err := cli.ShowSubcommandHelp(cCtx) + if err != nil { + fmt.Println(err) + } } return nil }, @@ -33,7 +38,9 @@ func Run() { { Name: "fastdocker", Aliases: []string{"docker"}, - Usage: "Finds the fastest docker registries for an specific docker image", + Usage: "Finds the fastest docker registries for a specific docker image", + Description: `Examples: + 403unlocker fastdocker gitlab/gitlab-ce:17.0.0-ce.0 --timeout 15`, Flags: []cli.Flag{ &cli.IntFlag{ Name: "timeout", @@ -46,7 +53,10 @@ func Run() { if docker.DockerImageValidator(cCtx.Args().First()) { return docker.CheckWithDockerImage(cCtx) } else { - fmt.Println("need a valid docker image example: gitlab/gitlab-ce:17.0.0-ce.0") + err := cli.ShowSubcommandHelp(cCtx) + if err != nil { + fmt.Println(err) + } } return nil }, @@ -54,7 +64,9 @@ func Run() { { Name: "bestdns", Aliases: []string{"dns"}, - Usage: "Finds the fastest DNS SNI-Proxy for downloading an specific URL", + Usage: "Finds the fastest DNS SNI-Proxy for downloading a specific URL", + Description: `Examples: + 403unlocker bestdns https://packages.gitlab.com/gitlab/gitlab-ce/packages/el/7/gitlab-ce-16.8.0-ce.0.el7.x86_64.rpm/download.rpm`, Flags: []cli.Flag{ &cli.IntFlag{ Name: "timeout", @@ -67,7 +79,10 @@ func Run() { if dns.URLValidator(cCtx.Args().First()) { return dns.CheckWithURL(cCtx) } else { - fmt.Println("need a valid URL example: \"https://packages.gitlab.com/gitlab/gitlab-ce/packages/el/7/gitlab-ce-16.8.0-ce.0.el7.x86_64.rpm/download.rpm\"") + err := cli.ShowSubcommandHelp(cCtx) + if err != nil { + fmt.Println(err) + } } return nil }, From dcf2a303ce8b293138319f305bd66f0adcd9395e Mon Sep 17 00:00:00 2001 From: ArmanTaheriGhaleTaki Date: Sat, 11 Jan 2025 01:57:22 -0600 Subject: [PATCH 19/19] feat(): Add color to result output --- internal/check/fucntion_test.go | 46 ++++++++++++++++++++++++++++++++ internal/check/function.go | 47 ++++++++++++++++++++++++--------- internal/common/function.go | 10 +++++++ internal/dns/function.go | 8 ++++-- internal/docker/function.go | 6 ++--- internal/unlockercli/cli.go | 4 +-- 6 files changed, 101 insertions(+), 20 deletions(-) diff --git a/internal/check/fucntion_test.go b/internal/check/fucntion_test.go index 6305164..bcbfaa9 100644 --- a/internal/check/fucntion_test.go +++ b/internal/check/fucntion_test.go @@ -81,3 +81,49 @@ func TestDomainValidator(t *testing.T) { }) } } + +func TestEnsureHTTPS(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "URL with http scheme", + url: "http://example.com", + expected: "https://example.com/", + }, + { + name: "URL with https scheme", + url: "https://example.com", + expected: "https://example.com/", + }, + { + name: "URL without scheme", + url: "example.com", + expected: "https://example.com/", + }, + { + name: "URL with path and query", + url: "http://example.com/path?query=123", + expected: "https://example.com/", + }, + { + name: "URL with subdomain", + url: "http://sub.example.com", + expected: "https://sub.example.com/", + }, + { + name: "URL with double slashes", + url: "http://example.com//path", + expected: "https://example.com/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ensureHTTPS(tt.url) + assert.Equal(t, tt.expected, result, "Test case: %s", tt.name) + }) + } +} diff --git a/internal/check/function.go b/internal/check/function.go index e901f84..f8a9a1b 100644 --- a/internal/check/function.go +++ b/internal/check/function.go @@ -5,11 +5,14 @@ import ( "fmt" "net" "net/http" + "net/url" "os" "regexp" + "strconv" "strings" "sync" + "github.com/salehborhani/403Unlocker-cli/internal/common" "github.com/urfave/cli/v2" ) @@ -36,12 +39,14 @@ func ChangeDNS(dns string) *http.Client { func CheckWithDNS(c *cli.Context) error { url := c.Args().First() + url = ensureHTTPS(url) + + fmt.Printf("check: %s\n", url) dnsList, err := ReadDNSFromFile("config/dns.conf") if err != nil { fmt.Println(err) return err } - url = ensureHTTPS(url) var wg sync.WaitGroup for _, dns := range dnsList { wg.Add(1) @@ -54,7 +59,17 @@ func CheckWithDNS(c *cli.Context) error { } defer resp.Body.Close() code := strings.Split(resp.Status, " ") - fmt.Printf("DNS: %s %s\n", dns, code[1]) + statusCodeInt, err := strconv.Atoi(code[0]) + if err != nil { + fmt.Println("Error converting status code:", err) + return + } + if statusCodeInt == http.StatusForbidden { + fmt.Printf("DNS: %s %s%s%s\n", dns, common.Red, code[1], common.Reset) + } else { + fmt.Printf("DNS: %s %s%s%s\n", dns, common.Green, code[1], common.Reset) + } + }(dns) } wg.Wait() @@ -90,26 +105,32 @@ func DomainValidator(domain string) bool { return true } -func ensureHTTPS(url string) string { +func ensureHTTPS(URL string) string { // Regex to check if the URL starts with https:// - regex := `^(https)://` - re, err := regexp.Compile(regex) + regexHTTPS := `^(https)://` + reHTTPS, err := regexp.Compile(regexHTTPS) if err != nil { fmt.Println("Error compiling regex:", err) - return url + return URL } regexHTTP := `^(http)://` reHTTP, err := regexp.Compile(regexHTTP) if err != nil { fmt.Println("Error compiling regex:", err) - return url + return URL } - if reHTTP.MatchString(url) { - url = strings.TrimPrefix(url, "http://") + if reHTTP.MatchString(URL) { + URL = strings.TrimPrefix(URL, "http://") } - // If the URL doesn't start with http:// or https://, prepend https:// - if !re.MatchString(url) { - url = "https://" + url + if reHTTPS.MatchString(URL) { + URL = strings.TrimPrefix(URL, "https://") + } + URL = "https://" + URL + // Parse the URL to extract the host + parsedURL, err := url.Parse(URL) + if err != nil { + fmt.Println("Error parsing URL:", err) } - return url + // Return only the scheme and host (e.g., https://example.com) + return "https://" + parsedURL.Host + "/" } diff --git a/internal/common/function.go b/internal/common/function.go index 539e7a0..2294e4c 100644 --- a/internal/common/function.go +++ b/internal/common/function.go @@ -2,6 +2,16 @@ package common import "fmt" +var Reset = "\033[0m" +var Red = "\033[31m" +var Green = "\033[32m" +var Yellow = "\033[33m" +var Blue = "\033[34m" +var Magenta = "\033[35m" +var Cyan = "\033[36m" +var Gray = "\033[37m" +var White = "\033[97m" + // FormatDataSize converts the size in bytes to a human-readable string in KB, MB, or GB. func FormatDataSize(bytes int64) string { const ( diff --git a/internal/dns/function.go b/internal/dns/function.go index ac50df6..bb0a8bc 100644 --- a/internal/dns/function.go +++ b/internal/dns/function.go @@ -59,7 +59,11 @@ func CheckWithURL(c *cli.Context) error { resp := client.Do(req) // Update the total size downloaded by this DNS dnsSizeMap[dns] += resp.BytesComplete() // Use BytesComplete() for partial downloads - fmt.Printf("%v\tDNS: %s\n", common.FormatDataSize(resp.BytesComplete()), dns) + if resp.BytesComplete() == 0 { + fmt.Printf("DNS: %s\t%s%v/s%s\n", dns, common.Red, common.FormatDataSize(resp.BytesComplete()/int64(timeout)), common.Reset) + } else { + fmt.Printf("DNS: %s\t%s/s\n", dns, common.FormatDataSize(resp.BytesComplete()/int64(timeout))) + } } // Determine which DNS downloaded the most data var maxDNS string @@ -71,7 +75,7 @@ func CheckWithURL(c *cli.Context) error { } } if maxDNS != "" { - fmt.Printf("best DNS is %s and downloaded the most data: %v\n", maxDNS, common.FormatDataSize(maxSize)) + fmt.Printf("best DNS is %s%s%s and downloaded the most data: %s%v/s%s\n", common.Green, maxDNS, common.Reset, common.Green, common.FormatDataSize(maxSize/int64(timeout)), common.Reset) } else { fmt.Println("No DNS server was able to download any data.") } diff --git a/internal/docker/function.go b/internal/docker/function.go index 6dfe633..fd0c1ee 100644 --- a/internal/docker/function.go +++ b/internal/docker/function.go @@ -122,11 +122,11 @@ func CheckWithDockerImage(c *cli.Context) error { defer cancel() size, err := DownloadDockerImage(ctx, imageName, registry, fmt.Sprintf("/tmp/%v", tempDir)) if err != nil { - fmt.Printf("%s: %v\n", registry, "failed") + fmt.Printf("%s: %s%v%s\n", registry, common.Red, "failed", common.Reset) continue } registrySizeMap[registry] += size - fmt.Printf("%s downloaded : %v\n", registry, common.FormatDataSize(size)) + fmt.Printf("%s downloaded : %v/s\n", registry, common.FormatDataSize(size/int64(timeout))) } // Determine which DNS downloaded the most data var maxRegistry string @@ -138,7 +138,7 @@ func CheckWithDockerImage(c *cli.Context) error { } } if maxRegistry != "" { - fmt.Printf("best Registry is %s and downloaded the most data: %v\n", maxRegistry, common.FormatDataSize(maxSize)) + fmt.Printf("best Registry is %s%s%s and downloaded the most data: %s%v/s%s\n", common.Green, maxRegistry, common.Reset, common.Green, common.FormatDataSize(maxSize/int64(timeout)), common.Reset) } else { fmt.Println("No DNS server was able to download any data.") } diff --git a/internal/unlockercli/cli.go b/internal/unlockercli/cli.go index d5c2b91..4c59a7f 100644 --- a/internal/unlockercli/cli.go +++ b/internal/unlockercli/cli.go @@ -40,7 +40,7 @@ func Run() { Aliases: []string{"docker"}, Usage: "Finds the fastest docker registries for a specific docker image", Description: `Examples: - 403unlocker fastdocker gitlab/gitlab-ce:17.0.0-ce.0 --timeout 15`, + 403unlocker --timeout 15 fastdocker gitlab/gitlab-ce:17.0.0-ce.0`, Flags: []cli.Flag{ &cli.IntFlag{ Name: "timeout", @@ -66,7 +66,7 @@ func Run() { Aliases: []string{"dns"}, Usage: "Finds the fastest DNS SNI-Proxy for downloading a specific URL", Description: `Examples: - 403unlocker bestdns https://packages.gitlab.com/gitlab/gitlab-ce/packages/el/7/gitlab-ce-16.8.0-ce.0.el7.x86_64.rpm/download.rpm`, + 403unlocker bestdns --timeout 15 https://packages.gitlab.com/gitlab/gitlab-ce/packages/el/7/gitlab-ce-16.8.0-ce.0.el7.x86_64.rpm/download.rpm`, Flags: []cli.Flag{ &cli.IntFlag{ Name: "timeout",