🐞 fix: first path segment in URL

.vscode/launch.json

@@ -7,7 +7,7 @@
       "request": "launch",
       "mode": "auto",
       "program": "${workspaceFolder}/cmd/",
-      "args": ["-s","-r", "https://www.hellokuya.co"]
-    },
+      "args": ["-s", "-r", "https://www.hellokuya.co"]
+    }
   ]
-}
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/shurco/goclone
 
-go 1.20
+go 1.22
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.1
@@ -10,30 +10,28 @@ require (
 
 require (
 	github.com/VividCortex/gohistogram v1.0.0 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/chromedp/cdproto v0.0.0-20220428002153-285dfb42699c // indirect
-	github.com/chromedp/chromedp v0.8.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chromedp/cdproto v0.0.0-20240214232516-ad4608604e9e // indirect
+	github.com/chromedp/chromedp v0.9.5 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
-	github.com/go-kit/kit v0.12.0 // indirect
+	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.1.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/gobwas/ws v1.3.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/prometheus/client_golang v1.12.1 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.34.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.46.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 )

pkg/crawler/base.go

@@ -0,0 +1,39 @@
+package crawler
+
+import "net/url"
+
+type Flags struct {
+	Open            bool
+	Serve           bool
+	ServePort       int
+	UserAgent       string
+	ProxyString     string
+	Cookies         bool
+	Robots          bool
+	BrowserEndpoint string
+}
+
+var (
+	files filesBase
+
+	projectURL  *url.URL
+	projectPath string
+	domain      string
+)
+
+type filesBase struct {
+	pages []string
+	css   []string
+	js    []string
+	img   []string
+	font  []string
+}
+
+func contains(slice []string, sub string) bool {
+	for _, s := range slice {
+		if s == sub {
+			return true
+		}
+	}
+	return false
+}