Replace Backend with Cross-Platform chromedp; Add go-modules support

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.go]
+indent_style = tab

.gitignore

@@ -7,3 +7,4 @@ html/
 aquatone_report.html
 aquatone_urls.txt
 aquatone_session.json
+debug.log

README.md

@@ -8,9 +8,17 @@ Aquatone is a tool for visual inspection of websites across a large amount of ho
 2. Download the [latest release](https://github.com/michenriksen/aquatone/releases/latest) of Aquatone for your operating system.
 3. Uncompress the zip file and move the `aquatone` binary to your desired location. You probably want to move it to a location in your `$PATH` for easier use.
 
-### Compiling the source code
+### From Source
+
+You'll need a working installation of Go with go 1.11+ modules support.
+
+```bash
+go get -u github.com/michenriksen/aquatone/...
+# if you already have $GOPATH/bin in your $PATH, you won't need this:
+aquatone || echo 'export $PATH="$PATH:'"$(go env GOPATH)/bin"'"' >> "$(ls ~/.bashrc || ls ~/.bash_profile)"
+aquatone || . "$(ls ~/.bashrc || ~/.bash_profile)"
+```
 
-If you for some reason don't trust the pre-compiled binaries, you can also compile the code yourself. **You are on your own if you want to do this. I do not support compiling problems. Good luck with it!**
 
 ## Usage
 

agents/url_screenshotter.go

@@ -6,11 +6,9 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"regexp"
-	"strconv"
-	"strings"
 	"time"
 
+	"github.com/chromedp/chromedp"
 	"github.com/michenriksen/aquatone/core"
 )
 
@@ -33,7 +31,6 @@ func (a *URLScreenshotter) Register(s *core.Session) error {
 	s.EventBus.SubscribeAsync(core.SessionEnd, a.OnSessionEnd, false)
 	a.session = s
 	a.createTempUserDir()
-	a.locateChrome()
 
 	return nil
 }
@@ -69,113 +66,55 @@ func (a *URLScreenshotter) createTempUserDir() {
 	a.tempUserDirPath = dir
 }
 
-func (a *URLScreenshotter) locateChrome() {
-	if *a.session.Options.ChromePath != "" {
-		a.chromePath = *a.session.Options.ChromePath
-		return
-	}
-
-	paths := []string{
-		"/usr/bin/google-chrome",
-		"/usr/bin/google-chrome-beta",
-		"/usr/bin/google-chrome-unstable",
-		"/usr/bin/chromium-browser",
-		"/usr/bin/chromium",
-		"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
-		"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
-		"/Applications/Chromium.app/Contents/MacOS/Chromium",
-		"C:/Program Files (x86)/Google/Chrome/Application/chrome.exe",
-	}
-
-	for _, path := range paths {
-		if _, err := os.Stat(path); os.IsNotExist(err) {
-			continue
-		}
-		a.chromePath = path
+func (a URLScreenshotter) getOpts() (options []chromedp.ExecAllocatorOption) {
+	if *a.session.Options.Proxy != "" {
+		options = append(options, chromedp.ProxyServer(*a.session.Options.Proxy))
 	}
 
-	if a.chromePath == "" {
-		a.session.Out.Fatal("Unable to locate a valid installation of Chrome. Install Google Chrome or try specifying a valid location with the -chrome-path option.\n")
-		os.Exit(1)
+	if *a.session.Options.ChromePath != "" {
+		options = append(options, chromedp.ExecPath(*a.session.Options.ChromePath))
 	}
 
-	if strings.Contains(strings.ToLower(a.chromePath), "chrome") {
-		a.session.Out.Warn("Using unreliable Google Chrome for screenshots. Install Chromium for better results.\n\n")
-	} else {
-		out, err := exec.Command(a.chromePath, "--version").Output()
-		if err != nil {
-			a.session.Out.Warn("An error occurred while trying to determine version of Chromium.\n\n")
-			return
-		}
-		version := string(out)
-		re := regexp.MustCompile(`(\d+)\.`)
-		match := re.FindStringSubmatch(version)
-		if len(match) <= 0 {
-			a.session.Out.Warn("Unable to determine version of Chromium. Screenshotting might be unreliable.\n\n")
-			return
-		}
-		majorVersion, _ := strconv.Atoi(match[1])
-		if majorVersion < 72 {
-			a.session.Out.Warn("An older version of Chromium is installed. Screenshotting of HTTPS URLs might be unreliable.\n\n")
-		}
-	}
+	return
+}
 
-	a.session.Out.Debug("[%s] Located Chrome/Chromium binary at %s\n", a.ID(), a.chromePath)
+// execAllocator turns a.getOpts() (the chrome instance allocator options) into a derivative context.Context
+func (a URLScreenshotter) execAllocator(parent context.Context) (context.Context, context.CancelFunc) {
+	return chromedp.NewExecAllocator(parent, a.getOpts()...)
 }
 
 func (a *URLScreenshotter) screenshotPage(page *core.Page) {
 	filePath := fmt.Sprintf("screenshots/%s.png", page.BaseFilename())
-	var chromeArguments = []string{
-		"--headless", "--disable-gpu", "--hide-scrollbars", "--mute-audio", "--disable-notifications",
-		"--no-first-run", "--disable-crash-reporter", "--ignore-certificate-errors", "--incognito",
-		"--disable-infobars", "--disable-sync", "--no-default-browser-check",
-		"--user-data-dir=" + a.tempUserDirPath,
-		"--user-agent=" + RandomUserAgent(),
-		"--window-size=" + *a.session.Options.Resolution,
-		"--screenshot=" + a.session.GetFilePath(filePath),
-	}
-
-	if os.Geteuid() == 0 {
-		chromeArguments = append(chromeArguments, "--no-sandbox")
-	}
-
-	if *a.session.Options.Proxy != "" {
-		chromeArguments = append(chromeArguments, "--proxy-server="+*a.session.Options.Proxy)
-	}
-
-	chromeArguments = append(chromeArguments, page.URL)
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*a.session.Options.ScreenshotTimeout)*time.Millisecond)
+	ctx, cancel = a.execAllocator(ctx)
+	ctx, cancel = chromedp.NewContext(ctx)
+
 	defer cancel()
 
-	cmd := exec.CommandContext(ctx, a.chromePath, chromeArguments...)
-	if err := cmd.Start(); err != nil {
-		a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err)
+	var pic []byte
+	if err := chromedp.Run(ctx, chromedp.Tasks{
+		chromedp.Navigate(page.URL),
+		chromedp.WaitReady("body", chromedp.ByQuery),
+		chromedp.Screenshot("body", &pic, chromedp.NodeVisible, chromedp.ByQuery),
+	}); err != nil {
+		a.session.Out.Debug("%s Error: %v\n", a.ID, err)
 		a.session.Stats.IncrementScreenshotFailed()
 		a.session.Out.Error("%s: screenshot failed: %s\n", page.URL, err)
-		a.killChromeProcessIfRunning(cmd)
 		return
 	}
 
-	if err := cmd.Wait(); err != nil {
+	if err := ioutil.WriteFile(filePath, pic, 0700); err != nil {
+		a.session.Out.Debug("%s Error: %v\n", a.ID(), err)
 		a.session.Stats.IncrementScreenshotFailed()
-		a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err)
-		if ctx.Err() == context.DeadlineExceeded {
-			a.session.Out.Error("%s: screenshot timed out\n", page.URL)
-			a.killChromeProcessIfRunning(cmd)
-			return
-		}
-
 		a.session.Out.Error("%s: screenshot failed: %s\n", page.URL, err)
-		a.killChromeProcessIfRunning(cmd)
 		return
 	}
 
 	a.session.Stats.IncrementScreenshotSuccessful()
 	a.session.Out.Info("%s: %s\n", page.URL, Green("screenshot successful"))
 	page.ScreenshotPath = filePath
 	page.HasScreenshot = true
-	a.killChromeProcessIfRunning(cmd)
 }
 
 func (a *URLScreenshotter) killChromeProcessIfRunning(cmd *exec.Cmd) {

go.mod

@@ -0,0 +1,24 @@
+module github.com/michenriksen/aquatone
+
+go 1.12
+
+require (
+	github.com/PuerkitoBio/goquery v1.5.0
+	github.com/asaskevich/EventBus v0.0.0-20180315140547-d46933a94f05
+	github.com/chromedp/chromedp v0.5.3
+	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/fatih/color v1.7.0
+	github.com/google/uuid v1.1.1
+	github.com/lair-framework/go-nmap v0.0.0-20181105160706-3b9bafddefee
+	github.com/mattn/go-colorable v0.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.7 // indirect
+	github.com/moul/http2curl v1.0.0 // indirect
+	github.com/parnurzeal/gorequest v0.2.15
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
+	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
+	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
+	mvdan.cc/xurls/v2 v2.0.0
+)