-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
9 changed files
with
584 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,242 @@ | ||
| package censys | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/base64" | ||
| "encoding/json" | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "log" | ||
| "net/http" | ||
| "path" | ||
| "time" | ||
|
|
||
| "github.com/lanrat/certgraph/driver" | ||
| "github.com/lanrat/certgraph/fingerprint" | ||
| "github.com/lanrat/certgraph/status" | ||
| ) | ||
|
|
||
| const driverName = "censys" | ||
|
|
||
| var debug = false | ||
|
|
||
| // TODO support rate limits & pagnation | ||
|
|
||
| var ( | ||
| defaultHTTPClient = &http.Client{} | ||
|
|
||
| appID = flag.String("censys-appid", "", "censys API AppID") | ||
| secret = flag.String("censys-secret", "", "censys API Secret") | ||
| ) | ||
|
|
||
| func init() { | ||
| driver.AddDriver(driverName) | ||
| } | ||
|
|
||
| type censys struct { | ||
| appID string | ||
| secret string | ||
| save bool | ||
| savePath string | ||
| includeSubdomains bool | ||
| includeExpired bool | ||
| } | ||
|
|
||
| type censysCertDriver struct { | ||
| host string | ||
| fingerprints driver.FingerprintMap | ||
| driver *censys | ||
| } | ||
|
|
||
| func (c *censysCertDriver) GetFingerprints() (driver.FingerprintMap, error) { | ||
| return c.fingerprints, nil | ||
| } | ||
|
|
||
| func (c *censysCertDriver) GetStatus() status.Map { | ||
| return status.NewMap(c.host, status.New(status.CT)) | ||
| } | ||
|
|
||
| func (c *censysCertDriver) GetRelated() ([]string, error) { | ||
| return make([]string, 0), nil | ||
| } | ||
|
|
||
| func (c *censysCertDriver) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) { | ||
| return c.driver.QueryCert(fp) | ||
| } | ||
|
|
||
| // TODO support pagnation | ||
| func domainSearchParam(domain string, includeExpired, includeSubdomain bool) certSearchParam { | ||
| var s certSearchParam | ||
| if includeSubdomain { | ||
| s.Query = fmt.Sprintf("(parsed.names: %s )", domain) | ||
| } else { | ||
| s.Query = fmt.Sprintf("(parsed.names.raw: %s)", domain) | ||
| } | ||
| if !includeExpired { | ||
| dateStr := time.Now().Format("2006-01-02") // YYYY-MM-DD | ||
| expQuery := fmt.Sprintf(" AND ((parsed.validity.end: [%s TO *]) AND (parsed.validity.start: [* TO %s]))", dateStr, dateStr) | ||
| s.Query = s.Query + expQuery | ||
| } | ||
| s.Page = 1 | ||
| s.Flatten = true | ||
| s.Fields = []string{"parsed.fingerprint_sha256", "parsed.names"} | ||
| return s | ||
| } | ||
|
|
||
| // Driver creates a new CT driver for censys | ||
| func Driver(savePath string, includeSubdomains, includeExpired bool) (driver.Driver, error) { | ||
| if *appID == "" || *secret == "" { | ||
| return nil, fmt.Errorf("censys requires an appID and secret to run") | ||
| } | ||
| d := new(censys) | ||
| d.appID = *appID | ||
| d.secret = *secret | ||
| d.savePath = savePath | ||
| d.includeSubdomains = includeSubdomains | ||
| d.includeExpired = includeExpired | ||
| return d, nil | ||
| } | ||
|
|
||
| func (d *censys) GetName() string { | ||
| return driverName | ||
| } | ||
|
|
||
| func (d *censys) request(method, url string, request io.Reader) (*http.Response, error) { | ||
| totalTrys := 3 | ||
| var err error | ||
| var req *http.Request | ||
| var resp *http.Response | ||
| for try := 1; try <= totalTrys; try++ { | ||
| req, err = http.NewRequest(method, url, request) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if request != nil { | ||
| req.Header.Add("Content-Type", "application/json") | ||
| } | ||
| req.Header.Add("Accept", "application/json") | ||
| req.SetBasicAuth(d.appID, d.secret) | ||
|
|
||
| resp, err = defaultHTTPClient.Do(req) | ||
| if err != nil { | ||
| err = fmt.Errorf("error on request [%d/%d] %s, got error %w: %+v", try, totalTrys, url, err, resp) | ||
| } else { | ||
| return resp, nil | ||
| } | ||
|
|
||
| // sleep only if we will try again | ||
| if try < totalTrys { | ||
| time.Sleep(time.Second * 10) | ||
| } | ||
| } | ||
| return resp, err | ||
| } | ||
|
|
||
| // jsonRequest performes a request to the API endpoint sending and receiving JSON objects | ||
| func (d *censys) jsonRequest(method, url string, request, response interface{}) error { | ||
| var payloadReader io.Reader | ||
| if request != nil { | ||
| jsonPayload, err := json.Marshal(request) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| payloadReader = bytes.NewReader(jsonPayload) | ||
| } | ||
|
|
||
| if debug { | ||
| log.Printf("DEBUG: request to %s %s", method, url) | ||
| if request != nil { | ||
| prettyJSONBytes, _ := json.MarshalIndent(request, "", "\t") | ||
| log.Printf("request payload:\n%s\n", string(prettyJSONBytes)) | ||
| } | ||
| } | ||
|
|
||
| resp, err := d.request(method, url, payloadReader) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| // got an error, decode it | ||
| if resp.StatusCode != http.StatusOK { | ||
| var errorResp errorResponse | ||
| err := fmt.Errorf("error on request %s, got Status %s %s", url, resp.Status, http.StatusText(resp.StatusCode)) | ||
| jsonError := json.NewDecoder(resp.Body).Decode(&errorResp) | ||
| if jsonError != nil { | ||
| return fmt.Errorf("error decoding json %w on errord request: %s", jsonError, err.Error()) | ||
| } | ||
| return fmt.Errorf("%w, HTTPStatus: %d Message: %q", err, errorResp.ErrorCode, errorResp.Error) | ||
| } | ||
|
|
||
| if response != nil { | ||
| err = json.NewDecoder(resp.Body).Decode(&response) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if debug { | ||
| prettyJSONBytes, _ := json.MarshalIndent(response, "", "\t") | ||
| log.Printf("response payload:\n%s\n", string(prettyJSONBytes)) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (d *censys) QueryDomain(domain string) (driver.Result, error) { | ||
| results := &censysCertDriver{ | ||
| host: domain, | ||
| fingerprints: make(driver.FingerprintMap), | ||
| driver: d, | ||
| } | ||
| params := domainSearchParam(domain, d.includeExpired, d.includeSubdomains) | ||
| url := "https://search.censys.io/api/v1/search/certificates" | ||
| var resp certSearchResponse | ||
| err := d.jsonRequest(http.MethodPost, url, params, &resp) | ||
| if err != nil { | ||
| return results, err | ||
| } | ||
|
|
||
| for _, r := range resp.Results { | ||
| fp := fingerprint.FromHexHash(r.Fingerprint) | ||
| results.fingerprints.Add(domain, fp) | ||
| } | ||
|
|
||
| if debug { | ||
| log.Printf("censys: got %d results for %s.", len(resp.Results), domain) | ||
| } | ||
|
|
||
| return results, nil | ||
| } | ||
|
|
||
| func (d *censys) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) { | ||
| certNode := new(driver.CertResult) | ||
| certNode.Fingerprint = fp | ||
| certNode.Domains = make([]string, 0, 5) | ||
|
|
||
| url := fmt.Sprintf("https://search.censys.io/api/v1/view/certificates/%s", fp.HexString()) | ||
| var resp certViewResponse | ||
| err := d.jsonRequest(http.MethodGet, url, nil, &resp) | ||
| if err != nil { | ||
| return certNode, err | ||
| } | ||
|
|
||
| if debug { | ||
| log.Printf("DEBUG QueryCert(%s): %v", fp.HexString(), resp.Parsed.Names) | ||
| } | ||
|
|
||
| certNode.Domains = append(certNode.Domains, resp.Parsed.Names...) | ||
|
|
||
| if d.save { | ||
| rawCert, err := base64.StdEncoding.DecodeString(resp.Raw) | ||
| if err != nil { | ||
| return certNode, err | ||
| } | ||
| err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem") | ||
| if err != nil { | ||
| return certNode, err | ||
| } | ||
| } | ||
|
|
||
| return certNode, nil | ||
| } |
Oops, something went wrong.