fix read body

engine/Cargo.toml

@@ -18,7 +18,7 @@ jsonpath-rust = "0.7.1"
 rustc_lexer = "0.1.0"
 serde_json = "1"
 #git = "https://github.com/emo-crab/slinger",
-slinger = { version = "0.1.9", features = [
+slinger = { version = "0.1.11", features = [
   "serde",
   "cookie",
   "tls",

engine/src/common/html.rs

@@ -1,3 +1,8 @@
+use super::uri::join;
+use fancy_regex::Regex;
+use std::str::FromStr;
+use std::sync::OnceLock;
+
 /// 获取标题
 pub fn extract_title(text: &str) -> Option<String> {
   let dom = tl::parse(text, tl::ParserOptions::default()).ok()?;
@@ -35,3 +40,60 @@ pub fn extract_title(text: &str) -> Option<String> {
   }
   None
 }
+static RE: OnceLock<Vec<Regex>> = OnceLock::new();
+pub fn extract_redirect(text: &str, cur_uri: &slinger::http::Uri) -> Option<slinger::http::Uri> {
+  let re = RE.get_or_init(|| {
+    let js_reg = [
+      r#"(?im)location\.(open|replace|href)=['"]\s*?(?P<name>.*?)['"]"#,
+      r#"(?im)location\.(open|replace|href)\((?P<name>.*?)\)"#,
+    ];
+    let re_list: Vec<Regex> = js_reg
+      .iter()
+      .map(|reg| Regex::new(reg).expect("RE_COMPILE_BY_JUMP"))
+      .collect();
+    re_list
+  });
+  let mut next_url_list = Vec::new();
+  let dom = tl::parse(text, tl::ParserOptions::default()).ok()?;
+  let parser = dom.parser();
+  if let Some(selector) = dom.query_selector("meta[http-equiv=refresh]") {
+    for meta in selector {
+      if let Some(meta) = meta.get(parser) {
+        let content = meta
+          .as_tag()
+          .and_then(|tag| {
+            tag
+              .attributes()
+              .get("content")
+              .and_then(|x| x.and_then(|x| x.try_as_utf8_str()))
+          })
+          .map(|x| x.trim().to_string())
+          .unwrap_or_default();
+        if let Some((_, u)) = content.split_once('=') {
+          let n = u.replace(['\'', '\"'], "");
+          next_url_list.push(n);
+        }
+      }
+    }
+  }
+  if next_url_list.is_empty() && text.len() <= 1024 {
+    for reg in re.iter() {
+      if let Ok(Some(x)) = reg.captures(text) {
+        let mut u = x.name("name").map_or("", |m| m.as_str()).to_string();
+        u = u.replace(['\'', '\"'], "");
+        next_url_list.push(u);
+      }
+    }
+  }
+  if let Some(next_url) = next_url_list.into_iter().next() {
+    return if next_url.starts_with("http://") || next_url.starts_with("https://") {
+      match slinger::http::Uri::from_str(&next_url) {
+        Ok(next_path) => Some(next_path),
+        Err(_) => None,
+      }
+    } else {
+      join(cur_uri, &next_url)
+    };
+  };
+  None
+}

engine/src/common/http.rs

@@ -1,3 +1,5 @@
+use super::html::extract_redirect;
+use super::uri::join;
 use crate::error::Error;
 use crate::matchers::FaviconMap;
 use base64::engine::general_purpose::STANDARD;
@@ -6,12 +8,9 @@ use md5::{Digest, Md5};
 use mime::Mime;
 use slinger::http::header;
 use slinger::http::header::HeaderMap;
-use slinger::http::uri::Uri;
 use slinger::{Body, ClientBuilder, Response};
 use std::collections::{BTreeMap, HashSet};
-use std::path::PathBuf;
 use std::str::FromStr;
-
 #[derive(Debug, Clone)]
 pub struct HttpRecord {
   response: Response,
@@ -87,6 +86,25 @@ impl HttpRecord {
   }
 }
 
+pub fn js_redirect(attempt: slinger::redirect::Attempt) -> slinger::redirect::Action {
+  match attempt.default_redirect() {
+    Some(next) => attempt.follow(next),
+    None => {
+      let body = attempt.response().text().unwrap_or_default();
+      match extract_redirect(&body, attempt.response().uri()) {
+        Some(next) => {
+          if attempt.previous().len() > 10 || next.to_string() == attempt.url().to_string() {
+            attempt.stop(next)
+          } else {
+            attempt.follow(next)
+          }
+        }
+        None => attempt.none(),
+      }
+    }
+  }
+}
+
 fn favicon_hash(content: &Body) -> FaviconMap {
   let mut hasher = Md5::new();
   hasher.update(content.to_vec());
@@ -162,16 +180,6 @@ fn get_favicon_link(response: &Response) -> HashSet<String> {
   icon_links
 }
 
-fn join(cur_uri: &Uri, val: &str) -> Option<Uri> {
-  let path = PathBuf::from(cur_uri.path()).join(val);
-  Uri::builder()
-    .scheme(cur_uri.scheme_str().unwrap_or_default())
-    .authority(cur_uri.authority()?.as_str())
-    .path_and_query(path.to_string_lossy().as_ref())
-    .build()
-    .ok()
-}
-
 pub fn murmur3_32(buf: &[u8], seed: u32) -> i32 {
   const fn pre_mix(buf: [u8; 4]) -> u32 {
     u32::from_le_bytes(buf)

engine/src/common/mod.rs

@@ -2,6 +2,7 @@ pub mod cert;
 mod generator;
 pub mod html;
 pub mod http;
+pub mod uri;
 // mod marker;
 
 pub use generator::{PayloadAttack, PayloadIterator};

engine/src/common/uri.rs

@@ -0,0 +1,11 @@
+use slinger::http::uri::Uri;
+use std::path::PathBuf;
+pub(crate) fn join(cur_uri: &Uri, val: &str) -> Option<Uri> {
+  let path = PathBuf::from(cur_uri.path()).join(val);
+  Uri::builder()
+    .scheme(cur_uri.scheme_str().unwrap_or_default())
+    .authority(cur_uri.authority()?.as_str())
+    .path_and_query(path.to_string_lossy().as_ref())
+    .build()
+    .ok()
+}

engine/src/request/http/option.rs

@@ -50,7 +50,7 @@ impl HttpOption {
   pub fn builder_client(&self) -> slinger::ClientBuilder {
     let redirect = if self.redirects {
       if self.host_redirects {
-        slinger::redirect::Policy::Custom(slinger::redirect::only_same_host)
+        slinger::redirect::Policy::Custom(crate::common::http::js_redirect)
       } else {
         slinger::redirect::Policy::Limit(self.max_redirects.unwrap_or(10))
       }

observer_ward/src/cli.rs

@@ -6,7 +6,7 @@ use engine::find_yaml_file;
 use engine::slinger::http::header::HeaderValue;
 use engine::slinger::http::Uri;
 use engine::slinger::http_serde;
-use engine::slinger::redirect::{only_same_host, Policy};
+use engine::slinger::redirect::Policy;
 use engine::slinger::{openssl, ClientBuilder, ConnectorBuilder, Proxy};
 use engine::template::Template;
 use log::{error, warn};
@@ -328,7 +328,7 @@ impl ObserverWardConfig {
       .danger_accept_invalid_certs(true)
       .danger_accept_invalid_hostnames(true)
       .min_tls_version(Some(engine::slinger::native_tls::Protocol::Tlsv10))
-      .redirect(Policy::Custom(only_same_host))
+      .redirect(Policy::Custom(engine::common::http::js_redirect))
       .timeout(Some(Duration::from_secs(self.timeout)));
     if let Ok(ua) = HeaderValue::from_str(&self.ua) {
       client_builder = client_builder.user_agent(ua);

observer_ward/src/lib.rs

@@ -12,7 +12,7 @@ use engine::results::{FingerprintResult, NucleiResult};
 use engine::slinger::http::header::HeaderValue;
 use engine::slinger::http::uri::{PathAndQuery, Uri};
 use engine::slinger::http::StatusCode;
-use engine::slinger::redirect::{only_same_host, Policy};
+use engine::slinger::redirect::Policy;
 use engine::slinger::{http_serde, Request, Response};
 use engine::template::Template;
 use error::Result;
@@ -222,7 +222,7 @@ impl ClusterExecuteRunner {
     for http in cluster.requests.http.iter() {
       let mut client_builder = http.http_option.builder_client();
       client_builder = client_builder.timeout(Some(Duration::from_secs(config.timeout)));
-      client_builder = client_builder.redirect(Policy::Custom(only_same_host));
+      client_builder = client_builder.redirect(Policy::Custom(engine::common::http::js_redirect));
       if let Ok(ua) = HeaderValue::from_str(&config.ua) {
         client_builder = client_builder.user_agent(ua);
       }