Merge pull request #1 from TheRealLorenz/features/encoding-rs

Decode response body with mime and encoding-rs

rq-cli/src/components/response_panel.rs

@@ -5,8 +5,11 @@ use ratatui::{
     text::{Line, Span},
     widgets::{Block, Borders, Paragraph, Scrollbar, ScrollbarState, Wrap},
 };
-use rq_core::request::{Content, Response, StatusCode};
-use std::fmt::{Display, Write};
+use rq_core::request::{mime::Payload, Response, StatusCode};
+use std::{
+    fmt::{Display, Write},
+    iter,
+};
 use tui_input::Input;
 
 use super::{
@@ -59,6 +62,7 @@ pub struct ResponsePanel {
     input_popup: Option<Popup<Input>>,
     save_option: SaveOption,
     save_menu: Option<Popup<Menu<SaveOption>>>,
+    show_raw: bool,
 }
 
 impl ResponsePanel {
@@ -80,9 +84,9 @@ impl ResponsePanel {
         self.scroll = self.scroll.saturating_sub(1);
     }
 
-    fn body(&self) -> anyhow::Result<Content> {
+    fn body(&self) -> anyhow::Result<Payload> {
         match &self.state {
-            State::Received(response) => Ok(response.body.clone()),
+            State::Received(response) => Ok(response.payload.clone()),
             State::Empty | State::Loading => Err(anyhow!("Request not sent")),
         }
     }
@@ -98,18 +102,44 @@ impl ResponsePanel {
                         acc
                     });
 
+                let body = self.body_as_string().join("\n");
+
                 let s = format!(
-                    "{} {}\n{headers}\n\n{}",
-                    response.version,
-                    response.status,
-                    self.body()?
+                    "{} {}\n{headers}\n\n{body}",
+                    response.version, response.status
                 );
 
                 Ok(s)
             }
             State::Empty | State::Loading => Err(anyhow!("Request not sent")),
         }
     }
+
+    fn body_as_string(&self) -> Vec<String> {
+        match self.body() {
+            Ok(body) => match body {
+                Payload::Text(t) => iter::once(format!("decoded with encoding: '{}'", t.charset))
+                    .chain(t.text.lines().map(|s| s.to_string()))
+                    .collect(),
+                Payload::Bytes(b) if self.show_raw => iter::once("lossy utf-8 decode".to_string())
+                    .chain(
+                        String::from_utf8_lossy(&b.bytes)
+                            .lines()
+                            .map(|s| s.to_string()),
+                    )
+                    .collect(),
+                Payload::Bytes(_) => vec!["raw bytes".into()],
+            },
+            Err(e) => vec![e.to_string()],
+        }
+    }
+
+    fn render_body(&self) -> Vec<Line> {
+        let mut lines: Vec<Line> = self.body_as_string().into_iter().map(Line::from).collect();
+        lines[0].patch_style(Style::default().add_modifier(Modifier::ITALIC));
+
+        lines
+    }
 }
 
 impl BlockComponent for ResponsePanel {
@@ -127,8 +157,8 @@ impl BlockComponent for ResponsePanel {
                     let to_save = match self.save_option {
                         SaveOption::All => self.to_string()?.into(),
                         SaveOption::Body => match self.body()? {
-                            Content::Bytes(b) => b,
-                            Content::Text(t) => t.into(),
+                            Payload::Bytes(b) => b.bytes,
+                            Payload::Text(t) => t.text.into(),
                         },
                     };
 
@@ -148,26 +178,39 @@ impl BlockComponent for ResponsePanel {
             }
         }
 
-        if let Some(menu) = self.save_menu.as_mut() {
-            match menu.on_event(key_event)? {
-                HandleSuccess::Consumed => return Ok(HandleSuccess::Consumed),
-                HandleSuccess::Ignored => (),
-            }
-
-            match key_event.code {
-                KeyCode::Enter => {
-                    self.save_option = *menu.selected();
-                    self.save_menu = None;
-                    self.input_popup = Some(Popup::new(Input::from("")));
-
-                    return Ok(HandleSuccess::Consumed);
+        if self.save_menu.is_some() {
+            let extension = self
+                .body()
+                .ok()
+                .map(|payload| match payload {
+                    Payload::Bytes(b) => b.extension.unwrap_or_default(),
+                    Payload::Text(t) => t.extension.unwrap_or_default(),
+                })
+                .unwrap_or_default();
+
+            if let Some(menu) = self.save_menu.as_mut() {
+                match menu.on_event(key_event)? {
+                    HandleSuccess::Consumed => return Ok(HandleSuccess::Consumed),
+                    HandleSuccess::Ignored => (),
                 }
-                KeyCode::Esc => {
-                    self.save_menu = None;
 
-                    return Ok(HandleSuccess::Consumed);
+                match key_event.code {
+                    KeyCode::Enter => {
+                        self.save_option = *menu.selected();
+                        self.save_menu = None;
+                        self.input_popup = Some(Popup::new(
+                            Input::from(".".to_string() + extension.as_str()).with_cursor(0),
+                        ));
+
+                        return Ok(HandleSuccess::Consumed);
+                    }
+                    KeyCode::Esc => {
+                        self.save_menu = None;
+
+                        return Ok(HandleSuccess::Consumed);
+                    }
+                    _ => (),
                 }
-                _ => (),
             }
         }
 
@@ -189,11 +232,6 @@ impl BlockComponent for ResponsePanel {
         area: ratatui::prelude::Rect,
         block: ratatui::widgets::Block,
     ) {
-        let body = match self.body() {
-            Ok(x) => x.to_string(),
-            Err(e) => e.to_string(),
-        };
-
         let content = match &self.state {
             State::Received(response) => {
                 let mut lines = vec![];
@@ -222,9 +260,7 @@ impl BlockComponent for ResponsePanel {
                 // Body
                 // with initial empty line
                 lines.push(Line::from(""));
-                for line in body.lines() {
-                    lines.push(line.into());
-                }
+                lines.append(&mut self.render_body());
 
                 lines
             }
@@ -245,7 +281,7 @@ impl BlockComponent for ResponsePanel {
         let content_length = content.len();
 
         let component = Paragraph::new(content)
-            .wrap(Wrap { trim: true })
+            .wrap(Wrap { trim: false })
             .scroll((self.scroll, 0))
             .block(block);
 

rq-core/Cargo.toml

@@ -12,3 +12,5 @@ pest = "2.7.4"
 pest_derive = "2.7.4"
 once_cell = "1.18.0"
 reqwest = { version = "0.11", features = ["json"] }
+encoding_rs = "0.8.33"
+mime = "0.3.17"

rq-core/src/request.rs

@@ -1,12 +1,16 @@
 extern crate reqwest;
 
-use bytes::Bytes;
 use once_cell::sync::Lazy;
 pub use reqwest::StatusCode;
 use reqwest::{header::HeaderMap, Client};
 
 use crate::parser::HttpRequest;
-use std::{fmt::Display, time::Duration};
+use std::time::Duration;
+
+use self::mime::Payload;
+
+mod decode;
+pub mod mime;
 
 static CLIENT: Lazy<Client> = Lazy::new(|| {
     Client::builder()
@@ -16,50 +20,26 @@ static CLIENT: Lazy<Client> = Lazy::new(|| {
         .unwrap()
 });
 
-#[derive(Clone)]
-pub enum Content {
-    Bytes(Bytes),
-    Text(String),
-}
-
-impl Display for Content {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Content::Bytes(_) => write!(f, "<raw bytes>"),
-            Content::Text(s) => write!(f, "{s}"),
-        }
-    }
-}
-
-impl From<Bytes> for Content {
-    fn from(value: Bytes) -> Self {
-        match String::from_utf8(value.clone().into()) {
-            Ok(s) => Content::Text(s),
-            Err(_) => Content::Bytes(value),
-        }
-    }
-}
-
 #[derive(Clone)]
 pub struct Response {
     pub status: StatusCode,
     pub version: String,
+    pub payload: Payload,
     pub headers: HeaderMap,
-    pub body: Content,
 }
 
 impl Response {
     async fn from_reqwest(value: reqwest::Response) -> Self {
         let status = value.status();
         let version = format!("{:?}", value.version());
         let headers = value.headers().clone();
-        let body = value.bytes().await.unwrap().into();
+        let payload = Payload::of_response(value).await;
 
         Self {
             status,
             version,
             headers,
-            body,
+            payload,
         }
     }
 }

rq-core/src/request/decode.rs

@@ -0,0 +1,12 @@
+use bytes::Bytes;
+use encoding_rs::{Encoding, UTF_8};
+
+pub async fn decode_with_encoding(
+    bytes: Bytes,
+    encoding_name: &str,
+) -> (String, &'static Encoding) {
+    let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8);
+
+    let (text, encoding, _) = encoding.decode(&bytes);
+    (text.into_owned(), encoding)
+}

rq-core/src/request/mime.rs

@@ -0,0 +1,85 @@
+use bytes::Bytes;
+use mime::{Mime, Name};
+use reqwest::{header::CONTENT_TYPE, Response};
+
+use super::decode::decode_with_encoding;
+
+#[derive(Debug, Clone)]
+pub struct BytePayload {
+    pub extension: Option<String>,
+    pub bytes: Bytes,
+}
+
+#[derive(Debug, Clone)]
+pub struct TextPayload {
+    pub extension: Option<String>,
+    pub charset: String,
+    pub text: String,
+}
+
+#[derive(Debug, Clone)]
+pub enum Payload {
+    Bytes(BytePayload),
+    Text(TextPayload),
+}
+
+impl Payload {
+    pub async fn of_response(response: Response) -> Payload {
+        let mime = response
+            .headers()
+            .get(CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<Mime>().ok());
+
+        match mime {
+            Some(mime) => {
+                let extension = mime.subtype();
+
+                match (mime.type_(), extension) {
+                    (mime::TEXT, _) | (_, mime::JSON) => {
+                        let charset = mime
+                            .get_param("charset")
+                            .map(|charset| charset.to_string())
+                            .unwrap_or("utf-8".into());
+                        let (text, encoding) =
+                            decode_with_encoding(response.bytes().await.unwrap(), &charset).await;
+                        Payload::Text(TextPayload {
+                            charset: encoding.name().to_owned(),
+                            text,
+                            extension: parse_extension(extension),
+                        })
+                    }
+                    (_, extension) => Payload::Bytes(BytePayload {
+                        extension: parse_extension(extension),
+                        bytes: response.bytes().await.unwrap(),
+                    }),
+                }
+            }
+            None => Payload::Bytes(BytePayload {
+                extension: None,
+                bytes: response.bytes().await.unwrap(),
+            }),
+        }
+    }
+}
+
+fn parse_extension(name: Name) -> Option<String> {
+    match name {
+        mime::PDF => Some("pdf"),
+        mime::HTML => Some("html"),
+        mime::BMP => Some("bmp"),
+        mime::CSS => Some("css"),
+        mime::CSV => Some("csv"),
+        mime::GIF => Some("gif"),
+        mime::JAVASCRIPT => Some("js"),
+        mime::JPEG => Some("jpg"),
+        mime::JSON => Some("json"),
+        mime::MP4 => Some("mp4"),
+        mime::MPEG => Some("mpeg"),
+        mime::PNG => Some("png"),
+        mime::SVG => Some("svg"),
+        mime::XML => Some("xml"),
+        _ => None,
+    }
+    .map(|extension| extension.to_string())
+}