From 1977d1e984fb743ddb11cf50883ce489ed0adcb7 Mon Sep 17 00:00:00 2001 From: TheRealLorenz Date: Wed, 22 Nov 2023 12:30:31 +0100 Subject: [PATCH 1/4] Decoding based on mime type --- rq-cli/src/components/response_panel.rs | 61 +++++++++++++++------- rq-core/Cargo.toml | 2 + rq-core/src/request.rs | 38 ++++---------- rq-core/src/request/decode.rs | 12 +++++ rq-core/src/request/mime.rs | 67 +++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 48 deletions(-) create mode 100644 rq-core/src/request/decode.rs create mode 100644 rq-core/src/request/mime.rs diff --git a/rq-cli/src/components/response_panel.rs b/rq-cli/src/components/response_panel.rs index 84d9f6b..4bd3c24 100644 --- a/rq-cli/src/components/response_panel.rs +++ b/rq-cli/src/components/response_panel.rs @@ -1,12 +1,15 @@ use anyhow::anyhow; use crossterm::event::KeyCode; use ratatui::{ - style::{Color, Style}, + style::{Color, Modifier, Style}, text::{Line, Span}, widgets::{Block, Borders, Paragraph, Scrollbar, ScrollbarState, Wrap}, }; -use rq_core::request::{Content, Response, StatusCode}; -use std::fmt::{Display, Write}; +use rq_core::request::{mime::Payload, Response, StatusCode}; +use std::{ + fmt::{Display, Write}, + iter, +}; use tui_input::Input; use super::{ @@ -51,6 +54,7 @@ pub struct ResponsePanel { input_popup: Option>, save_option: SaveOption, save_menu: Option>>, + show_raw: bool, } impl From for ResponsePanel { @@ -73,9 +77,9 @@ impl ResponsePanel { self.scroll = self.scroll.saturating_sub(1); } - fn body(&self) -> anyhow::Result { + fn body(&self) -> anyhow::Result { match &self.content { - Some(response) => Ok(response.body.clone()), + Some(response) => Ok(response.payload.clone()), None => Err(anyhow!("Request not sent")), } } @@ -91,11 +95,11 @@ impl ResponsePanel { acc }); + let body = self.body_as_string().join("\n"); + let s = format!( - "{} {}\n{headers}\n\n{}", - response.version, - response.status, - self.body()? + "{} {}\n{headers}\n\n{body}", + response.version, response.status ); Ok(s) @@ -103,6 +107,32 @@ impl ResponsePanel { None => Err(anyhow!("Request not sent")), } } + + fn body_as_string(&self) -> Vec { + match self.body() { + Ok(body) => match body { + Payload::Text(t) => iter::once(format!("decoded with encoding: '{}'", t.charset)) + .chain(t.text.lines().map(|s| s.to_string())) + .collect(), + Payload::Bytes(b) if self.show_raw => iter::once("lossy utf-8 decode".to_string()) + .chain( + String::from_utf8_lossy(&b.bytes) + .lines() + .map(|s| s.to_string()), + ) + .collect(), + Payload::Bytes(_) => vec!["raw bytes".into()], + }, + Err(e) => vec![e.to_string()], + } + } + + fn render_body(&self) -> Vec { + let mut lines: Vec = self.body_as_string().into_iter().map(Line::from).collect(); + lines[0].patch_style(Style::default().add_modifier(Modifier::ITALIC)); + + lines + } } impl BlockComponent for ResponsePanel { @@ -120,8 +150,8 @@ impl BlockComponent for ResponsePanel { let to_save = match self.save_option { SaveOption::All => self.to_string()?.into(), SaveOption::Body => match self.body()? { - Content::Bytes(b) => b, - Content::Text(t) => t.into(), + Payload::Bytes(b) => b.bytes, + Payload::Text(t) => t.text.into(), }, }; @@ -182,11 +212,6 @@ impl BlockComponent for ResponsePanel { area: ratatui::prelude::Rect, block: ratatui::widgets::Block, ) { - let body = match self.body() { - Ok(x) => x.to_string(), - Err(e) => e.to_string(), - }; - let content = match &self.content { Some(response) => { let mut lines = vec![]; @@ -215,9 +240,7 @@ impl BlockComponent for ResponsePanel { // Body // with initial empty line lines.push(Line::from("")); - for line in body.lines() { - lines.push(line.into()); - } + lines.append(&mut self.render_body()); lines } diff --git a/rq-core/Cargo.toml b/rq-core/Cargo.toml index ab3c440..2a90e8b 100644 --- a/rq-core/Cargo.toml +++ b/rq-core/Cargo.toml @@ -12,3 +12,5 @@ pest = "2.7.4" pest_derive = "2.7.4" once_cell = "1.18.0" reqwest = { version = "0.11", features = ["json"] } +encoding_rs = "0.8.33" +mime = "0.3.17" diff --git a/rq-core/src/request.rs b/rq-core/src/request.rs index 0558ef7..27db961 100644 --- a/rq-core/src/request.rs +++ b/rq-core/src/request.rs @@ -1,12 +1,16 @@ extern crate reqwest; -use bytes::Bytes; use once_cell::sync::Lazy; pub use reqwest::StatusCode; use reqwest::{header::HeaderMap, Client}; use crate::parser::HttpRequest; -use std::{fmt::Display, time::Duration}; +use std::time::Duration; + +use self::mime::Payload; + +mod decode; +pub mod mime; static CLIENT: Lazy = Lazy::new(|| { Client::builder() @@ -16,36 +20,12 @@ static CLIENT: Lazy = Lazy::new(|| { .unwrap() }); -#[derive(Clone)] -pub enum Content { - Bytes(Bytes), - Text(String), -} - -impl Display for Content { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Content::Bytes(_) => write!(f, ""), - Content::Text(s) => write!(f, "{s}"), - } - } -} - -impl From for Content { - fn from(value: Bytes) -> Self { - match String::from_utf8(value.clone().into()) { - Ok(s) => Content::Text(s), - Err(_) => Content::Bytes(value), - } - } -} - #[derive(Clone)] pub struct Response { pub status: StatusCode, pub headers: HeaderMap, pub version: String, - pub body: Content, + pub payload: Payload, } impl Response { @@ -53,13 +33,13 @@ impl Response { let status = value.status(); let version = format!("{:?}", value.version()); let headers = value.headers().clone(); - let body = value.bytes().await.unwrap().into(); + let payload = Payload::of_response(value).await; Self { status, version, headers, - body, + payload, } } } diff --git a/rq-core/src/request/decode.rs b/rq-core/src/request/decode.rs new file mode 100644 index 0000000..7fa8eaf --- /dev/null +++ b/rq-core/src/request/decode.rs @@ -0,0 +1,12 @@ +use bytes::Bytes; +use encoding_rs::{Encoding, UTF_8}; + +pub async fn decode_with_encoding( + bytes: Bytes, + encoding_name: &str, +) -> (String, &'static Encoding) { + let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8); + + let (text, encoding, _) = encoding.decode(&bytes); + (text.into_owned(), encoding) +} diff --git a/rq-core/src/request/mime.rs b/rq-core/src/request/mime.rs new file mode 100644 index 0000000..0a2a000 --- /dev/null +++ b/rq-core/src/request/mime.rs @@ -0,0 +1,67 @@ +use bytes::Bytes; +use mime::Mime; +use reqwest::{header::CONTENT_TYPE, Response}; + +use super::decode::decode_with_encoding; + +#[derive(Debug, Clone)] +pub struct BytePayload { + pub extension: Option, + pub bytes: Bytes, +} + +#[derive(Debug, Clone)] +pub struct TextPayload { + pub charset: String, + pub text: String, +} + +#[derive(Debug, Clone)] +pub enum Payload { + Bytes(BytePayload), + Text(TextPayload), +} + +impl Payload { + pub async fn of_response(response: Response) -> Payload { + let mime = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + + match mime { + Some(mime) => match (mime.type_(), mime.subtype()) { + (mime::TEXT, _) => { + let charset = mime + .get_param("charset") + .map(|charset| charset.to_string()) + .unwrap_or("utf-8".into()); + let (text, encoding) = + decode_with_encoding(response.bytes().await.unwrap(), &charset).await; + Payload::Text(TextPayload { + charset: encoding.name().to_owned(), + text, + }) + } + _ => Payload::Bytes(BytePayload { + extension: None, + bytes: response.bytes().await.unwrap(), + }), + }, + None => Payload::Bytes(BytePayload { + extension: None, + bytes: response.bytes().await.unwrap(), + }), + } + } +} + +// impl Display for Payload { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// match self { +// Payload::Bytes(_) => write!(f, "raw bytes"), +// Payload::Text(t) => write!(f, "decoded with encoding: '{}'\n{}", t.charset, t.text), +// } +// } +// } From fbe61f118525d448f4327ea4b2d32d9382323e12 Mon Sep 17 00:00:00 2001 From: TheRealLorenz Date: Thu, 23 Nov 2023 16:37:47 +0100 Subject: [PATCH 2/4] Get extension from mime type --- rq-cli/src/components/response_panel.rs | 43 ++++++++++++++++--------- rq-core/src/request/mime.rs | 38 +++++++++++++++------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/rq-cli/src/components/response_panel.rs b/rq-cli/src/components/response_panel.rs index 4bd3c24..c1d4c31 100644 --- a/rq-cli/src/components/response_panel.rs +++ b/rq-cli/src/components/response_panel.rs @@ -171,26 +171,37 @@ impl BlockComponent for ResponsePanel { } } - if let Some(menu) = self.save_menu.as_mut() { - match menu.on_event(key_event)? { - HandleSuccess::Consumed => return Ok(HandleSuccess::Consumed), - HandleSuccess::Ignored => (), - } + if self.save_menu.is_some() { + let extension = self + .body() + .ok() + .map(|payload| match payload { + Payload::Bytes(b) => b.extension.unwrap_or_default(), + Payload::Text(t) => t.extension.unwrap_or_default(), + }) + .unwrap_or_default(); + + if let Some(menu) = self.save_menu.as_mut() { + match menu.on_event(key_event)? { + HandleSuccess::Consumed => return Ok(HandleSuccess::Consumed), + HandleSuccess::Ignored => (), + } - match key_event.code { - KeyCode::Enter => { - self.save_option = *menu.selected(); - self.save_menu = None; - self.input_popup = Some(Popup::new(Input::from(""))); + match key_event.code { + KeyCode::Enter => { + self.save_option = *menu.selected(); + self.save_menu = None; + self.input_popup = Some(Popup::new(Input::from(extension))); - return Ok(HandleSuccess::Consumed); - } - KeyCode::Esc => { - self.save_menu = None; + return Ok(HandleSuccess::Consumed); + } + KeyCode::Esc => { + self.save_menu = None; - return Ok(HandleSuccess::Consumed); + return Ok(HandleSuccess::Consumed); + } + _ => (), } - _ => (), } } diff --git a/rq-core/src/request/mime.rs b/rq-core/src/request/mime.rs index 0a2a000..82a6cd8 100644 --- a/rq-core/src/request/mime.rs +++ b/rq-core/src/request/mime.rs @@ -1,5 +1,5 @@ use bytes::Bytes; -use mime::Mime; +use mime::{Mime, Name}; use reqwest::{header::CONTENT_TYPE, Response}; use super::decode::decode_with_encoding; @@ -12,6 +12,7 @@ pub struct BytePayload { #[derive(Debug, Clone)] pub struct TextPayload { + pub extension: Option, pub charset: String, pub text: String, } @@ -32,7 +33,7 @@ impl Payload { match mime { Some(mime) => match (mime.type_(), mime.subtype()) { - (mime::TEXT, _) => { + (mime::TEXT, extension) => { let charset = mime .get_param("charset") .map(|charset| charset.to_string()) @@ -42,10 +43,11 @@ impl Payload { Payload::Text(TextPayload { charset: encoding.name().to_owned(), text, + extension: parse_extension(extension), }) } - _ => Payload::Bytes(BytePayload { - extension: None, + (_, extension) => Payload::Bytes(BytePayload { + extension: parse_extension(extension), bytes: response.bytes().await.unwrap(), }), }, @@ -57,11 +59,23 @@ impl Payload { } } -// impl Display for Payload { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// match self { -// Payload::Bytes(_) => write!(f, "raw bytes"), -// Payload::Text(t) => write!(f, "decoded with encoding: '{}'\n{}", t.charset, t.text), -// } -// } -// } +fn parse_extension(name: Name) -> Option { + match name { + mime::PDF => Some("pdf"), + mime::HTML => Some("html"), + mime::BMP => Some("bmp"), + mime::CSS => Some("css"), + mime::CSV => Some("csv"), + mime::GIF => Some("gif"), + mime::JAVASCRIPT => Some("js"), + mime::JPEG => Some("jpg"), + mime::JSON => Some("json"), + mime::MP4 => Some("mp4"), + mime::MPEG => Some("mpeg"), + mime::PNG => Some("png"), + mime::SVG => Some("svg"), + mime::XML => Some("xml"), + _ => None, + } + .map(|extension| extension.to_string()) +} From 35507531b34c00cbd3b3ef10cf1d2e1be9f23ebc Mon Sep 17 00:00:00 2001 From: TheRealLorenz Date: Sun, 26 Nov 2023 13:19:05 +0100 Subject: [PATCH 3/4] Handle JSON --- rq-cli/src/components/response_panel.rs | 2 +- rq-core/src/request/mime.rs | 38 ++++++++++++++----------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/rq-cli/src/components/response_panel.rs b/rq-cli/src/components/response_panel.rs index c1d4c31..eaecbee 100644 --- a/rq-cli/src/components/response_panel.rs +++ b/rq-cli/src/components/response_panel.rs @@ -261,7 +261,7 @@ impl BlockComponent for ResponsePanel { let content_length = content.len(); let component = Paragraph::new(content) - .wrap(Wrap { trim: true }) + .wrap(Wrap { trim: false }) .scroll((self.scroll, 0)) .block(block); diff --git a/rq-core/src/request/mime.rs b/rq-core/src/request/mime.rs index 82a6cd8..afc10e8 100644 --- a/rq-core/src/request/mime.rs +++ b/rq-core/src/request/mime.rs @@ -32,25 +32,29 @@ impl Payload { .and_then(|value| value.parse::().ok()); match mime { - Some(mime) => match (mime.type_(), mime.subtype()) { - (mime::TEXT, extension) => { - let charset = mime - .get_param("charset") - .map(|charset| charset.to_string()) - .unwrap_or("utf-8".into()); - let (text, encoding) = - decode_with_encoding(response.bytes().await.unwrap(), &charset).await; - Payload::Text(TextPayload { - charset: encoding.name().to_owned(), - text, + Some(mime) => { + let extension = mime.subtype(); + + match (mime.type_(), extension) { + (mime::TEXT, _) | (_, mime::JSON) => { + let charset = mime + .get_param("charset") + .map(|charset| charset.to_string()) + .unwrap_or("utf-8".into()); + let (text, encoding) = + decode_with_encoding(response.bytes().await.unwrap(), &charset).await; + Payload::Text(TextPayload { + charset: encoding.name().to_owned(), + text, + extension: parse_extension(extension), + }) + } + (_, extension) => Payload::Bytes(BytePayload { extension: parse_extension(extension), - }) + bytes: response.bytes().await.unwrap(), + }), } - (_, extension) => Payload::Bytes(BytePayload { - extension: parse_extension(extension), - bytes: response.bytes().await.unwrap(), - }), - }, + } None => Payload::Bytes(BytePayload { extension: None, bytes: response.bytes().await.unwrap(), From faefd4c92eed460231e353f3c22b2f446ee01929 Mon Sep 17 00:00:00 2001 From: TheRealLorenz Date: Sun, 26 Nov 2023 16:39:45 +0100 Subject: [PATCH 4/4] Add '.' before extension and move cursor at 0 in input popup --- rq-cli/src/components/response_panel.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rq-cli/src/components/response_panel.rs b/rq-cli/src/components/response_panel.rs index eaecbee..916d3fd 100644 --- a/rq-cli/src/components/response_panel.rs +++ b/rq-cli/src/components/response_panel.rs @@ -191,7 +191,9 @@ impl BlockComponent for ResponsePanel { KeyCode::Enter => { self.save_option = *menu.selected(); self.save_menu = None; - self.input_popup = Some(Popup::new(Input::from(extension))); + self.input_popup = Some(Popup::new( + Input::from(".".to_string() + extension.as_str()).with_cursor(0), + )); return Ok(HandleSuccess::Consumed); }