Merge pull request #88 from epochtalk/md-parser

Md parser

lib/epochtalk_server/models/post.ex

@@ -176,7 +176,11 @@ defmodule EpochtalkServer.Models.Post do
       create(%{
         "thread_id" => post_attrs["thread_id"],
         "user_id" => user_id,
-        "content" => %{title: post_attrs["title"], body: post_attrs["body"]},
+        "content" => %{
+          title: post_attrs["title"],
+          body: post_attrs["body"],
+          body_html: post_attrs["body_html"]
+        },
         "deleted" => post_attrs["deleted"],
         "locked" => post_attrs["locked"]
       })
@@ -248,7 +252,7 @@ defmodule EpochtalkServer.Models.Post do
         """
           SELECT
             p.thread_id, t.board_id, t.slug, b.right_to_left, p.user_id, p.content ->> \'title\' as title,
-            p.content ->> \'body\' as body, p.metadata, p.deleted, p.locked, p.created_at,
+            p.content ->> \'body\' as body, p.content ->> \'body_html\' as body_html, p.metadata, p.deleted, p.locked, p.created_at,
             p.updated_at, p.imported_at, CASE WHEN EXISTS (
               SELECT rp.id
               FROM administration.reports_posts rp
@@ -314,6 +318,7 @@ defmodule EpochtalkServer.Models.Post do
       user_id: p1.user_id,
       title: p1.title,
       body: p1.body,
+      body_html: p1.body_html,
       deleted: p1.deleted,
       locked: p1.locked,
       right_to_left: p1.right_to_left,

lib/epochtalk_server/models/thread.ex

@@ -703,7 +703,11 @@ defmodule EpochtalkServer.Models.Thread do
     post_attrs = %{
       thread_id: thread_id,
       user_id: user.id,
-      content: %{title: thread_attrs["title"], body: thread_attrs["body"]}
+      content: %{
+        title: thread_attrs["title"],
+        body: thread_attrs["body"],
+        body_html: thread_attrs["body_html"]
+      }
     }
 
     case Post.create(post_attrs) do

lib/epochtalk_server_web/controllers/post.ex

@@ -9,6 +9,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do
   alias EpochtalkServerWeb.Helpers.Validate
   alias EpochtalkServerWeb.Helpers.ACL
   alias EpochtalkServerWeb.Helpers.Sanitize
+  alias EpochtalkServerWeb.Helpers.Parse
   alias EpochtalkServer.Models.Post
   alias EpochtalkServer.Models.Poll
   alias EpochtalkServer.Models.Thread
@@ -47,9 +48,9 @@ defmodule EpochtalkServerWeb.Controllers.Post do
          post_max_length <-
            Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"],
          thread_id <- Validate.cast(attrs, "thread_id", :integer, required: true),
-         title <-
+         _title <-
            Validate.cast(attrs, "title", :string, required: true, max: @max_post_title_length),
-         body <- Validate.cast(attrs, "body", :string, required: true, max: post_max_length),
+         _body <- Validate.cast(attrs, "body", :string, required: true, max: post_max_length),
          user_priority <- ACL.get_user_priority(conn),
          {:bypass_lock, true} <-
            {:bypass_lock, can_authed_user_bypass_thread_lock(user, thread_id)},
@@ -63,17 +64,18 @@ defmodule EpochtalkServerWeb.Controllers.Post do
            {:board_banned, BoardBan.is_banned_from_board(user, thread_id: thread_id)},
          attrs <- AutoModeration.moderate(user, attrs),
          attrs <- Mention.username_to_user_id(user, attrs),
-         attrs <- Sanitize.html_from_title(title, attrs),
-         attrs <- Sanitize.html_from_body(body, attrs),
+         attrs <- Sanitize.html_and_entities_from_title(attrs),
+         attrs <- Sanitize.html_and_entities_from_body(attrs),
+         attrs <- Parse.markdown_within_body(attrs),
          # TODO(akinsey): Implement the following for completion
          # Plugins
          # 1) Track IP (done)
 
          # Pre Processing
 
-         # 1) clean post title (html_sanitize_ex) (done)
-         # 2) parse/clean post body
-         # 3) handle uploaded images
+         # 1) clean post title (done)
+         # 2) parse/clean post body (wip)
+         # 3) handle uploaded images (wip)
          # 4) handle filtering out newbie images
 
          # Hooks

lib/epochtalk_server_web/controllers/thread.ex

@@ -9,6 +9,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do
   alias EpochtalkServerWeb.Helpers.Validate
   alias EpochtalkServerWeb.Helpers.ACL
   alias EpochtalkServerWeb.Helpers.Sanitize
+  alias EpochtalkServerWeb.Helpers.Parse
   alias EpochtalkServer.Models.Thread
   alias EpochtalkServer.Models.User
   alias EpochtalkServer.Models.Board
@@ -68,9 +69,9 @@ defmodule EpochtalkServerWeb.Controllers.Thread do
          # pre/parallel hooks
          attrs <- AutoModeration.moderate(user, attrs),
          attrs <- Mention.username_to_user_id(user, attrs),
-         attrs <- Sanitize.html_from_title(attrs["title"], attrs),
-         attrs <- Sanitize.html_from_body(attrs["body"], attrs),
-
+         attrs <- Sanitize.html_and_entities_from_title(attrs),
+         attrs <- Sanitize.html_and_entities_from_body(attrs),
+         attrs <- Parse.markdown_within_body(attrs),
          # thread creation
          {:ok, thread_data} <- Thread.create(attrs, user) do
       # post hooks
@@ -105,7 +106,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do
 
       # Pre Processing
 
-      # 1) clean post (html_sanitize_ex)
+      # 1) clean post
       # 2) parse post
       # 3) handle uploaded images
       # 4) handle filtering out newbie images

lib/epochtalk_server_web/helpers/parse.ex

@@ -0,0 +1,12 @@
+defmodule EpochtalkServerWeb.Helpers.Parse do
+  @moduledoc """
+  Helper for parsing `User` input from the front end client
+  """
+
+  @doc """
+  Used to parse markdown within `Thread` or `Post` body, assumes
+  """
+  @spec markdown_within_body(attrs :: map) :: map()
+  def markdown_within_body(attrs) when is_map(attrs),
+    do: Map.put(attrs, "body_html", Earmark.as_html!(attrs["body_html"] || attrs["body"]))
+end

lib/epochtalk_server_web/helpers/sanitize.ex

@@ -4,40 +4,60 @@ defmodule EpochtalkServerWeb.Helpers.Sanitize do
   """
 
   @doc """
-  Used to sanitize html from `Thread` or `Post` title
+  Used to sanitize html and entities from `Thread` or `Post` title
 
   ## Example
       iex> alias EpochtalkServerWeb.Helpers.Sanitize
-      iex> attrs = %{"title" => "<strong>Hello World</strong><br /><script></script><a href='google.com'></a>"}
-      iex> Sanitize.html_from_title(attrs["title"], attrs)
-      %{"title" => "Hello World"}
+      iex> attrs = %{"title" => "&nbsp;<strong>Hello World</strong><br /><script></script><a href='google.com'></a>"}
+      iex> Sanitize.html_and_entities_from_title(attrs)
+      %{"title" => "&#38;nbsp;&lt;strong&gt;Hello World&lt;/strong&gt;&lt;br /&gt;&lt;script&gt;&lt;/script&gt;&lt;a href='google.com'&gt;&lt;/a&gt;"}
   """
-  @spec html_from_title(title :: String.t(), attrs :: map) :: map()
-  def html_from_title(title, attrs) when is_binary(title) and is_map(attrs),
-    do: Map.put(attrs, "title", HtmlSanitizeEx.strip_tags(title))
+  @spec html_and_entities_from_title(attrs :: map) :: map()
+  def html_and_entities_from_title(attrs) when is_map(attrs) do
+    sanitized_title =
+      attrs["title"]
+      |> String.replace(~r/(?:&)/, "&#38;")
+      |> String.replace(~r/(?:<)/, "&lt;")
+      |> String.replace(~r/(?:>)/, "&gt;")
+
+    Map.put(attrs, "title", sanitized_title)
+  end
 
   @doc """
-  Used to sanitize html from `Message` subject
+  Used to sanitize html and entities from `Message` subject
 
   ## Example
       iex> alias EpochtalkServerWeb.Helpers.Sanitize
-      iex> attrs = %{"subject" => "<strong>Hey this is</strong><br /> <script>a</script> <a href='google.com'>message</a>"}
-      iex> Sanitize.html_from_subject(attrs["subject"], attrs)
-      %{"subject" => "Hey this is a message"}
+      iex> attrs = %{"subject" => "&nbsp;<strong>Hello World</strong><br /><script></script><a href='google.com'></a>"}
+      iex> Sanitize.html_and_entities_from_subject(attrs)
+      %{"subject" => "&#38;nbsp;&lt;strong&gt;Hello World&lt;/strong&gt;&lt;br /&gt;&lt;script&gt;&lt;/script&gt;&lt;a href='google.com'&gt;&lt;/a&gt;"}
   """
-  @spec html_from_subject(subject :: String.t(), attrs :: map) :: map()
-  def html_from_subject(subject, attrs) when is_binary(subject) and is_map(attrs),
-    do: Map.put(attrs, "subject", HtmlSanitizeEx.strip_tags(subject))
+  @spec html_and_entities_from_subject(attrs :: map) :: map()
+  def html_and_entities_from_subject(attrs) when is_map(attrs) do
+    sanitized_subject =
+      attrs["subject"]
+      |> String.replace(~r/(?:&)/, "&#38;")
+      |> String.replace(~r/(?:<)/, "&lt;")
+      |> String.replace(~r/(?:>)/, "&gt;")
+
+    Map.put(attrs, "subject", sanitized_subject)
+  end
 
   @doc """
-  Used to sanitize all html except basic formatting html from `Thread` or `Post` body
+  Used to sanitize html and entities from `Thread` or `Post` body, store sanitized body in `body_html`
     ## Example
       iex> alias EpochtalkServerWeb.Helpers.Sanitize
-      iex> attrs = %{"body" => "<i>Hey <b>this</b> is</i><br /> <h1><script>a</script></h1> <a href='google.com'>post</a>"}
-      iex> Sanitize.html_from_body(attrs["body"], attrs)
-      %{"body" => "<i>Hey <b>this</b> is</i><br /> <h1>a</h1> <a href=\\"google.com\\">post</a>"}
+      iex> attrs = %{"body" => "<i>Hey <b>this</b> is</i><br /> <h1><script>a</script></h1> <a href='google.com'>post</a> &nbsp;"}
+      iex> Sanitize.html_and_entities_from_body(attrs)
+      %{"body" => "<i>Hey <b>this</b> is</i><br /> <h1><script>a</script></h1> <a href='google.com'>post</a> &nbsp;", "body_html" => "&lt;i>Hey &lt;b>this&lt;/b> is&lt;/i>&lt;br /> &lt;h1>&lt;script>a&lt;/script>&lt;/h1> &lt;a href='google.com'>post&lt;/a> &#38;nbsp;"}
   """
-  @spec html_from_body(body :: String.t(), attrs :: map) :: map()
-  def html_from_body(body, attrs) when is_binary(body) and is_map(attrs),
-    do: Map.put(attrs, "body", HtmlSanitizeEx.basic_html(body))
+  @spec html_and_entities_from_body(attrs :: map) :: map()
+  def html_and_entities_from_body(attrs) when is_map(attrs) do
+    sanitized_body =
+      attrs["body"]
+      |> String.replace(~r/(?:&)/, "&#38;")
+      |> String.replace(~r/(?:<)/, "&lt;")
+
+    Map.put(attrs, "body_html", sanitized_body)
+  end
 end

lib/epochtalk_server_web/json/post_json.ex

@@ -265,8 +265,8 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do
 
   defp format_post_data_for_by_thread(post) do
     post
-    # TODO(akinsey): this is a temp hack to get posts to display
-    |> Map.put(:body_html, post.body)
+    # if body_html does not exist, default to post.body
+    |> Map.put(:body_html, post.body_html || post.body)
     |> Map.put(:user, %{
       id: post.user_id,
       name: post.name,

mix.exs

@@ -37,6 +37,7 @@ defmodule EpochtalkServer.MixProject do
       {:corsica, "~> 1.3.0"},
       {:credo, "~> 1.6", only: [:dev, :test], runtime: false},
       {:dialyxir, "~> 1.2", only: [:dev], runtime: false},
+      {:earmark, "~> 1.4"},
       {:ecto_sql, "~> 3.6"},
       {:ex_doc, "~> 0.29.4"},
       {:ex_machina, "~> 2.7.0", only: :test},
@@ -46,7 +47,6 @@ defmodule EpochtalkServer.MixProject do
       {:guardian_phoenix, "~> 2.0"},
       {:guardian_db, "~> 2.1"},
       {:guardian_redis, "~> 0.1"},
-      {:html_sanitize_ex, "~> 1.4"},
       {:iteraptor, git: "https://github.com/epochtalk/elixir-iteraptor.git", tag: "1.13.1"},
       {:jason, "~> 1.4.0"},
       {:mimic, "~> 1.7.4", only: :test},