From bef1c3d8490dc759c14ef9ee1ba404785eb7e079 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:18:49 -1000 Subject: [PATCH 01/20] perf(bbc_parser): don't block with receive when starting bbcode parsers handle_info will receive asynchronously and print when parsers have started --- lib/epochtalk_server/bbc_parser.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 17142f9e..13d3cda7 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -18,6 +18,12 @@ defmodule EpochtalkServer.BBCParser do @impl true def init(:ok), do: {:ok, load()} + @impl true + def handle_info({_pid, :data, :out, data}, state) do + Logger.debug("#{__MODULE__}(info): #{inspect(data)}") + {:noreply, state} + end + @impl true def handle_call({:parse, ""}, _from, {proc, pid}), do: {:reply, {:ok, ""}, {proc, pid}} @@ -88,9 +94,6 @@ defmodule EpochtalkServer.BBCParser do Proc.send_input(proc, "require 'parsing.php';\n") Logger.debug("#{__MODULE__}(LOAD): #{inspect(pid)}") # clear initial php interactive shell message - receive do - {^pid, :data, :out, data} -> Logger.debug("#{__MODULE__}: #{inspect(data)}") - end {proc, pid} end From e4e269f37b89d366ead2bf08ffc6e0979e76866e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:24:02 -1000 Subject: [PATCH 02/20] refactor(bbc_parser): split out parse with proc functionality can be reused later by batch processing functions --- lib/epochtalk_server/bbc_parser.ex | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 13d3cda7..6e1c7b17 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -29,17 +29,29 @@ defmodule EpochtalkServer.BBCParser do do: {:reply, {:ok, ""}, {proc, pid}} def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do - Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") + Logger.debug("#{__MODULE__}(start parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + parsed = parse_with_proc(bbcode_data, {proc, pid}) + Logger.debug("#{__MODULE__}(finish parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + {:reply, parsed, {proc, pid}} + end - parsed = - receive do - {^pid, :data, :out, data} -> {:ok, data} - after - # time out after not receiving any data - @receive_timeout -> {:timeout, bbcode_data} - end + defp parse_with_proc(nil, {_proc, _pid}), do: {:ok, nil} + defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} + defp parse_with_proc(bbcode_data, {proc, pid}) do + Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") - {:reply, parsed, {proc, pid}} + receive do + {^pid, :data, :out, data} -> + {:ok, data} + after + # time out after not receiving any data + @receive_timeout -> + Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(bbcode_data)}") + bbcode_data = + "

((bbcode parse timeout))


" <> + bbcode_data + {:timeout, bbcode_data} + end end ## === parser api functions ==== From 6ef431a87813e2cb5bdd2044e6fa7d572b165bd7 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:24:47 -1000 Subject: [PATCH 03/20] feat(bbc_parser): use parse_with_proc for parsing list and list tuple --- lib/epochtalk_server/bbc_parser.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 6e1c7b17..86a8ea00 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -35,6 +35,17 @@ defmodule EpochtalkServer.BBCParser do {:reply, parsed, {proc, pid}} end + defp parse_list_tuple_with_proc({left, right}, {proc, pid}) do + left = parse_list_with_proc(left, {proc, pid}) + right = parse_list_with_proc(right, {proc, pid}) + {left, right} + end + + defp parse_list_with_proc(bbcode_data_list, {proc, pid}) do + bbcode_data_list + |> Enum.map(&(parse_with_proc(&1, {proc, pid}))) + end + defp parse_with_proc(nil, {_proc, _pid}), do: {:ok, nil} defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} defp parse_with_proc(bbcode_data, {proc, pid}) do From f27dbfc163f0113cad06c433e8d1e77b5c18eff4 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:26:16 -1000 Subject: [PATCH 04/20] refactor(bbc_parser): remove timeout error handling from parse convenience function moved to parse_with_proc --- lib/epochtalk_server/bbc_parser.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 86a8ea00..3bf08700 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -90,10 +90,7 @@ defmodule EpochtalkServer.BBCParser do # on parse timeout, log and return unparsed data {:timeout, unparsed} -> - Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(unparsed)}") - - "

((bbcode parse timeout))


" <> - unparsed + unparsed end catch e, r -> From 81b1bf7251a57f7ca198830db43c6d90ec43e201 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:29:13 -1000 Subject: [PATCH 05/20] feat(post_json): implement format_proxy_posts_for_by_thread extracts bodies and signatures as lists from post will be processed later as a batch --- lib/epochtalk_server_web/json/post_json.ex | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 99611f13..47372606 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -444,6 +444,31 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do |> Map.delete(:role_name) end + defp format_proxy_posts_for_by_thread(posts) do + # extract body/signature lists from posts + {body_list, signature_list} = + posts + |> Enum.reduce({[], []}, fn post, {body_list, signature_list} -> + body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") + + # add space to end if the last character is a backslash (fix for parser) + body_len = String.length(body) + last_char = String.slice(body, (body_len - 1)..body_len) + body = if last_char == "\\", do: body <> " ", else: body + + signature = + if Map.get(post.user, :signature), + do: String.replace(post.user.signature, "'", "\'"), + else: nil + + # return body/signature lists in reverse order + {[body | body_list], [signature | signature_list]} + end) + + # reverse body/signature lists + {body_list, signature_list} = {Enum.reverse(body_list), Enum.reverse(signature_list)} + end + defp format_proxy_post_data_for_by_thread(post) do body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") From 76adcf60a4bd6585390632728c20be1d472c1fa9 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:30:41 -1000 Subject: [PATCH 06/20] feat(bbc_parser): implement call handler for parsing list tuple --- lib/epochtalk_server/bbc_parser.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 3bf08700..2d02ef31 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -35,6 +35,13 @@ defmodule EpochtalkServer.BBCParser do {:reply, parsed, {proc, pid}} end + def handle_call({:parse_list_tuple, {left_list, right_list}}, _from, {proc, pid}) do + Logger.debug("#{__MODULE__}(start parse list tuple): #{NaiveDateTime.utc_now()}") + parsed = parse_list_tuple_with_proc({left_list, right_list}, {proc, pid}) + Logger.debug("#{__MODULE__}(finish parse list tuple): #{NaiveDateTime.utc_now()}") + {:reply, {:ok, parsed}, {proc, pid}} + end + defp parse_list_tuple_with_proc({left, right}, {proc, pid}) do left = parse_list_with_proc(left, {proc, pid}) right = parse_list_with_proc(right, {proc, pid}) From 81469e5ece036e1cb06e9567ef5a66a860dea6d8 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:32:25 -1000 Subject: [PATCH 07/20] feat(bbc_parser): implement list tuple parser batch handler uses a single process to parse all posts/signatures for a thread page --- lib/epochtalk_server/bbc_parser.ex | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 2d02ef31..6f32a1e5 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -82,6 +82,27 @@ defmodule EpochtalkServer.BBCParser do @doc """ Uses poolboy to call parser """ + def parse_list_tuple({left_bbcode_data, right_bbcode_data}) do + :poolboy.transaction( + :bbc_parser, + fn pid -> + try do + Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") + + GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}) + catch + e, r -> + # something went wrong, log the error + Logger.error( + "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" + ) + + {:error, {left_bbcode_data, right_bbcode_data}} + end + end, + @call_timeout + ) + end def parse(bbcode_data) do :poolboy.transaction( :bbc_parser, From c3836c9dcc83ecf668451894a9d755c60b90d0e0 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:34:21 -1000 Subject: [PATCH 08/20] feat(post_json): use BBCParser.parse_list_tuple on body and signature lists batch process bbcode parsing --- lib/epochtalk_server_web/json/post_json.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 47372606..39580609 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -467,6 +467,17 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do # reverse body/signature lists {body_list, signature_list} = {Enum.reverse(body_list), Enum.reverse(signature_list)} + + # parse body/signature lists + {parsed_body_list, parsed_signature_list} = + {body_list, signature_list} + |> EpochtalkServer.BBCParser.parse_list_tuple() + |> case do + {:ok, parsed_tuple} -> parsed_tuple + {:error, unparsed_tuple} -> + Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") + unparsed_tuple + end end defp format_proxy_post_data_for_by_thread(post) do From 07124211ddd1738bf5fea2d7cc5e1a5e42ebae17 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:36:09 -1000 Subject: [PATCH 09/20] feat(post_json): zip parsed body/signature data back into posts --- lib/epochtalk_server_web/json/post_json.ex | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 39580609..a5eb545b 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -478,6 +478,29 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") unparsed_tuple end + + # zip posts with body/signature lists + Enum.zip_with( + [posts, parsed_body_list, parsed_signature_list], + fn [post, parsed_body, parsed_signature] -> + parsed_body = case parsed_body do + {:ok, parsed_body} -> + parsed_body + {:timeout, unparsed_body} -> + unparsed_body + end + parsed_signature = case parsed_signature do + {:ok, parsed_signature} -> + parsed_signature + {:timeout, unparsed_signature} -> + unparsed_signature + end + user = post.user |> Map.put(:signature, parsed_signature) + post + |> Map.put(:body_html, parsed_body) + |> Map.put(:user, user) + end + ) end defp format_proxy_post_data_for_by_thread(post) do From 43c01028e307ab4a9989a36f6ea56edbc1a7d075 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:36:51 -1000 Subject: [PATCH 10/20] feat(post_json): add logging to batch processing/zip --- lib/epochtalk_server_web/json/post_json.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index a5eb545b..f82fe4d9 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -2,6 +2,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do alias EpochtalkServerWeb.Controllers.BoardJSON alias EpochtalkServerWeb.Controllers.ThreadJSON alias EpochtalkServerWeb.Helpers.ACL + require Logger @moduledoc """ Renders and formats `Post` data, in JSON format for frontend @@ -485,14 +486,18 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do fn [post, parsed_body, parsed_signature] -> parsed_body = case parsed_body do {:ok, parsed_body} -> + Logger.debug("#{__MODULE__}(body): post_id #{inspect(post.id)}") parsed_body {:timeout, unparsed_body} -> + Logger.error("#{__MODULE__}(body timeout): post_id #{inspect(post.id)}") unparsed_body end parsed_signature = case parsed_signature do {:ok, parsed_signature} -> + Logger.debug("#{__MODULE__}(signature): user_id #{inspect(post.user.id)}") parsed_signature {:timeout, unparsed_signature} -> + Logger.error("#{__MODULE__}(signature timeout): user_id #{inspect(post.user.id)}") unparsed_signature end user = post.user |> Map.put(:signature, parsed_signature) From e90317d1535771a7a52d65223b1625322c1f1866 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:37:29 -1000 Subject: [PATCH 11/20] perf(post_json): use batch processor for formatting proxy posts --- lib/epochtalk_server_web/json/post_json.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index f82fe4d9..761011b1 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -141,7 +141,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do # format post data posts = posts - |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) + |> format_proxy_posts_for_by_thread() # build by_thread results %{ @@ -193,7 +193,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do when is_list(posts) do posts = posts - |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) + |> format_proxy_posts_for_by_thread() %{ posts: posts, From 4a5e52fe96b663ad4b0aa4e551b81e746e5e2a11 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:39:01 -1000 Subject: [PATCH 12/20] refactor(post_json): remove unused proxy format function --- lib/epochtalk_server_web/json/post_json.ex | 24 ---------------------- 1 file changed, 24 deletions(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 761011b1..9859212f 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -507,28 +507,4 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do end ) end - - defp format_proxy_post_data_for_by_thread(post) do - body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") - - # add space to end if the last character is a backslash (fix for parser) - body_len = String.length(body) - last_char = String.slice(body, (body_len - 1)..body_len) - body = if last_char == "\\", do: body <> " ", else: body - - parsed_body = EpochtalkServer.BBCParser.parse(body) - - signature = - if Map.get(post.user, :signature), - do: String.replace(post.user.signature, "'", "\'"), - else: nil - - parsed_signature = - if signature, - do: EpochtalkServer.BBCParser.parse(signature), - else: nil - - user = post.user |> Map.put(:signature, parsed_signature) - post |> Map.put(:body_html, parsed_body) |> Map.put(:user, user) - end end From a4991fa258eefec95f9a65c038bfd62d45477c72 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:40:12 -1000 Subject: [PATCH 13/20] style(post_json,bbc_parser): mix format --- lib/epochtalk_server/bbc_parser.ex | 17 ++++++-- lib/epochtalk_server_web/json/post_json.ex | 47 +++++++++++++--------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 6f32a1e5..c42e5bf7 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -29,9 +29,16 @@ defmodule EpochtalkServer.BBCParser do do: {:reply, {:ok, ""}, {proc, pid}} def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do - Logger.debug("#{__MODULE__}(start parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + Logger.debug( + "#{__MODULE__}(start parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}" + ) + parsed = parse_with_proc(bbcode_data, {proc, pid}) - Logger.debug("#{__MODULE__}(finish parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + + Logger.debug( + "#{__MODULE__}(finish parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}" + ) + {:reply, parsed, {proc, pid}} end @@ -50,11 +57,12 @@ defmodule EpochtalkServer.BBCParser do defp parse_list_with_proc(bbcode_data_list, {proc, pid}) do bbcode_data_list - |> Enum.map(&(parse_with_proc(&1, {proc, pid}))) + |> Enum.map(&parse_with_proc(&1, {proc, pid})) end defp parse_with_proc(nil, {_proc, _pid}), do: {:ok, nil} defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} + defp parse_with_proc(bbcode_data, {proc, pid}) do Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") @@ -65,9 +73,11 @@ defmodule EpochtalkServer.BBCParser do # time out after not receiving any data @receive_timeout -> Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(bbcode_data)}") + bbcode_data = "

((bbcode parse timeout))


" <> bbcode_data + {:timeout, bbcode_data} end end @@ -103,6 +113,7 @@ defmodule EpochtalkServer.BBCParser do @call_timeout ) end + def parse(bbcode_data) do :poolboy.transaction( :bbc_parser, diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 9859212f..6c66f6bf 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -474,7 +474,9 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do {body_list, signature_list} |> EpochtalkServer.BBCParser.parse_list_tuple() |> case do - {:ok, parsed_tuple} -> parsed_tuple + {:ok, parsed_tuple} -> + parsed_tuple + {:error, unparsed_tuple} -> Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") unparsed_tuple @@ -484,26 +486,33 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do Enum.zip_with( [posts, parsed_body_list, parsed_signature_list], fn [post, parsed_body, parsed_signature] -> - parsed_body = case parsed_body do - {:ok, parsed_body} -> - Logger.debug("#{__MODULE__}(body): post_id #{inspect(post.id)}") - parsed_body - {:timeout, unparsed_body} -> - Logger.error("#{__MODULE__}(body timeout): post_id #{inspect(post.id)}") - unparsed_body - end - parsed_signature = case parsed_signature do - {:ok, parsed_signature} -> - Logger.debug("#{__MODULE__}(signature): user_id #{inspect(post.user.id)}") - parsed_signature - {:timeout, unparsed_signature} -> - Logger.error("#{__MODULE__}(signature timeout): user_id #{inspect(post.user.id)}") - unparsed_signature - end + parsed_body = + case parsed_body do + {:ok, parsed_body} -> + Logger.debug("#{__MODULE__}(body): post_id #{inspect(post.id)}") + parsed_body + + {:timeout, unparsed_body} -> + Logger.error("#{__MODULE__}(body timeout): post_id #{inspect(post.id)}") + unparsed_body + end + + parsed_signature = + case parsed_signature do + {:ok, parsed_signature} -> + Logger.debug("#{__MODULE__}(signature): user_id #{inspect(post.user.id)}") + parsed_signature + + {:timeout, unparsed_signature} -> + Logger.error("#{__MODULE__}(signature timeout): user_id #{inspect(post.user.id)}") + unparsed_signature + end + user = post.user |> Map.put(:signature, parsed_signature) + post - |> Map.put(:body_html, parsed_body) - |> Map.put(:user, user) + |> Map.put(:body_html, parsed_body) + |> Map.put(:user, user) end ) end From 885396a7c6c2ff3267c818ba6dbc7c3f95322484 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:59:13 -1000 Subject: [PATCH 14/20] refactor(bbc_parser): update timeouts split genserver and poolboy timeouts reduce receive timeout to ensure quick loading --- lib/epochtalk_server/bbc_parser.ex | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index c42e5bf7..8b300fbd 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -3,11 +3,15 @@ defmodule EpochtalkServer.BBCParser do require Logger alias Porcelain.Process, as: Proc - # poolboy genserver call timeout (ms) - # should be greater than internal porcelain php call - @call_timeout 500 + # genserver call timeouts (ms) + @genserver_parse_timeout 5000 + @genserver_parse_tuple_timeout 5000 + + # poolboy timeout (ms) + @poolboy_transaction_timeout 200 + # porcelain php parser call timeout (ms) - @receive_timeout 400 + @receive_timeout 20 @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser @@ -99,7 +103,7 @@ defmodule EpochtalkServer.BBCParser do try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}) + GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}, @genserver_parse_tuple_timeout) catch e, r -> # something went wrong, log the error @@ -110,7 +114,7 @@ defmodule EpochtalkServer.BBCParser do {:error, {left_bbcode_data, right_bbcode_data}} end end, - @call_timeout + @poolboy_transaction_timeout ) end @@ -121,7 +125,7 @@ defmodule EpochtalkServer.BBCParser do try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse, bbcode_data}, @call_timeout) + GenServer.call(pid, {:parse, bbcode_data}, @genserver_parse_timeout) |> case do # on success, return parsed data {:ok, parsed} -> @@ -141,7 +145,7 @@ defmodule EpochtalkServer.BBCParser do bbcode_data end end, - @call_timeout + @poolboy_transaction_timeout ) end From 80a30e5d3a75b1d41908eef484f05163569a6173 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 13:20:16 -1000 Subject: [PATCH 15/20] fix(bbc_parser): add :timeout atom to genserver tuple timeout return --- lib/epochtalk_server/bbc_parser.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 8b300fbd..1e21d219 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -111,6 +111,8 @@ defmodule EpochtalkServer.BBCParser do "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" ) + left_bbcode_data = left_bbcode_data |> Enum.map(&({:timeout, &1})) + right_bbcode_data = right_bbcode_data |> Enum.map(&({:timeout, &1})) {:error, {left_bbcode_data, right_bbcode_data}} end end, From 3b8b579f8df5a9cb380bdd53eab21700ead32db4 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 13:21:10 -1000 Subject: [PATCH 16/20] perf(bbc_parser): reduce genserver tuple call timeout don't allow call to go over half a second --- lib/epochtalk_server/bbc_parser.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 1e21d219..ec2e01e0 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -5,7 +5,7 @@ defmodule EpochtalkServer.BBCParser do # genserver call timeouts (ms) @genserver_parse_timeout 5000 - @genserver_parse_tuple_timeout 5000 + @genserver_parse_tuple_timeout 500 # poolboy timeout (ms) @poolboy_transaction_timeout 200 From c921a2683e290b9eb3643499707665b6388d1d47 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 13:21:42 -1000 Subject: [PATCH 17/20] refactor(post_json): log unparsed tuple instead of post on error --- lib/epochtalk_server_web/json/post_json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 6c66f6bf..c56edbea 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -478,7 +478,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do parsed_tuple {:error, unparsed_tuple} -> - Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") + Logger.error("#{__MODULE__}(tuple parse): #{inspect(unparsed_tuple)}") unparsed_tuple end From cf4bd4991a7ee451a77d92345addd0590d90fee9 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 23:03:20 -1000 Subject: [PATCH 18/20] style(bbc_parser): mix format --- lib/epochtalk_server/bbc_parser.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index ec2e01e0..ab607196 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -103,7 +103,11 @@ defmodule EpochtalkServer.BBCParser do try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}, @genserver_parse_tuple_timeout) + GenServer.call( + pid, + {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}, + @genserver_parse_tuple_timeout + ) catch e, r -> # something went wrong, log the error @@ -111,8 +115,8 @@ defmodule EpochtalkServer.BBCParser do "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" ) - left_bbcode_data = left_bbcode_data |> Enum.map(&({:timeout, &1})) - right_bbcode_data = right_bbcode_data |> Enum.map(&({:timeout, &1})) + left_bbcode_data = left_bbcode_data |> Enum.map(&{:timeout, &1}) + right_bbcode_data = right_bbcode_data |> Enum.map(&{:timeout, &1}) {:error, {left_bbcode_data, right_bbcode_data}} end end, From cfefcd6ed17bfa205e0d04a64d5e510e8e56f0f3 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 17 Dec 2024 10:35:27 -1000 Subject: [PATCH 19/20] refactor(post_json): extract zip posts function fixes credo complexity issue --- lib/epochtalk_server_web/json/post_json.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index c56edbea..e5545e3f 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -482,6 +482,9 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do unparsed_tuple end + zip_posts(posts, parsed_body_list, parsed_signature_list) + end + defp zip_posts(posts, parsed_body_list, parsed_signature_list) do # zip posts with body/signature lists Enum.zip_with( [posts, parsed_body_list, parsed_signature_list], From 857f2755c571423896c72e97c351b54caf1675ae Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 17 Dec 2024 10:36:24 -1000 Subject: [PATCH 20/20] style(post_json): mix format --- lib/epochtalk_server_web/json/post_json.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index e5545e3f..1a845943 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -484,6 +484,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do zip_posts(posts, parsed_body_list, parsed_signature_list) end + defp zip_posts(posts, parsed_body_list, parsed_signature_list) do # zip posts with body/signature lists Enum.zip_with(