From 91f2d850d72ddd4b33fa57381634054e1080cbae Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 21 Dec 2023 13:24:48 -0800 Subject: [PATCH 001/231] feat(proxy): Setting up proxy repo, supervisor and conversion code; Added sequences to setup DB with IDs after the proxy DB IDs --- .../conversion/proxy_supervisor.ex | 17 + lib/epochtalk_server/smf_repo.ex | 5 + .../helpers/proxy_conversion.ex | 318 ++++++++++++++++++ mix.exs | 1 + mix.lock | 1 + .../20231110161916_add_sequences.exs | 10 + 6 files changed, 352 insertions(+) create mode 100644 lib/epochtalk_server/conversion/proxy_supervisor.ex create mode 100644 lib/epochtalk_server/smf_repo.ex create mode 100644 lib/epochtalk_server_web/helpers/proxy_conversion.ex create mode 100644 priv/repo/migrations/20231110161916_add_sequences.exs diff --git a/lib/epochtalk_server/conversion/proxy_supervisor.ex b/lib/epochtalk_server/conversion/proxy_supervisor.ex new file mode 100644 index 00000000..a3cca086 --- /dev/null +++ b/lib/epochtalk_server/conversion/proxy_supervisor.ex @@ -0,0 +1,17 @@ +defmodule EpochtalkServer.ProxySupervisor do + use Supervisor + + def start_link(arg) do + Supervisor.start_link(__MODULE__, arg, name: __MODULE__) + end + + @impl true + def init(_arg) do + children = [ + EpochtalkServer.SmfRepo + # Add reporters as children of your supervision tree. + ] + + Supervisor.init(children, strategy: :one_for_one) + end +end diff --git a/lib/epochtalk_server/smf_repo.ex b/lib/epochtalk_server/smf_repo.ex new file mode 100644 index 00000000..3d3bbb47 --- /dev/null +++ b/lib/epochtalk_server/smf_repo.ex @@ -0,0 +1,5 @@ +defmodule EpochtalkServer.SmfRepo do + use Ecto.Repo, + otp_app: :epochtalk_server, + adapter: Ecto.Adapters.MyXQL +end diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex new file mode 100644 index 00000000..e9228613 --- /dev/null +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -0,0 +1,318 @@ +defmodule EpochtalkServerWeb.Helpers.ProxyConversion do + use GenServer + import Ecto.Query + alias EpochtalkServer.SmfRepo + alias EpochtalkServer.ProxySupervisor + + def start_link(arg) do + GenServer.start_link(__MODULE__, arg, name: __MODULE__) + end + + @impl true + def init(opts) do + {:ok, opts} + end + + def build_model(model_type, ids) when is_nil(model_type) or is_nil(ids) do + {:ok, %{}} + end + + def build_model(_, ids) when length(ids) > 10 do + {:error, "Limit too large, please try again"} + end + + def build_model(model_type, id) when is_integer(id) do + ProxySupervisor.start_link([]) + + case model_type do + "threads.by_board" -> + build_thread_model_by_board(id) + + _ -> + build_model(model_type, [id]) + end + end + + def build_model(model_type, ids) do + ProxySupervisor.start_link([]) + + case model_type do + "category" -> + build_category_model(ids) + + "board" -> + build_board_model(ids) + + "thread" -> + build_thread_model(ids) + + "post" -> + build_post_model(ids) + + _ -> + build_model(nil, nil) + end + end + + def build_category_model(ids) do + from(c in "smf_categories", + limit: ^length(ids), + where: c.id_cat in ^ids, + select: %{ + id: c.id_cat, + name: c.name, + view_order: c.catOrder + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Categories not found for ids: #{ids}"} + + categories -> + return_tuple(categories) + end + end + + def build_board_model(ids) do + from(b in "smf_boards", + limit: ^length(ids), + where: b.id_board in ^ids, + select: %{ + id: b.id_board, + cat_id: b.id_cat, + name: b.name, + description: b.description, + thread_count: b.numTopics, + post_count: b.numPosts + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Boards not found for ids: #{ids}"} + + boards -> + boards = + Enum.map(boards, fn board -> + board = + board + |> Map.put(:slug, String.downcase(String.replace(board.name, ~r/\s+/, "_"))) + |> Map.put(:children, []) + + board = if board.cat_id, do: build_category_and_board_mapping(board) + board + end) + + return_tuple(boards) + end + end + + def build_thread_model_by_board(id) do + from(t in "smf_topics", + limit: 10, + where: t.id_board == ^id, + order_by: [desc: t.id_last_msg], + select: %{ + id: t.id_topic, + board_id: t.id_board, + sticky: t.isSticky, + locked: t.locked, + view_count: t.numViews, + first_post_id: t.id_first_msg, + last_post_id: t.id_last_msg, + started_user_id: t.id_member_started, + updated_user_id: t.id_member_updated, + moderated: t.selfModerated, + post_count: t.numReplies + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Threads not found for board_id: #{id}"} + + threads -> + threads = + Enum.reduce(threads, [], fn thread, acc -> + first_post = get_post(thread.first_post_id) + last_post = get_post(thread.last_post_id) + + thread = + thread + |> Map.put(:title, first_post.title) + |> Map.put(:slug, first_post.title) + |> Map.put(:user_id, first_post.user_id) + |> Map.put(:username, first_post.username) + |> Map.put(:user_deleted, false) + |> Map.put(:last_post_id, last_post.id) + |> Map.put(:last_post_created_at, last_post.created_at * 1000) + |> Map.put(:last_post_deleted, false) + |> Map.put(:last_post_user_id, last_post.user_id) + |> Map.put(:last_post_username, last_post.username) + |> Map.put(:last_post_user_deleted, false) + |> Map.put(:last_post_avatar, last_post.avatar) + |> Map.put(:last_viewed, nil) + |> Map.put(:created_at, first_post.created_at * 1000) + + acc = [thread | acc] + acc + end) + + return_tuple(Enum.reverse(threads)) + end + end + + def build_thread_model(ids) do + from(t in "smf_topics", + limit: ^length(ids), + where: t.id_topic in ^ids, + select: %{ + id: t.id_topic, + board_id: t.id_board, + sticky: t.isSticky, + locked: t.locked, + metadata_views: t.numViews + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Threads not found for ids: #{ids}"} + + threads -> + threads = + Enum.map(threads, fn thread -> + thread = if thread.board_id, do: build_thread_and_board_mapping(thread) + thread + end) + + return_tuple(threads) + end + end + + def build_post_model(ids) do + from(m in "smf_messages", + limit: ^length(ids), + where: m.id_msg in ^ids, + select: %{ + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + user_id: m.id_member, + title: m.subject, + body: m.body + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Posts not found for ids: #{ids}"} + + posts -> + posts = + Enum.map(posts, fn post -> + post = if post.thread_id, do: build_post_and_thread_mapping(post) + post + end) + + return_tuple(posts) + end + end + + defp build_category_and_board_mapping(board) do + {:ok, category} = build_category_model([board.cat_id]) + + board = + board + |> Map.put(:category, category) + |> Map.put(:board_mapping, build_board_mapping(category, board)) + |> Map.delete(:cat_id) + + board + end + + def build_thread_and_board_mapping(thread) do + {:ok, board} = build_board_model([thread.board_id]) + + thread = + thread + |> Map.put(:board, board) + |> Map.delete(:board_id) + + thread + end + + defp build_post_and_thread_mapping(post) do + {:ok, thread} = build_thread_model([post.thread_id]) + + content = %{ + "title" => post.title, + "body" => post.body + } + + post = + post + |> Map.put(:thread, thread) + |> Map.delete(:thread_id) + |> Map.put(:content, content) + |> Map.delete(:title) + |> Map.delete(:body) + + post + end + + defp build_board_mapping(category, board) do + [ + %{ + id: category.id, + name: category.name, + type: "category", + view_order: category.view_order + }, + %{ + id: board.id, + name: board.name, + type: "board", + category_id: category.id, + view_order: category.view_order + 1 + } + ] + end + + defp return_tuple(object) do + if length(object) > 1 do + {:ok, object} + else + {:ok, List.first(object)} + end + end + + defp get_post(id) do + from(m in "smf_messages", + limit: 1, + where: m.id_msg == ^id, + select: %{ + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + user_id: m.id_member, + title: m.subject, + body: m.body, + username: m.posterName, + user_email: m.posterEmail, + created_at: m.posterTime, + modified_time: m.modifiedTime, + avatar: m.icon + } + ) + |> SmfRepo.one() + |> case do + nil -> + {:error, "Posts not found for id: #{id}"} + + post -> + post + end + end +end diff --git a/mix.exs b/mix.exs index 8df02883..405a7330 100644 --- a/mix.exs +++ b/mix.exs @@ -50,6 +50,7 @@ defmodule EpochtalkServer.MixProject do {:iteraptor, git: "https://github.com/epochtalk/elixir-iteraptor.git", tag: "1.13.1"}, {:jason, "~> 1.4.0"}, {:mimic, "~> 1.7.4", only: :test}, + {:myxql, "~> 0.6.3"}, {:phoenix, "~> 1.7.2"}, {:phoenix_ecto, "~> 4.4"}, {:phoenix_html, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 597556bc..18221461 100644 --- a/mix.lock +++ b/mix.lock @@ -37,6 +37,7 @@ "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimic": {:hex, :mimic, "1.7.4", "cd2772ffbc9edefe964bc668bfd4059487fa639a5b7f1cbdf4fd22946505aa4f", [:mix], [], "hexpm", "437c61041ecf8a7fae35763ce89859e4973bb0666e6ce76d75efc789204447c3"}, "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, + "myxql": {:hex, :myxql, "0.6.3", "3d77683a09f1227abb8b73d66b275262235c5cae68182f0cfa5897d72a03700e", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:geo, "~> 3.4", [hex: :geo, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "af9eb517ddaced5c5c28e8749015493757fd4413f2cfccea449c466d405d9f51"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, diff --git a/priv/repo/migrations/20231110161916_add_sequences.exs b/priv/repo/migrations/20231110161916_add_sequences.exs new file mode 100644 index 00000000..655d260f --- /dev/null +++ b/priv/repo/migrations/20231110161916_add_sequences.exs @@ -0,0 +1,10 @@ +defmodule EpochtalkServer.Repo.Migrations.AddSequences do + use Ecto.Migration + + def change do + execute "ALTER SEQUENCE categories_id_seq RESTART WITH 50;" + execute "ALTER SEQUENCE boards_id_seq RESTART WITH 500;" + execute "ALTER SEQUENCE threads_id_seq RESTART WITH 2000000;" + execute "ALTER SEQUENCE posts_id_seq RESTART WITH 60000000;" + end +end From a6df7b421f082fc0e996f15284ed3546a8fcbb30 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 21 Dec 2023 13:35:04 -0800 Subject: [PATCH 002/231] feat(proxy): Use proxy to handle by_board route in threads controller --- .../controllers/thread.ex | 41 +++++++++++++++++++ lib/epochtalk_server_web/json/thread_json.ex | 29 +++++++++++++ 2 files changed, 70 insertions(+) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index a4658f85..b10c1185 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -23,6 +23,9 @@ defmodule EpochtalkServerWeb.Controllers.Thread do alias EpochtalkServer.Models.UserActivity alias EpochtalkServer.Models.ThreadSubscription alias EpochtalkServer.Models.Mention + alias EpochtalkServerWeb.Helpers.ProxyConversion + + plug :check_proxy when action in [:by_board] @doc """ Used to retrieve recent threads @@ -341,4 +344,42 @@ defmodule EpochtalkServerWeb.Controllers.Thread do defp update_user_thread_view_count(user, thread_id), do: UserThreadView.upsert(user.id, thread_id) + + defp check_proxy(conn, _) do + conn = + case conn.private.phoenix_action do + :by_board -> + if Validate.cast(conn.params, "board_id", :integer, required: true) < + System.get_env("BOARDS_SEQ") |> String.to_integer() do + conn + |> proxy_by_board(conn.params) + |> halt() + else + conn + end + _ -> + conn + end + + conn + end + + defp proxy_by_board(conn, attrs) do + with board_id <- Validate.cast(attrs, "board_id", :integer, required: true), + page <- Validate.cast(attrs, "page", :integer, default: 1), + limit <- Validate.cast(attrs, "limit", :integer, default: 5), + user <- Guardian.Plug.current_resource(conn), + :ok <- ACL.allow!(conn, "threads.byBoard"), + {:ok, threads} <- ProxyConversion.build_model("threads.by_board", board_id) do + render(conn, :by_board_proxy, %{ + threads: threads, + user: user, + page: page, + limit: limit + }) + else + _ -> + ErrorHelpers.render_json_error(conn, 400, "Error, cannot get threads by board") + end + end end diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 26891063..002ccaec 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -1,5 +1,6 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do alias EpochtalkServerWeb.Controllers.BoardJSON + alias EpochtalkServerWeb.Helpers.ProxyConversion @moduledoc """ Renders and formats `Thread` data, in JSON format for frontend @@ -89,6 +90,34 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do if board_banned, do: Map.put(result, :board_banned, board_banned), else: result end + def by_board_proxy(%{ + threads: threads, + user: user, + page: page, + limit: limit + }) do + # format board data + board = + if is_map(threads) do + ProxyConversion.build_thread_and_board_mapping(threads) + else + ProxyConversion.build_thread_and_board_mapping(List.first(threads)) + end + + # format thread data + user_id = if is_nil(user), do: nil, else: user.id + normal = threads |> Enum.map(&format_thread_data(&1, user_id)) + # sticky = threads.sticky |> Enum.map(&format_thread_data(&1, user_id)) + + # build by_board results + %{ + normal: normal, + board: board.board, + page: page, + limit: limit + } + end + @doc """ Renders `Thread` id for slug to id route. """ From 4f524519899ca8dec7483121038e5212e691cc59 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 21 Dec 2023 13:35:51 -0800 Subject: [PATCH 003/231] feat(proxy): Use proxy to handle slug_to_id route in board controller --- lib/epochtalk_server_web/controllers/board.ex | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index c4901d75..7a3c6065 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -12,6 +12,8 @@ defmodule EpochtalkServerWeb.Controllers.Board do alias EpochtalkServerWeb.Helpers.Validate alias EpochtalkServerWeb.Helpers.ACL + plug :check_proxy when action in [:slug_to_id] + @doc """ Used to retrieve categorized boards """ @@ -105,4 +107,27 @@ defmodule EpochtalkServerWeb.Controllers.Board do ErrorHelpers.render_json_error(conn, 400, "Error, cannot convert board slug to id") end end + + defp check_proxy(conn, _) do + conn = + case conn.private.phoenix_action do + :slug_to_id -> + case Integer.parse(conn.params["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) + + if slug_as_id < System.get_env("BOARDS_SEQ") |> String.to_integer() do + conn + |> render(:slug_to_id, id: slug_as_id) + |> halt() + end + + _ -> + conn + end + _ -> + conn + end + conn + end end From e5722f3c5b7d3b4c3f9f44549336ad3d992315e5 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 21 Dec 2023 13:36:34 -0800 Subject: [PATCH 004/231] refactor(proxy): mix format updates --- lib/epochtalk_server_web/controllers/board.ex | 24 ++++++++++--------- .../controllers/thread.ex | 1 + 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 7a3c6065..f151c074 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -112,22 +112,24 @@ defmodule EpochtalkServerWeb.Controllers.Board do conn = case conn.private.phoenix_action do :slug_to_id -> - case Integer.parse(conn.params["slug"]) do - {_, ""} -> - slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) + case Integer.parse(conn.params["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - if slug_as_id < System.get_env("BOARDS_SEQ") |> String.to_integer() do - conn - |> render(:slug_to_id, id: slug_as_id) - |> halt() - end - - _ -> + if slug_as_id < System.get_env("BOARDS_SEQ") |> String.to_integer() do conn - end + |> render(:slug_to_id, id: slug_as_id) + |> halt() + end + + _ -> + conn + end + _ -> conn end + conn end end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index b10c1185..16042e37 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -357,6 +357,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do else conn end + _ -> conn end From ad9ce9555014b5721b602e69ad484a3894d604df Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 22 Dec 2023 10:45:53 -0800 Subject: [PATCH 005/231] feat(proxy-posts): Use proxy to handle slug_to_id route in thread controller and by_thread route in posts controller to display posts on the frontend --- lib/epochtalk_server_web/controllers/board.ex | 2 + lib/epochtalk_server_web/controllers/post.ex | 42 +++++++++ .../controllers/thread.ex | 23 ++++- .../helpers/proxy_conversion.ex | 94 ++++++++++++++++--- lib/epochtalk_server_web/json/post_json.ex | 60 ++++++++++++ lib/epochtalk_server_web/json/thread_json.ex | 8 +- .../20231110161916_add_sequences.exs | 2 +- 7 files changed, 211 insertions(+), 20 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index f151c074..42872cd2 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -120,6 +120,8 @@ defmodule EpochtalkServerWeb.Controllers.Board do conn |> render(:slug_to_id, id: slug_as_id) |> halt() + else + conn end _ -> diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 9aabe100..e134219b 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -10,6 +10,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do alias EpochtalkServerWeb.Helpers.ACL alias EpochtalkServerWeb.Helpers.Sanitize alias EpochtalkServerWeb.Helpers.Parse + alias EpochtalkServerWeb.Helpers.ProxyConversion alias EpochtalkServer.Models.Post alias EpochtalkServer.Models.Poll alias EpochtalkServer.Models.Thread @@ -30,6 +31,8 @@ defmodule EpochtalkServerWeb.Controllers.Post do @max_post_title_length 255 + plug :check_proxy when action in [:by_thread] + @doc """ Used to create posts """ @@ -266,4 +269,43 @@ defmodule EpochtalkServerWeb.Controllers.Post do has_bypass or thread_not_locked or is_mod end + + defp check_proxy(conn, _) do + conn = + case conn.private.phoenix_action do + :by_thread -> + if Validate.cast(conn.params, "thread_id", :integer, required: true) < + System.get_env("THREADS_SEQ") |> String.to_integer() do + conn + |> proxy_by_thread(conn.params) + |> halt() + else + conn + end + + _ -> + conn + end + + conn + end + + defp proxy_by_thread(conn, attrs) do + with thread_id <- Validate.cast(attrs, "thread_id", :integer, required: true), + page <- Validate.cast(attrs, "page", :integer, default: 1), + limit <- Validate.cast(attrs, "limit", :integer, default: 5), + user <- Guardian.Plug.current_resource(conn), + :ok <- ACL.allow!(conn, "posts.byThread"), + {:ok, posts} <- ProxyConversion.build_model("posts.by_thread", thread_id) do + render(conn, :by_thread_proxy, %{ + posts: posts, + user: user, + page: page, + limit: limit + }) + else + _ -> + ErrorHelpers.render_json_error(conn, 400, "Error, cannot get posts by thread") + end + end end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 16042e37..e9db0987 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -25,7 +25,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do alias EpochtalkServer.Models.Mention alias EpochtalkServerWeb.Helpers.ProxyConversion - plug :check_proxy when action in [:by_board] + plug :check_proxy when action in [:by_board, :slug_to_id, :viewed] @doc """ Used to retrieve recent threads @@ -357,7 +357,26 @@ defmodule EpochtalkServerWeb.Controllers.Thread do else conn end - + :slug_to_id -> + case Integer.parse(conn.params["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) + + if slug_as_id < System.get_env("THREADS_SEQ") |> String.to_integer() do + conn + |> render(:slug_to_id, id: slug_as_id) + |> halt() + else + conn + end + + _ -> + conn + end + :viewed -> + conn + |> send_resp(200, []) + |> halt() _ -> conn end diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index e9228613..56671b6e 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -27,7 +27,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do case model_type do "threads.by_board" -> build_thread_model_by_board(id) - + "posts.by_thread" -> + build_post_model_by_thread(id) _ -> build_model(model_type, [id]) end @@ -80,6 +81,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do where: b.id_board in ^ids, select: %{ id: b.id_board, + slug: b.id_board, cat_id: b.id_cat, name: b.name, description: b.description, @@ -97,11 +99,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do Enum.map(boards, fn board -> board = board - |> Map.put(:slug, String.downcase(String.replace(board.name, ~r/\s+/, "_"))) |> Map.put(:children, []) - - board = if board.cat_id, do: build_category_and_board_mapping(board) - board end) return_tuple(boards) @@ -115,6 +113,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do order_by: [desc: t.id_last_msg], select: %{ id: t.id_topic, + slug: t.id_topic, board_id: t.id_board, sticky: t.isSticky, locked: t.locked, @@ -141,7 +140,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do thread = thread |> Map.put(:title, first_post.title) - |> Map.put(:slug, first_post.title) |> Map.put(:user_id, first_post.user_id) |> Map.put(:username, first_post.username) |> Map.put(:user_deleted, false) @@ -169,10 +167,17 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do where: t.id_topic in ^ids, select: %{ id: t.id_topic, + slug: t.id_topic, board_id: t.id_board, sticky: t.isSticky, locked: t.locked, - metadata_views: t.numViews + view_count: t.numViews, + first_post_id: t.id_first_msg, + last_post_id: t.id_last_msg, + started_user_id: t.id_member_started, + updated_user_id: t.id_member_updated, + moderated: t.selfModerated, + post_count: t.numReplies } ) |> SmfRepo.all() @@ -182,13 +187,32 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do threads -> threads = - Enum.map(threads, fn thread -> - thread = if thread.board_id, do: build_thread_and_board_mapping(thread) - thread + Enum.reduce(threads, [], fn thread, acc -> + first_post = get_post(thread.first_post_id) + last_post = get_post(thread.last_post_id) + + thread = + thread + |> Map.put(:title, first_post.title) + |> Map.put(:user_id, first_post.user_id) + |> Map.put(:username, first_post.username) + |> Map.put(:user_deleted, false) + |> Map.put(:last_post_id, last_post.id) + |> Map.put(:last_post_created_at, last_post.created_at * 1000) + |> Map.put(:last_post_deleted, false) + |> Map.put(:last_post_user_id, last_post.user_id) + |> Map.put(:last_post_username, last_post.username) + |> Map.put(:last_post_user_deleted, false) + |> Map.put(:last_post_avatar, last_post.avatar) + |> Map.put(:last_viewed, nil) + |> Map.put(:created_at, first_post.created_at * 1000) + + acc = [thread | acc] + acc end) - return_tuple(threads) - end + return_tuple(Enum.reverse(threads)) + end end def build_post_model(ids) do @@ -201,7 +225,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do board_id: m.id_board, user_id: m.id_member, title: m.subject, - body: m.body + body: m.body, + updated_at: m.modifiedTime, + avatar: m.icon } ) |> SmfRepo.all() @@ -220,6 +246,48 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_post_model_by_thread(id) do + from(m in "smf_messages", + limit: 10, + where: m.id_topic == ^id, + order_by: [desc: m.posterTime], + select: %{ + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + user_id: m.id_member, + title: m.subject, + body: m.body, + updated_at: m.modifiedTime, + avatar: m.icon, + username: m.posterName, + user_email: m.posterEmail, + poster_time: m.posterTime, + poster_name: m.posterName, + modified_time: m.modifiedTime + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Posts not found for thread_id: #{id}"} + + posts -> + posts = + Enum.reduce(posts, [], fn post, acc -> + post = + post + |> Map.put(:created_at, post.poster_time * 1000) + |> Map.delete(:poster_time) + + acc = [post | acc] + acc + end) + + return_tuple(posts) + end + end + defp build_category_and_board_mapping(board) do {:ok, category} = build_category_model([board.cat_id]) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index de2acfad..5b9a8391 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -2,6 +2,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do alias EpochtalkServerWeb.Controllers.BoardJSON alias EpochtalkServerWeb.Controllers.ThreadJSON alias EpochtalkServerWeb.Helpers.ACL + alias EpochtalkServerWeb.Helpers.ProxyConversion @moduledoc """ Renders and formats `Post` data, in JSON format for frontend @@ -76,6 +77,41 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do } end + def by_thread_proxy(%{ + posts: posts, + page: page, + limit: limit + }) do + + {:ok, thread} = + if is_map(posts) do + ProxyConversion.build_thread_model([posts.thread_id]) + else + ProxyConversion.build_thread_model([List.first(posts).thread_id]) + end + + # format board data + {:ok, board} = ProxyConversion.build_board_model([thread.board_id]) + + board = + board + |> Map.put(:moderators, []) + + # format post data + posts = + posts + |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) + + # build by_thread results + %{ + posts: posts, + thread: thread, + board: board, + page: page, + limit: limit + } + end + ## === Private Helper Functions === defp handle_deleted_posts(posts, thread, user, authed_user_priority, view_deleted_posts) do @@ -295,4 +331,28 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do |> Map.delete(:highlight_color) |> Map.delete(:role_name) end + + defp format_proxy_post_data_for_by_thread(post) do + post + # if body_html does not exist, default to post.body + |> Map.put(:body_html, post.body) + |> Map.put(:user, %{ + id: post.user_id, + username: post.poster_name + # original_poster: post.original_poster, + # username: post.username, + # priority: if(is_nil(post.priority), do: post.default_priority, else: post.priority) + # deleted: post.user_deleted, + # signature: post.signature, + # post_count: post.post_count, + # highlight_color: post.highlight_color, + # role_name: post.role_name, + # stats: Map.get(post, :user_trust_stats), + # ignored: Map.get(post, :user_ignored), + # _ignored: Map.get(post, :user_ignored), + # activity: Map.get(post, :user_activity) + }) + |> Map.delete(:user_id) + |> Map.delete(:poster_name) + end end diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 002ccaec..5f0e3b72 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -97,11 +97,11 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do limit: limit }) do # format board data - board = + {:ok, board} = if is_map(threads) do - ProxyConversion.build_thread_and_board_mapping(threads) + ProxyConversion.build_board_model([threads.board_id]) else - ProxyConversion.build_thread_and_board_mapping(List.first(threads)) + ProxyConversion.build_board_model([List.first(threads).board_id]) end # format thread data @@ -112,7 +112,7 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do # build by_board results %{ normal: normal, - board: board.board, + board: board, page: page, limit: limit } diff --git a/priv/repo/migrations/20231110161916_add_sequences.exs b/priv/repo/migrations/20231110161916_add_sequences.exs index 655d260f..b8e68b64 100644 --- a/priv/repo/migrations/20231110161916_add_sequences.exs +++ b/priv/repo/migrations/20231110161916_add_sequences.exs @@ -4,7 +4,7 @@ defmodule EpochtalkServer.Repo.Migrations.AddSequences do def change do execute "ALTER SEQUENCE categories_id_seq RESTART WITH 50;" execute "ALTER SEQUENCE boards_id_seq RESTART WITH 500;" - execute "ALTER SEQUENCE threads_id_seq RESTART WITH 2000000;" + execute "ALTER SEQUENCE threads_id_seq RESTART WITH 6000000;" execute "ALTER SEQUENCE posts_id_seq RESTART WITH 60000000;" end end From b66786fe6860b53d2ae8cea40ae2932015ee2dea Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Mon, 8 Jan 2024 19:49:06 -0800 Subject: [PATCH 006/231] feat(proxy-pagination): Added pagination handling for threads_by_board and posts_by_thread through proxy --- lib/epochtalk_server_web/controllers/post.ex | 5 +- .../controllers/thread.ex | 5 +- .../helpers/proxy_conversion.ex | 160 ++++++------------ .../helpers/proxy_pagination.ex | 115 +++++++++++++ lib/epochtalk_server_web/json/post_json.ex | 7 +- lib/epochtalk_server_web/json/thread_json.ex | 5 +- 6 files changed, 184 insertions(+), 113 deletions(-) create mode 100644 lib/epochtalk_server_web/helpers/proxy_pagination.ex diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index e134219b..05b5b376 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -296,12 +296,13 @@ defmodule EpochtalkServerWeb.Controllers.Post do limit <- Validate.cast(attrs, "limit", :integer, default: 5), user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "posts.byThread"), - {:ok, posts} <- ProxyConversion.build_model("posts.by_thread", thread_id) do + {:ok, posts, data} <- ProxyConversion.build_model("posts.by_thread", thread_id, page, limit) do render(conn, :by_thread_proxy, %{ posts: posts, user: user, page: page, - limit: limit + limit: limit, + pagination_data: data }) else _ -> diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index e9db0987..f444b4ff 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -390,12 +390,13 @@ defmodule EpochtalkServerWeb.Controllers.Thread do limit <- Validate.cast(attrs, "limit", :integer, default: 5), user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "threads.byBoard"), - {:ok, threads} <- ProxyConversion.build_model("threads.by_board", board_id) do + {:ok, threads, data} <- ProxyConversion.build_model("threads.by_board", board_id, page, limit) do render(conn, :by_board_proxy, %{ threads: threads, user: user, page: page, - limit: limit + limit: limit, + pagination_data: data }) else _ -> diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 56671b6e..3cf39696 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -3,6 +3,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do import Ecto.Query alias EpochtalkServer.SmfRepo alias EpochtalkServer.ProxySupervisor + alias EpochtalkServerWeb.Helpers.ProxyPagination def start_link(arg) do GenServer.start_link(__MODULE__, arg, name: __MODULE__) @@ -13,49 +14,47 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do {:ok, opts} end - def build_model(model_type, ids) when is_nil(model_type) or is_nil(ids) do - {:ok, %{}} + def build_model(model_type, ids, _, _) when is_nil(model_type) or is_nil(ids) do + {:ok, %{}, %{}} end - def build_model(_, ids) when length(ids) > 10 do + def build_model(_, ids, _, _) when length(ids) > 25 do {:error, "Limit too large, please try again"} end - def build_model(model_type, id) when is_integer(id) do + def build_model(model_type, id, page, per_page) when is_integer(id) do ProxySupervisor.start_link([]) case model_type do "threads.by_board" -> - build_thread_model_by_board(id) + build_threads_by_board(id, page, per_page) "posts.by_thread" -> - build_post_model_by_thread(id) + build_posts_by_thread(id, page, per_page) _ -> - build_model(model_type, [id]) + build_model(model_type, [id], nil, nil) end end - def build_model(model_type, ids) do - ProxySupervisor.start_link([]) - + def build_model(model_type, ids, _, _) do case model_type do "category" -> - build_category_model(ids) + build_categories(ids) "board" -> - build_board_model(ids) + build_boards(ids) "thread" -> - build_thread_model(ids) + build_threads(ids) "post" -> - build_post_model(ids) + build_posts(ids) _ -> - build_model(nil, nil) + build_model(nil, nil, nil, nil) end end - def build_category_model(ids) do + def build_categories(ids) do from(c in "smf_categories", limit: ^length(ids), where: c.id_cat in ^ids, @@ -75,7 +74,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end - def build_board_model(ids) do + def build_boards(ids) do from(b in "smf_boards", limit: ^length(ids), where: b.id_board in ^ids, @@ -100,15 +99,17 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do board = board |> Map.put(:children, []) + board end) return_tuple(boards) end end - def build_thread_model_by_board(id) do + def build_threads_by_board(id, page, per_page) do + count_query = from t in "smf_topics", where: t.id_board == ^id, select: %{count: count(t.id_topic)} + from(t in "smf_topics", - limit: 10, where: t.id_board == ^id, order_by: [desc: t.id_last_msg], select: %{ @@ -126,12 +127,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do post_count: t.numReplies } ) - |> SmfRepo.all() + |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do - [] -> + {:ok, [], _} -> {:error, "Threads not found for board_id: #{id}"} - threads -> + {:ok, threads, data} -> threads = Enum.reduce(threads, [], fn thread, acc -> first_post = get_post(thread.first_post_id) @@ -153,15 +154,14 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> Map.put(:last_viewed, nil) |> Map.put(:created_at, first_post.created_at * 1000) - acc = [thread | acc] - acc + [thread | acc] end) - return_tuple(Enum.reverse(threads)) + return_tuple(Enum.reverse(threads), data) end end - def build_thread_model(ids) do + def build_threads(ids) do from(t in "smf_topics", limit: ^length(ids), where: t.id_topic in ^ids, @@ -207,15 +207,14 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> Map.put(:last_viewed, nil) |> Map.put(:created_at, first_post.created_at * 1000) - acc = [thread | acc] - acc + [thread | acc] end) return_tuple(Enum.reverse(threads)) end end - def build_post_model(ids) do + def build_posts(ids) do from(m in "smf_messages", limit: ^length(ids), where: m.id_msg in ^ids, @@ -236,21 +235,28 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do {:error, "Posts not found for ids: #{ids}"} posts -> - posts = - Enum.map(posts, fn post -> - post = if post.thread_id, do: build_post_and_thread_mapping(post) - post + posts = + Enum.reduce(posts, [], fn post, acc -> + post = + post + |> Map.put(:created_at, post.poster_time * 1000) + |> Map.delete(:poster_time) + + [post | acc] end) return_tuple(posts) end end - def build_post_model_by_thread(id) do + def build_posts_by_thread(id, page, per_page) do + count_query = from m in "smf_messages", where: m.id_topic == ^id, select: %{count: count(m.id_topic)} + + from(m in "smf_messages", - limit: 10, + limit: 15, where: m.id_topic == ^id, - order_by: [desc: m.posterTime], + order_by: [asc: m.posterTime], select: %{ id: m.id_msg, thread_id: m.id_topic, @@ -267,12 +273,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do modified_time: m.modifiedTime } ) - |> SmfRepo.all() + |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do - [] -> + {:ok, [], _} -> {:error, "Posts not found for thread_id: #{id}"} - posts -> + {:ok, posts, data} -> posts = Enum.reduce(posts, [], fn post, acc -> post = @@ -280,74 +286,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> Map.put(:created_at, post.poster_time * 1000) |> Map.delete(:poster_time) - acc = [post | acc] - acc + [post | acc] end) - - return_tuple(posts) + return_tuple(Enum.reverse(posts), data) end end - defp build_category_and_board_mapping(board) do - {:ok, category} = build_category_model([board.cat_id]) - - board = - board - |> Map.put(:category, category) - |> Map.put(:board_mapping, build_board_mapping(category, board)) - |> Map.delete(:cat_id) - - board - end - - def build_thread_and_board_mapping(thread) do - {:ok, board} = build_board_model([thread.board_id]) - - thread = - thread - |> Map.put(:board, board) - |> Map.delete(:board_id) - - thread - end - - defp build_post_and_thread_mapping(post) do - {:ok, thread} = build_thread_model([post.thread_id]) - - content = %{ - "title" => post.title, - "body" => post.body - } - - post = - post - |> Map.put(:thread, thread) - |> Map.delete(:thread_id) - |> Map.put(:content, content) - |> Map.delete(:title) - |> Map.delete(:body) - - post - end - - defp build_board_mapping(category, board) do - [ - %{ - id: category.id, - name: category.name, - type: "category", - view_order: category.view_order - }, - %{ - id: board.id, - name: board.name, - type: "board", - category_id: category.id, - view_order: category.view_order + 1 - } - ] - end - defp return_tuple(object) do if length(object) > 1 do {:ok, object} @@ -356,6 +300,14 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + defp return_tuple(object, data) do + if length(object) > 1 do + {:ok, object, data} + else + {:ok, List.first(object), data} + end + end + defp get_post(id) do from(m in "smf_messages", limit: 1, @@ -377,7 +329,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> SmfRepo.one() |> case do nil -> - {:error, "Posts not found for id: #{id}"} + {:error, "Post not found for id: #{id}"} post -> post diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex new file mode 100644 index 00000000..edca6efe --- /dev/null +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -0,0 +1,115 @@ +defmodule EpochtalkServerWeb.Helpers.ProxyPagination do + @moduledoc """ + Helper for paginating database queries + """ + import Ecto.Query + alias EpochtalkServer.SmfRepo + alias EpochtalkServerWeb.Helpers.Validate + + @doc """ + Takes in a query, page and per_page option, returns paginated data and relevant + pagination data for frontend (ex. page, limit, next, prev) + + ## Example + iex> import Ecto.Query + iex> alias EpochtalkServer.Models.{ Mention, Invitation } + iex> alias EpochtalkServerWeb.Helpers.Pagination + iex> Mention + ...> |> order_by(asc: :id) + ...> |> Pagination.page_simple(1, per_page: 25) + {:ok, [], %{next: false, + page: 1, + per_page: 25, + prev: false, + total_pages: 1, + total_records: 0}} + iex> Invitation + ...> |> order_by(desc: :email) + ...> |> Pagination.page_simple(1, per_page: 10) + {:ok, [], %{next: false, + page: 1, + per_page: 10, + prev: false, + total_pages: 1, + total_records: 0}} + """ + @spec page_simple(query :: Ecto.Queryable.t(), count_query :: Ecto.Queryable.t(), page :: integer | String.t() | nil, + per_page: integer | String.t() | nil + ) :: {:ok, list :: [term()] | [], pagination_data :: map()} + def page_simple(query, count_query, nil, per_page: nil), + do: page_simple(query, count_query, 1, per_page: 15) + + def page_simple(query, count_query, page, per_page: nil) when is_integer(page), + do: page_simple(query, count_query, page, per_page: 15) + + def page_simple(query, count_query, nil, per_page: per_page) when is_integer(per_page), + do: page_simple(query, count_query, 1, per_page: per_page) + + def page_simple(query, count_query, nil, per_page: per_page) when is_binary(per_page), + do: + page_simple(query, count_query, 1, per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1)) + + def page_simple(query, count_query, page, per_page: nil) when is_binary(page), + do: page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), per_page: 15) + + def page_simple(query, count_query, page, per_page: per_page) + when is_binary(page) and is_binary(per_page), + do: + page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), + per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1) + ) + + def page_simple(query, count_query, page, per_page: per_page) do + options = [prefix: "public"] + + total_records = + Keyword.get_lazy(options, :total_records, fn -> + total_records(count_query) + end) + + IO.inspect(total_records) + total_pages = total_pages(total_records, per_page) + IO.inspect(total_records) + + result = records(query, page, total_pages, per_page) + + pagination_data = %{ + next: page < total_pages, + prev: page > 1, + page: page, + per_page: per_page, + total_records: total_records, + total_pages: total_pages + } + + {:ok, result, pagination_data} + end + + defp records(_, page, total_pages, _) when page > total_pages, do: [] + + defp records(query, page, _, per_page) do + IO.inspect(per_page * (page - 1)) + query + |> limit(^per_page) + |> offset(^(per_page * (page - 1))) + |> SmfRepo.all() + end + + defp total_records(query) do + total_records = + query + |> exclude(:preload) + |> exclude(:order_by) + |> SmfRepo.one() + + total_records[:count] || 0 + end + + defp total_pages(0, _), do: 1 + + defp total_pages(total_records, per_page) do + (total_records / per_page) + |> Float.ceil() + |> round + end +end diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 5b9a8391..17e3cc85 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -85,13 +85,14 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do {:ok, thread} = if is_map(posts) do - ProxyConversion.build_thread_model([posts.thread_id]) + ProxyConversion.build_model("thread", [posts.thread_id], page, limit) else - ProxyConversion.build_thread_model([List.first(posts).thread_id]) + ProxyConversion.build_model("thread", [List.first(posts).thread_id], page, limit) end # format board data - {:ok, board} = ProxyConversion.build_board_model([thread.board_id]) + {:ok, board} = ProxyConversion.build_model("board", [thread.board_id], 1, 1) + board = board diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 5f0e3b72..2f69abb9 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -96,12 +96,13 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do page: page, limit: limit }) do + # format board data {:ok, board} = if is_map(threads) do - ProxyConversion.build_board_model([threads.board_id]) + ProxyConversion.build_model("board", [threads.board_id], 1, 1) else - ProxyConversion.build_board_model([List.first(threads).board_id]) + ProxyConversion.build_model("board", [List.first(threads).board_id], 1, 1) end # format thread data From 132311daa4d83ef895960dfcfa666a1fc4d073e6 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Tue, 9 Jan 2024 12:43:40 -0800 Subject: [PATCH 007/231] fix(format): Running mix format and removing any unused code --- lib/epochtalk_server_web/controllers/post.ex | 3 ++- lib/epochtalk_server_web/controllers/thread.ex | 6 +++++- .../helpers/proxy_conversion.ex | 15 ++++++++++----- .../helpers/proxy_pagination.ex | 17 +++++++++++------ lib/epochtalk_server_web/json/post_json.ex | 2 -- lib/epochtalk_server_web/json/thread_json.ex | 1 - 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 05b5b376..4e9cc841 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -296,7 +296,8 @@ defmodule EpochtalkServerWeb.Controllers.Post do limit <- Validate.cast(attrs, "limit", :integer, default: 5), user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "posts.byThread"), - {:ok, posts, data} <- ProxyConversion.build_model("posts.by_thread", thread_id, page, limit) do + {:ok, posts, data} <- + ProxyConversion.build_model("posts.by_thread", thread_id, page, limit) do render(conn, :by_thread_proxy, %{ posts: posts, user: user, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index f444b4ff..dc4ff163 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -357,6 +357,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do else conn end + :slug_to_id -> case Integer.parse(conn.params["slug"]) do {_, ""} -> @@ -373,10 +374,12 @@ defmodule EpochtalkServerWeb.Controllers.Thread do _ -> conn end + :viewed -> conn |> send_resp(200, []) |> halt() + _ -> conn end @@ -390,7 +393,8 @@ defmodule EpochtalkServerWeb.Controllers.Thread do limit <- Validate.cast(attrs, "limit", :integer, default: 5), user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "threads.byBoard"), - {:ok, threads, data} <- ProxyConversion.build_model("threads.by_board", board_id, page, limit) do + {:ok, threads, data} <- + ProxyConversion.build_model("threads.by_board", board_id, page, limit) do render(conn, :by_board_proxy, %{ threads: threads, user: user, diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 3cf39696..175c1769 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -28,8 +28,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do case model_type do "threads.by_board" -> build_threads_by_board(id, page, per_page) + "posts.by_thread" -> build_posts_by_thread(id, page, per_page) + _ -> build_model(model_type, [id], nil, nil) end @@ -99,6 +101,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do board = board |> Map.put(:children, []) + board end) @@ -107,7 +110,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_threads_by_board(id, page, per_page) do - count_query = from t in "smf_topics", where: t.id_board == ^id, select: %{count: count(t.id_topic)} + count_query = + from t in "smf_topics", where: t.id_board == ^id, select: %{count: count(t.id_topic)} from(t in "smf_topics", where: t.id_board == ^id, @@ -211,7 +215,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end) return_tuple(Enum.reverse(threads)) - end + end end def build_posts(ids) do @@ -235,7 +239,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do {:error, "Posts not found for ids: #{ids}"} posts -> - posts = + posts = Enum.reduce(posts, [], fn post, acc -> post = post @@ -250,8 +254,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_posts_by_thread(id, page, per_page) do - count_query = from m in "smf_messages", where: m.id_topic == ^id, select: %{count: count(m.id_topic)} - + count_query = + from m in "smf_messages", where: m.id_topic == ^id, select: %{count: count(m.id_topic)} from(m in "smf_messages", limit: 15, @@ -288,6 +292,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [post | acc] end) + return_tuple(Enum.reverse(posts), data) end end diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index edca6efe..6769893f 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -33,7 +33,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do total_pages: 1, total_records: 0}} """ - @spec page_simple(query :: Ecto.Queryable.t(), count_query :: Ecto.Queryable.t(), page :: integer | String.t() | nil, + @spec page_simple( + query :: Ecto.Queryable.t(), + count_query :: Ecto.Queryable.t(), + page :: integer | String.t() | nil, per_page: integer | String.t() | nil ) :: {:ok, list :: [term()] | [], pagination_data :: map()} def page_simple(query, count_query, nil, per_page: nil), @@ -47,10 +50,15 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do def page_simple(query, count_query, nil, per_page: per_page) when is_binary(per_page), do: - page_simple(query, count_query, 1, per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1)) + page_simple(query, count_query, 1, + per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1) + ) def page_simple(query, count_query, page, per_page: nil) when is_binary(page), - do: page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), per_page: 15) + do: + page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), + per_page: 15 + ) def page_simple(query, count_query, page, per_page: per_page) when is_binary(page) and is_binary(per_page), @@ -67,9 +75,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do total_records(count_query) end) - IO.inspect(total_records) total_pages = total_pages(total_records, per_page) - IO.inspect(total_records) result = records(query, page, total_pages, per_page) @@ -88,7 +94,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do defp records(_, page, total_pages, _) when page > total_pages, do: [] defp records(query, page, _, per_page) do - IO.inspect(per_page * (page - 1)) query |> limit(^per_page) |> offset(^(per_page * (page - 1))) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 17e3cc85..da9e9015 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -82,7 +82,6 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do page: page, limit: limit }) do - {:ok, thread} = if is_map(posts) do ProxyConversion.build_model("thread", [posts.thread_id], page, limit) @@ -93,7 +92,6 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do # format board data {:ok, board} = ProxyConversion.build_model("board", [thread.board_id], 1, 1) - board = board |> Map.put(:moderators, []) diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 2f69abb9..f8d0d6d3 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -96,7 +96,6 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do page: page, limit: limit }) do - # format board data {:ok, board} = if is_map(threads) do From 506cb0fa96aa2ff32557198a101f0b7887f63579 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Tue, 9 Jan 2024 13:49:25 -0800 Subject: [PATCH 008/231] feat(smfrepo-config): Adding config info for smfrepo --- config/dev.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/dev.exs b/config/dev.exs index dd9eb900..b7398a50 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -10,6 +10,16 @@ config :epochtalk_server, EpochtalkServer.Repo, show_sensitive_data_on_connection_error: true, pool_size: 10 +config :epochtalk_server, EpochtalkServer.SmfRepo, + username: System.get_env("SMF_REPO_USERNAME"), + password: System.get_env("SMF_REPO_PASSWORD"), + hostname: System.get_env("SMF_REPO_HOSTNAME"), + database: System.get_env("SMF_REPO_DATABASE"), + port: System.get_env("SMF_REPO_PORT"), + stacktrace: true, + show_sensitive_data_on_connection_error: true, + pool_size: 3 + # For development, we disable any cache and enable # debugging and code reloading. # From 2c486562ca3ec5b4c9ad198597ea8a8db726ec57 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Dec 2023 14:32:23 -1000 Subject: [PATCH 009/231] ci(docker.secret): add frontend configs --- config/docker.secret.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/docker.secret.exs b/config/docker.secret.exs index b2def989..699ab7e4 100644 --- a/config/docker.secret.exs +++ b/config/docker.secret.exs @@ -8,3 +8,13 @@ config :epochtalk_server, EpochtalkServer.Repo, database: System.get_env("DATABASE_NAME"), hostname: System.get_env("DATABASE_HOST"), pool_size: 10 + +config :epochtalk_server, + ecto_repos: [EpochtalkServer.Repo], + frontend_config: %{ + frontend_url: System.get_env("FRONTEND_URL") || "http://localhost:8000", + backend_url: System.get_env("BACKEND_URL") || "http://localhost:4000", + ga_key: System.get_env("GA_KEY") || "UA-XXXXX-Y", + emailer: %{ses_mode: System.get_env("EMAILER_SES_MODE") || false, options: %{from_address: System.get_env("EMAILER_FROM_ADDRESS") || "info@epochtalk.com"}}, + images: %{s3_mode: System.get_env("IMAGES_S3_MODE") || false, options: %{local_host: System.get_env("IMAGES_LOCAL_HOST") || "http://localhost:4000"}} + } From de2f451b44884ab3b8fb0007380769ebbf0731e6 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Dec 2023 20:56:02 -1000 Subject: [PATCH 010/231] ci(docker.secret): configure redis and endpoint --- config/docker.secret.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/docker.secret.exs b/config/docker.secret.exs index 699ab7e4..cef4f08f 100644 --- a/config/docker.secret.exs +++ b/config/docker.secret.exs @@ -18,3 +18,12 @@ config :epochtalk_server, emailer: %{ses_mode: System.get_env("EMAILER_SES_MODE") || false, options: %{from_address: System.get_env("EMAILER_FROM_ADDRESS") || "info@epochtalk.com"}}, images: %{s3_mode: System.get_env("IMAGES_S3_MODE") || false, options: %{local_host: System.get_env("IMAGES_LOCAL_HOST") || "http://localhost:4000"}} } + +# configure redis for production +config :guardian_redis, :redis, + host: System.get_env("REDIS_HOST") || "redis", + port: System.get_env("REDIS_PORT") || 6379, + pool_size: 10 + +config :epochtalk_server, EpochtalkServerWeb.Endpoint, + url: [host: System.get_env("FRONTEND_URL")] From 18ff5f40556ad094d42fa26e845609c42c9d5429 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 20 Dec 2023 08:44:36 -1000 Subject: [PATCH 011/231] ci(config/runtime): configure frontend --- config/runtime.exs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/config/runtime.exs b/config/runtime.exs index 1386c43d..662c0644 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -103,4 +103,46 @@ if config_env() == :prod do # Configure Redis for Session Storage config :epochtalk_server, :redix, host: System.get_env("REDIS_HOST") || "127.0.0.1" + + # Configure frontend + config :epochtalk_server, frontend_config: %{ + frontend_url: System.get_env("FRONTEND_URL") || "http://localhost:8000", + backend_url: System.get_env("BACKEND_URL") || "http://localhost:4000", + newbie_enabled: System.get_env("NEWBIE_ENABLED") || false, + login_required: System.get_env("LOGIN_REQUIRED") || false, + invite_only: System.get_env("INVITE_ONLY") || false, + verify_registration: System.get_env("VERIFY_REGISTRATION") || true, + post_max_length: System.get_env("POST_MAX_LENGTH") || 10_000, + max_image_size: System.get_env("MAX_IMAGE_SIZE") || 10_485_760, + max_avatar_size: System.get_env("MAX_AVATAR_SIZE") || 102_400, + mobile_break_width: System.get_env("MOBILE_BREAK_WIDTH") || 767, + ga_key: System.get_env("GA_KEY") || "UA-XXXXX-Y", + revision: nil, + website: %{ + title: System.get_env("WEBSITE_TITLE") || "Epochtalk Forums", + description: System.get_env("WEBSITE_DESCRIPTION") || "Open source forum software", + keywords: System.get_env("WEBSITE_KEYWORDS") || "open source, free forum, forum software, forum", + logo: System.get_env("WEBSITE_LOGO") || nil, + favicon: System.get_env("WEBSITE_FAVICON") || nil, + default_avatar: System.get_env("WEBSITE_DEFAULT_AVATAR")"/images/avatar.png", + default_avatar_shape: System.get_env("WEBSITE_DEFAULT_AVATAR_SHAPE") || "circle" + }, + portal: %{ + enabled: System.get_env("PORTAL_ENABLED") || false, + board_id: System.get_env("PORTAL_BOARD_ID") || nil + }, + emailer: %{ + ses_mode: System.get_env("EMAILER_SES_MODE") || false, + options: %{ + from_address: System.get_env("EMAILER_OPTIONS_FROM_ADDRESS") || "info@epochtalk.com" + } + }, + images: %{ + s3_mode: System.get_env("IMAGES_S3_MODE") || false, + options: %{ + local_host: System.get_env("IMAGES_OPTIONS_LOCAL_HOST") || "http://localhost:4000" + } + }, + rate_limiting: %{} + } end From 01c5cc06744f40dbacf2eac7be74dd82cb9a2fd2 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 20 Dec 2023 08:45:42 -1000 Subject: [PATCH 012/231] ci(config/docker): remove docker config this project actually loads production configs from config/runtime.exs --- Dockerfile | 3 --- config/docker.secret.exs | 29 ----------------------------- 2 files changed, 32 deletions(-) delete mode 100644 config/docker.secret.exs diff --git a/Dockerfile b/Dockerfile index 4b37ba14..ab78109a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,9 +7,6 @@ RUN mix local.rebar --force ADD . . RUN mix deps.get -# enable configuration by environment -COPY config/docker.secret.exs config/prod.secret.exs - # compile for production ENV MIX_ENV=prod RUN mix compile diff --git a/config/docker.secret.exs b/config/docker.secret.exs deleted file mode 100644 index cef4f08f..00000000 --- a/config/docker.secret.exs +++ /dev/null @@ -1,29 +0,0 @@ -import Config - -# Configure your database -config :epochtalk_server, EpochtalkServer.Repo, - adapter: Ecto.Adapters.Postgres, - username: System.get_env("DATABASE_USER"), - password: System.get_env("DATABASE_PASSWORD"), - database: System.get_env("DATABASE_NAME"), - hostname: System.get_env("DATABASE_HOST"), - pool_size: 10 - -config :epochtalk_server, - ecto_repos: [EpochtalkServer.Repo], - frontend_config: %{ - frontend_url: System.get_env("FRONTEND_URL") || "http://localhost:8000", - backend_url: System.get_env("BACKEND_URL") || "http://localhost:4000", - ga_key: System.get_env("GA_KEY") || "UA-XXXXX-Y", - emailer: %{ses_mode: System.get_env("EMAILER_SES_MODE") || false, options: %{from_address: System.get_env("EMAILER_FROM_ADDRESS") || "info@epochtalk.com"}}, - images: %{s3_mode: System.get_env("IMAGES_S3_MODE") || false, options: %{local_host: System.get_env("IMAGES_LOCAL_HOST") || "http://localhost:4000"}} - } - -# configure redis for production -config :guardian_redis, :redis, - host: System.get_env("REDIS_HOST") || "redis", - port: System.get_env("REDIS_PORT") || 6379, - pool_size: 10 - -config :epochtalk_server, EpochtalkServerWeb.Endpoint, - url: [host: System.get_env("FRONTEND_URL")] From 39b9662f4fa23011ea6eb734cabecdcbd71d4ee5 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 20 Dec 2023 10:10:51 -1000 Subject: [PATCH 013/231] ci(config/runtime): fix || --- config/runtime.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index 662c0644..11fc34cf 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -124,7 +124,7 @@ if config_env() == :prod do keywords: System.get_env("WEBSITE_KEYWORDS") || "open source, free forum, forum software, forum", logo: System.get_env("WEBSITE_LOGO") || nil, favicon: System.get_env("WEBSITE_FAVICON") || nil, - default_avatar: System.get_env("WEBSITE_DEFAULT_AVATAR")"/images/avatar.png", + default_avatar: System.get_env("WEBSITE_DEFAULT_AVATAR") || "/images/avatar.png", default_avatar_shape: System.get_env("WEBSITE_DEFAULT_AVATAR_SHAPE") || "circle" }, portal: %{ From 04c9501e654f31b5d7a01ffc89fcb58150e8f8d3 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 20 Dec 2023 11:46:13 -1000 Subject: [PATCH 014/231] ci(config/runtime): set configurations for ses --- config/runtime.exs | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 11fc34cf..6ba1186b 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -80,11 +80,34 @@ if config_env() == :prod do # config :swoosh, :api_client, Swoosh.ApiClient.Hackney # # See https://hexdocs.pm/swoosh/Swoosh.html#module-installation for details. - config :epochtalk_server, EpochtalkServer.Mailer, - relay: System.get_env("EMAILER_SMTP_RELAY") || "smtp.example.com", - username: System.get_env("EMAILER_SMTP_USERNAME") || "username", - password: System.get_env("EMAILER_SMTP_PASSWORD") || "password", - port: System.get_env("EMAILER_SMTP_PORT") || 465 + if System.get_env("EMAILER_SES_MODE") do + emailer_ses_region = + System.get_env("EMAILER_SES_REGION") || + raise """ + environment variable EMAILER_SES_REGION is missing. + """ + emailer_ses_aws_access_key = + System.get_env("EMAILER_SES_AWS_ACCESS_KEY") || + raise """ + environment variable EMAILER_SES_AWS_ACCESS_KEY missing. + """ + emailer_ses_aws_secret_key = + System.get_env("EMAILER_SES_AWS_SECRET_KEY") || + raise """ + environment variable EMAILER_SES_AWS_SECRET_KEY missing. + """ + config :epochtalk_server, EpochtalkServer.Mailer, + adapter: Swoosh.Adapters.AmazonSES, + region: emailer_ses_region, + access_key: emailer_ses_aws_access_key, + secret: emailer_ses_aws_secret_key + else + config :epochtalk_server, EpochtalkServer.Mailer, + relay: System.get_env("EMAILER_SMTP_RELAY") || "smtp.example.com", + username: System.get_env("EMAILER_SMTP_USERNAME") || "username", + password: System.get_env("EMAILER_SMTP_PASSWORD") || "password", + port: System.get_env("EMAILER_SMTP_PORT") || 465 + end # Configure Guardian for Runtime config :epochtalk_server, EpochtalkServer.Auth.Guardian, From 120ef18c7e573f6841fc37be982cb9774f4bf537 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 9 Jan 2024 11:06:22 -1000 Subject: [PATCH 015/231] style(config/runtime.exs): mix format --- config/runtime.exs | 85 ++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 6ba1186b..b557a2ef 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -86,16 +86,19 @@ if config_env() == :prod do raise """ environment variable EMAILER_SES_REGION is missing. """ + emailer_ses_aws_access_key = System.get_env("EMAILER_SES_AWS_ACCESS_KEY") || raise """ environment variable EMAILER_SES_AWS_ACCESS_KEY missing. """ + emailer_ses_aws_secret_key = System.get_env("EMAILER_SES_AWS_SECRET_KEY") || raise """ environment variable EMAILER_SES_AWS_SECRET_KEY missing. """ + config :epochtalk_server, EpochtalkServer.Mailer, adapter: Swoosh.Adapters.AmazonSES, region: emailer_ses_region, @@ -128,44 +131,46 @@ if config_env() == :prod do config :epochtalk_server, :redix, host: System.get_env("REDIS_HOST") || "127.0.0.1" # Configure frontend - config :epochtalk_server, frontend_config: %{ - frontend_url: System.get_env("FRONTEND_URL") || "http://localhost:8000", - backend_url: System.get_env("BACKEND_URL") || "http://localhost:4000", - newbie_enabled: System.get_env("NEWBIE_ENABLED") || false, - login_required: System.get_env("LOGIN_REQUIRED") || false, - invite_only: System.get_env("INVITE_ONLY") || false, - verify_registration: System.get_env("VERIFY_REGISTRATION") || true, - post_max_length: System.get_env("POST_MAX_LENGTH") || 10_000, - max_image_size: System.get_env("MAX_IMAGE_SIZE") || 10_485_760, - max_avatar_size: System.get_env("MAX_AVATAR_SIZE") || 102_400, - mobile_break_width: System.get_env("MOBILE_BREAK_WIDTH") || 767, - ga_key: System.get_env("GA_KEY") || "UA-XXXXX-Y", - revision: nil, - website: %{ - title: System.get_env("WEBSITE_TITLE") || "Epochtalk Forums", - description: System.get_env("WEBSITE_DESCRIPTION") || "Open source forum software", - keywords: System.get_env("WEBSITE_KEYWORDS") || "open source, free forum, forum software, forum", - logo: System.get_env("WEBSITE_LOGO") || nil, - favicon: System.get_env("WEBSITE_FAVICON") || nil, - default_avatar: System.get_env("WEBSITE_DEFAULT_AVATAR") || "/images/avatar.png", - default_avatar_shape: System.get_env("WEBSITE_DEFAULT_AVATAR_SHAPE") || "circle" - }, - portal: %{ - enabled: System.get_env("PORTAL_ENABLED") || false, - board_id: System.get_env("PORTAL_BOARD_ID") || nil - }, - emailer: %{ - ses_mode: System.get_env("EMAILER_SES_MODE") || false, - options: %{ - from_address: System.get_env("EMAILER_OPTIONS_FROM_ADDRESS") || "info@epochtalk.com" - } - }, - images: %{ - s3_mode: System.get_env("IMAGES_S3_MODE") || false, - options: %{ - local_host: System.get_env("IMAGES_OPTIONS_LOCAL_HOST") || "http://localhost:4000" - } - }, - rate_limiting: %{} - } + config :epochtalk_server, + frontend_config: %{ + frontend_url: System.get_env("FRONTEND_URL") || "http://localhost:8000", + backend_url: System.get_env("BACKEND_URL") || "http://localhost:4000", + newbie_enabled: System.get_env("NEWBIE_ENABLED") || false, + login_required: System.get_env("LOGIN_REQUIRED") || false, + invite_only: System.get_env("INVITE_ONLY") || false, + verify_registration: System.get_env("VERIFY_REGISTRATION") || true, + post_max_length: System.get_env("POST_MAX_LENGTH") || 10_000, + max_image_size: System.get_env("MAX_IMAGE_SIZE") || 10_485_760, + max_avatar_size: System.get_env("MAX_AVATAR_SIZE") || 102_400, + mobile_break_width: System.get_env("MOBILE_BREAK_WIDTH") || 767, + ga_key: System.get_env("GA_KEY") || "UA-XXXXX-Y", + revision: nil, + website: %{ + title: System.get_env("WEBSITE_TITLE") || "Epochtalk Forums", + description: System.get_env("WEBSITE_DESCRIPTION") || "Open source forum software", + keywords: + System.get_env("WEBSITE_KEYWORDS") || "open source, free forum, forum software, forum", + logo: System.get_env("WEBSITE_LOGO") || nil, + favicon: System.get_env("WEBSITE_FAVICON") || nil, + default_avatar: System.get_env("WEBSITE_DEFAULT_AVATAR") || "/images/avatar.png", + default_avatar_shape: System.get_env("WEBSITE_DEFAULT_AVATAR_SHAPE") || "circle" + }, + portal: %{ + enabled: System.get_env("PORTAL_ENABLED") || false, + board_id: System.get_env("PORTAL_BOARD_ID") || nil + }, + emailer: %{ + ses_mode: System.get_env("EMAILER_SES_MODE") || false, + options: %{ + from_address: System.get_env("EMAILER_OPTIONS_FROM_ADDRESS") || "info@epochtalk.com" + } + }, + images: %{ + s3_mode: System.get_env("IMAGES_S3_MODE") || false, + options: %{ + local_host: System.get_env("IMAGES_OPTIONS_LOCAL_HOST") || "http://localhost:4000" + } + }, + rate_limiting: %{} + } end From d49c28557cbe49493bf4820f356737ffceb25907 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Tue, 9 Jan 2024 13:33:47 -0800 Subject: [PATCH 016/231] feat(runtime): setup config for smfrepo in runtime.exs to use environment variables --- config/dev.exs | 2 +- config/runtime.exs | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/config/dev.exs b/config/dev.exs index b7398a50..e51b6265 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -18,7 +18,7 @@ config :epochtalk_server, EpochtalkServer.SmfRepo, port: System.get_env("SMF_REPO_PORT"), stacktrace: true, show_sensitive_data_on_connection_error: true, - pool_size: 3 + pool_size: 5 # For development, we disable any cache and enable # debugging and code reloading. diff --git a/config/runtime.exs b/config/runtime.exs index b557a2ef..d617be5f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -112,6 +112,47 @@ if config_env() == :prod do port: System.get_env("EMAILER_SMTP_PORT") || 465 end + # Configure SmfRepo for proxy + smf_repo_username = + System.get_env("SMF_REPO_USERNAME") || + raise """ + environment variable SMF_REPO_USERNAME is missing. + """ + + smf_repo_password = + System.get_env("SMF_REPO_PASSWORD") || + raise """ + environment variable SMF_REPO_PASSWORD is missing. + """ + + smf_repo_hostname = + System.get_env("SMF_REPO_HOSTNAME") || + raise """ + environment variable SMF_REPO_HOSTNAME is missing. + """ + + smf_repo_database = + System.get_env("SMF_REPO_DATABASE") || + raise """ + environment variable SMF_REPO_DATABASE is missing. + """ + + smf_repo_port = + System.get_env("SMF_REPO_PORT") || + raise """ + environment variable SMF_REPO_PORT is missing. + """ + + config :epochtalk_server, EpochtalkServer.SmfRepo, + username: smf_repo_username, + password: smf_repo_password, + hostname: smf_repo_hostname, + database: smf_repo_database, + port: smf_repo_port, + stacktrace: true, + show_sensitive_data_on_connection_error: true, + pool_size: 5 + # Configure Guardian for Runtime config :epochtalk_server, EpochtalkServer.Auth.Guardian, secret_key: From e6239a0b0a7bfc5146003aecfbad90f1fc242103 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Tue, 9 Jan 2024 18:53:48 -0800 Subject: [PATCH 017/231] fix(runtime): show sensitive data set to false --- config/runtime.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index d617be5f..f0e4c64c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -150,7 +150,7 @@ if config_env() == :prod do database: smf_repo_database, port: smf_repo_port, stacktrace: true, - show_sensitive_data_on_connection_error: true, + show_sensitive_data_on_connection_error: false, pool_size: 5 # Configure Guardian for Runtime From 2c53a196ef12bc55b84a78e773dde36d6455b722 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Tue, 9 Jan 2024 18:57:30 -0800 Subject: [PATCH 018/231] fix(smfrepo-supervisor): removed proxy supervisor call, moved starting smfrepo into application.ex --- lib/epochtalk_server/application.ex | 2 ++ lib/epochtalk_server_web/helpers/proxy_conversion.ex | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 15e929cf..57799cb7 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -19,6 +19,8 @@ defmodule EpochtalkServer.Application do {Redix, host: redix_config()[:host], name: redix_config()[:name]}, # Start the Ecto repository EpochtalkServer.Repo, + # Start the Smf repository + EpochtalkServer.SmfRepo, # Start Role Cache EpochtalkServer.Cache.Role, # Warm frontend_config variable (referenced by api controllers) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 175c1769..be1914a4 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -2,7 +2,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do use GenServer import Ecto.Query alias EpochtalkServer.SmfRepo - alias EpochtalkServer.ProxySupervisor alias EpochtalkServerWeb.Helpers.ProxyPagination def start_link(arg) do @@ -23,8 +22,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_model(model_type, id, page, per_page) when is_integer(id) do - ProxySupervisor.start_link([]) - case model_type do "threads.by_board" -> build_threads_by_board(id, page, per_page) From a090366d0c0d1367614ddb293baa39cf8c938a94 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Wed, 10 Jan 2024 15:14:39 -0800 Subject: [PATCH 019/231] fix(proxy-conversion): Added is_proxy flag to threads; Removed unused avatar field from proxy data pull --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index be1914a4..21500e16 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -151,9 +151,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> Map.put(:last_post_user_id, last_post.user_id) |> Map.put(:last_post_username, last_post.username) |> Map.put(:last_post_user_deleted, false) - |> Map.put(:last_post_avatar, last_post.avatar) |> Map.put(:last_viewed, nil) |> Map.put(:created_at, first_post.created_at * 1000) + |> Map.put(:is_proxy, true) [thread | acc] end) @@ -204,7 +204,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> Map.put(:last_post_user_id, last_post.user_id) |> Map.put(:last_post_username, last_post.username) |> Map.put(:last_post_user_deleted, false) - |> Map.put(:last_post_avatar, last_post.avatar) |> Map.put(:last_viewed, nil) |> Map.put(:created_at, first_post.created_at * 1000) @@ -226,8 +225,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do user_id: m.id_member, title: m.subject, body: m.body, - updated_at: m.modifiedTime, - avatar: m.icon + updated_at: m.modifiedTime } ) |> SmfRepo.all() @@ -266,7 +264,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do title: m.subject, body: m.body, updated_at: m.modifiedTime, - avatar: m.icon, username: m.posterName, user_email: m.posterEmail, poster_time: m.posterTime, @@ -324,8 +321,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do username: m.posterName, user_email: m.posterEmail, created_at: m.posterTime, - modified_time: m.modifiedTime, - avatar: m.icon + modified_time: m.modifiedTime } ) |> SmfRepo.one() From 33ffd22eba95395891092e9c879371d8f4ab3072 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Wed, 10 Jan 2024 15:20:18 -0800 Subject: [PATCH 020/231] refactor(proxy-conversion): removed unused genserver code --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 21500e16..4d16a790 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -1,18 +1,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do - use GenServer import Ecto.Query alias EpochtalkServer.SmfRepo alias EpochtalkServerWeb.Helpers.ProxyPagination - def start_link(arg) do - GenServer.start_link(__MODULE__, arg, name: __MODULE__) - end - - @impl true - def init(opts) do - {:ok, opts} - end - def build_model(model_type, ids, _, _) when is_nil(model_type) or is_nil(ids) do {:ok, %{}, %{}} end From ddd04f5e78054a1f14d599142406f59c46efffbf Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Wed, 10 Jan 2024 15:57:47 -0800 Subject: [PATCH 021/231] refactor(smfrepo): Removed SmfRepo from application.ex; Added ProxySupervisor to proxy_conversion and proxy_pagination --- lib/epochtalk_server/application.ex | 2 -- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 3 +++ lib/epochtalk_server_web/helpers/proxy_pagination.ex | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 57799cb7..15e929cf 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -19,8 +19,6 @@ defmodule EpochtalkServer.Application do {Redix, host: redix_config()[:host], name: redix_config()[:name]}, # Start the Ecto repository EpochtalkServer.Repo, - # Start the Smf repository - EpochtalkServer.SmfRepo, # Start Role Cache EpochtalkServer.Cache.Role, # Warm frontend_config variable (referenced by api controllers) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 4d16a790..c6aed799 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -1,6 +1,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do import Ecto.Query alias EpochtalkServer.SmfRepo + alias EpochtalkServer.ProxySupervisor alias EpochtalkServerWeb.Helpers.ProxyPagination def build_model(model_type, ids, _, _) when is_nil(model_type) or is_nil(ids) do @@ -12,6 +13,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_model(model_type, id, page, per_page) when is_integer(id) do + ProxySupervisor.start_link([]) + case model_type do "threads.by_board" -> build_threads_by_board(id, page, per_page) diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index 6769893f..35b3b359 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -4,6 +4,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do """ import Ecto.Query alias EpochtalkServer.SmfRepo + alias EpochtalkServer.ProxySupervisor alias EpochtalkServerWeb.Helpers.Validate @doc """ @@ -68,6 +69,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do ) def page_simple(query, count_query, page, per_page: per_page) do + ProxySupervisor.start_link([]) + options = [prefix: "public"] total_records = From 2377786fa8d2726f7bfe58fd3baacb4add87adce Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Wed, 10 Jan 2024 17:09:39 -0800 Subject: [PATCH 022/231] fix(runtime): pipe String.to_integer() for port env --- config/runtime.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index f0e4c64c..7bfad201 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -138,7 +138,7 @@ if config_env() == :prod do """ smf_repo_port = - System.get_env("SMF_REPO_PORT") || + System.get_env("SMF_REPO_PORT") |> String.to_integer() || raise """ environment variable SMF_REPO_PORT is missing. """ From 269cc1a3340b8d5b9c0d55643399a06d267b743e Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Wed, 10 Jan 2024 17:30:26 -0800 Subject: [PATCH 023/231] refactor(env): Use application.get_env for board and thread seq env variables --- config/config.exs | 7 +++++++ config/runtime.exs | 7 +++++++ lib/epochtalk_server_web/controllers/board.ex | 5 +++-- lib/epochtalk_server_web/controllers/thread.ex | 9 ++++++--- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index b00e0781..6fda5187 100644 --- a/config/config.exs +++ b/config/config.exs @@ -37,6 +37,13 @@ config :epochtalk_server, rate_limiting: %{} } +# Configure proxy +config :epochtalk_server, + proxy_config: %{ + threads_seq: System.get_env("THREADS_SEQ") || "6000000", + boards_seq: System.get_env("BOARDS_SEQ") || "500" + } + # Configure Guardian config :epochtalk_server, EpochtalkServer.Auth.Guardian, issuer: "EpochtalkServer", diff --git a/config/runtime.exs b/config/runtime.exs index f0e4c64c..b4e6a20b 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -171,6 +171,13 @@ if config_env() == :prod do # Configure Redis for Session Storage config :epochtalk_server, :redix, host: System.get_env("REDIS_HOST") || "127.0.0.1" + # Configure proxy + config :epochtalk_server, + proxy_config: %{ + threads_seq: System.get_env("THREADS_SEQ") || "6000000", + boards_seq: System.get_env("BOARDS_SEQ") || "500" + } + # Configure frontend config :epochtalk_server, frontend_config: %{ diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 42872cd2..f7ec7d81 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -115,8 +115,9 @@ defmodule EpochtalkServerWeb.Controllers.Board do case Integer.parse(conn.params["slug"]) do {_, ""} -> slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - - if slug_as_id < System.get_env("BOARDS_SEQ") |> String.to_integer() do + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() + if slug_as_id < boards_seq do conn |> render(:slug_to_id, id: slug_as_id) |> halt() diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index dc4ff163..e9c7db0e 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -349,8 +349,9 @@ defmodule EpochtalkServerWeb.Controllers.Thread do conn = case conn.private.phoenix_action do :by_board -> - if Validate.cast(conn.params, "board_id", :integer, required: true) < - System.get_env("BOARDS_SEQ") |> String.to_integer() do + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() + if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do conn |> proxy_by_board(conn.params) |> halt() @@ -363,7 +364,9 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {_, ""} -> slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - if slug_as_id < System.get_env("THREADS_SEQ") |> String.to_integer() do + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() + if slug_as_id < threads_seq do conn |> render(:slug_to_id, id: slug_as_id) |> halt() From c7554da167578611066388028532b41d29cfe603 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 12 Jan 2024 13:43:05 -0800 Subject: [PATCH 024/231] fix(dev): Parse port env variable as integer --- config/dev.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/dev.exs b/config/dev.exs index e51b6265..61884ed4 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -15,7 +15,7 @@ config :epochtalk_server, EpochtalkServer.SmfRepo, password: System.get_env("SMF_REPO_PASSWORD"), hostname: System.get_env("SMF_REPO_HOSTNAME"), database: System.get_env("SMF_REPO_DATABASE"), - port: System.get_env("SMF_REPO_PORT"), + port: System.get_env("SMF_REPO_PORT") |> String.to_integer(), stacktrace: true, show_sensitive_data_on_connection_error: true, pool_size: 5 From 30be85faf615105ffcf4e5ae82fdad368194eb5a Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 12 Jan 2024 13:45:35 -0800 Subject: [PATCH 025/231] fix(runtime): Increased pool_size for smf_repo to help with connection issues --- config/dev.exs | 2 +- config/runtime.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 61884ed4..b251dfcb 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -18,7 +18,7 @@ config :epochtalk_server, EpochtalkServer.SmfRepo, port: System.get_env("SMF_REPO_PORT") |> String.to_integer(), stacktrace: true, show_sensitive_data_on_connection_error: true, - pool_size: 5 + pool_size: 10 # For development, we disable any cache and enable # debugging and code reloading. diff --git a/config/runtime.exs b/config/runtime.exs index b4e6a20b..a55352e0 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -151,7 +151,7 @@ if config_env() == :prod do port: smf_repo_port, stacktrace: true, show_sensitive_data_on_connection_error: false, - pool_size: 5 + pool_size: 10 # Configure Guardian for Runtime config :epochtalk_server, EpochtalkServer.Auth.Guardian, From 2215475708de5997a30c3136ec0416f0c125b5c5 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 12 Jan 2024 15:21:50 -0800 Subject: [PATCH 026/231] refactor(proxy): Added .env to gitignore; Added moduledocs to new modules; Removed proxy_supervisor and related code --- .gitignore | 3 +++ .../conversion/proxy_supervisor.ex | 17 ----------------- lib/epochtalk_server/smf_repo.ex | 3 +++ lib/epochtalk_server_web/controllers/board.ex | 1 + lib/epochtalk_server_web/controllers/thread.ex | 2 ++ .../helpers/proxy_conversion.ex | 7 ++++--- .../helpers/proxy_pagination.ex | 3 --- 7 files changed, 13 insertions(+), 23 deletions(-) delete mode 100644 lib/epochtalk_server/conversion/proxy_supervisor.ex diff --git a/.gitignore b/.gitignore index a5795582..dc338a57 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ priv/repo/seeds/permissions.json # Ignore secret config files config/*.secret.exs + +#Ignore env file +/.env diff --git a/lib/epochtalk_server/conversion/proxy_supervisor.ex b/lib/epochtalk_server/conversion/proxy_supervisor.ex deleted file mode 100644 index a3cca086..00000000 --- a/lib/epochtalk_server/conversion/proxy_supervisor.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule EpochtalkServer.ProxySupervisor do - use Supervisor - - def start_link(arg) do - Supervisor.start_link(__MODULE__, arg, name: __MODULE__) - end - - @impl true - def init(_arg) do - children = [ - EpochtalkServer.SmfRepo - # Add reporters as children of your supervision tree. - ] - - Supervisor.init(children, strategy: :one_for_one) - end -end diff --git a/lib/epochtalk_server/smf_repo.ex b/lib/epochtalk_server/smf_repo.ex index 3d3bbb47..af431a36 100644 --- a/lib/epochtalk_server/smf_repo.ex +++ b/lib/epochtalk_server/smf_repo.ex @@ -1,4 +1,7 @@ defmodule EpochtalkServer.SmfRepo do + @moduledoc """ + SmfRepo, for connecting to old btct DB for proxy data pulling + """ use Ecto.Repo, otp_app: :epochtalk_server, adapter: Ecto.Adapters.MyXQL diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index f7ec7d81..2aff6cf9 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -117,6 +117,7 @@ defmodule EpochtalkServerWeb.Controllers.Board do slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) boards_seq = boards_seq |> String.to_integer() + if slug_as_id < boards_seq do conn |> render(:slug_to_id, id: slug_as_id) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index e9c7db0e..20a2fc28 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -351,6 +351,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do :by_board -> %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) boards_seq = boards_seq |> String.to_integer() + if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do conn |> proxy_by_board(conn.params) @@ -366,6 +367,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) threads_seq = threads_seq |> String.to_integer() + if slug_as_id < threads_seq do conn |> render(:slug_to_id, id: slug_as_id) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index c6aed799..7032badc 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -1,9 +1,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do import Ecto.Query alias EpochtalkServer.SmfRepo - alias EpochtalkServer.ProxySupervisor alias EpochtalkServerWeb.Helpers.ProxyPagination + @moduledoc """ + Helper for pulling and formatting data from SmfRepo + """ + def build_model(model_type, ids, _, _) when is_nil(model_type) or is_nil(ids) do {:ok, %{}, %{}} end @@ -13,8 +16,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_model(model_type, id, page, per_page) when is_integer(id) do - ProxySupervisor.start_link([]) - case model_type do "threads.by_board" -> build_threads_by_board(id, page, per_page) diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index 35b3b359..6769893f 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -4,7 +4,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do """ import Ecto.Query alias EpochtalkServer.SmfRepo - alias EpochtalkServer.ProxySupervisor alias EpochtalkServerWeb.Helpers.Validate @doc """ @@ -69,8 +68,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do ) def page_simple(query, count_query, page, per_page: per_page) do - ProxySupervisor.start_link([]) - options = [prefix: "public"] total_records = From 3012aaae6869ab358c0c6ddf6fd46ab95127a5f9 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 12 Jan 2024 15:23:01 -0800 Subject: [PATCH 027/231] refactor(config): Set all SmfRepo config values to pull from env variables --- config/dev.exs | 7 ++++--- config/runtime.exs | 9 +++++---- lib/epochtalk_server/application.ex | 2 ++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index b251dfcb..d161cae4 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -16,9 +16,10 @@ config :epochtalk_server, EpochtalkServer.SmfRepo, hostname: System.get_env("SMF_REPO_HOSTNAME"), database: System.get_env("SMF_REPO_DATABASE"), port: System.get_env("SMF_REPO_PORT") |> String.to_integer(), - stacktrace: true, - show_sensitive_data_on_connection_error: true, - pool_size: 10 + stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, + show_sensitive_data_on_connection_error: + System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || true, + pool_size: System.get_env("SMF_REPO_POOL_SIZE") |> String.to_integer() || 10 # For development, we disable any cache and enable # debugging and code reloading. diff --git a/config/runtime.exs b/config/runtime.exs index 5e955461..a591cc09 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -138,7 +138,7 @@ if config_env() == :prod do """ smf_repo_port = - System.get_env("SMF_REPO_PORT") |> String.to_integer() || + System.get_env("SMF_REPO_PORT") |> String.to_integer() || raise """ environment variable SMF_REPO_PORT is missing. """ @@ -149,9 +149,10 @@ if config_env() == :prod do hostname: smf_repo_hostname, database: smf_repo_database, port: smf_repo_port, - stacktrace: true, - show_sensitive_data_on_connection_error: false, - pool_size: 10 + stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, + show_sensitive_data_on_connection_error: + System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || false, + pool_size: System.get_env("SMF_REPO_POOL_SIZE") |> String.to_integer() || 10 # Configure Guardian for Runtime config :epochtalk_server, EpochtalkServer.Auth.Guardian, diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 15e929cf..57799cb7 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -19,6 +19,8 @@ defmodule EpochtalkServer.Application do {Redix, host: redix_config()[:host], name: redix_config()[:name]}, # Start the Ecto repository EpochtalkServer.Repo, + # Start the Smf repository + EpochtalkServer.SmfRepo, # Start Role Cache EpochtalkServer.Cache.Role, # Warm frontend_config variable (referenced by api controllers) From 91c85b0f9425754361d4b5459ef80291b595953a Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 12 Jan 2024 15:24:26 -0800 Subject: [PATCH 028/231] refactor(post-controller): Use application.get_env for threads_seq env variable --- lib/epochtalk_server_web/controllers/post.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 4e9cc841..bbedb60b 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -274,8 +274,10 @@ defmodule EpochtalkServerWeb.Controllers.Post do conn = case conn.private.phoenix_action do :by_thread -> - if Validate.cast(conn.params, "thread_id", :integer, required: true) < - System.get_env("THREADS_SEQ") |> String.to_integer() do + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() + + if Validate.cast(conn.params, "thread_id", :integer, required: true) < threads_seq do conn |> proxy_by_thread(conn.params) |> halt() From d6b532a68bcce8705feb041504f41412633eff13 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 17 Jan 2024 12:12:08 -1000 Subject: [PATCH 029/231] fix(config/dev.exs): set default values for PORT and POOL_SIZE fixes... (ArgumentError) errors were found at the given arguments: 1st argument: not a binary solution: set default for these values before casting to integer --- config/dev.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index d161cae4..65ab452f 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -15,11 +15,11 @@ config :epochtalk_server, EpochtalkServer.SmfRepo, password: System.get_env("SMF_REPO_PASSWORD"), hostname: System.get_env("SMF_REPO_HOSTNAME"), database: System.get_env("SMF_REPO_DATABASE"), - port: System.get_env("SMF_REPO_PORT") |> String.to_integer(), + port: System.get_env("SMF_REPO_PORT") || "3306" |> String.to_integer(), stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, show_sensitive_data_on_connection_error: System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || true, - pool_size: System.get_env("SMF_REPO_POOL_SIZE") |> String.to_integer() || 10 + pool_size: System.get_env("SMF_REPO_POOL_SIZE") || "10" |> String.to_integer() # For development, we disable any cache and enable # debugging and code reloading. From dfaf0f22454f9e6f63d15a593654019615357b70 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 17 Jan 2024 12:14:56 -1000 Subject: [PATCH 030/231] ci(Dockerfile): only add mix deps/lock before running mix deps.get no other files should interfere with the deps step --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ab78109a..a44a7ebf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,13 @@ RUN mkdir -p /app WORKDIR /app RUN mix local.hex --force RUN mix local.rebar --force -ADD . . -RUN mix deps.get # compile for production ENV MIX_ENV=prod +COPY mix.exs . +COPY mix.lock . +RUN mix deps.get +COPY . . RUN mix compile CMD until mix ecto.setup; do sleep 1; done From ec48acb988655f23b9acc29fffa41c938b89f49e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 24 Jan 2024 14:15:49 -1000 Subject: [PATCH 031/231] fix(config/runtime.exs): set default values for PORT and POOL_SIZE --- config/runtime.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index a591cc09..4a3f56b5 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -138,7 +138,7 @@ if config_env() == :prod do """ smf_repo_port = - System.get_env("SMF_REPO_PORT") |> String.to_integer() || + System.get_env("SMF_REPO_PORT") || "3306" |> String.to_integer() || raise """ environment variable SMF_REPO_PORT is missing. """ @@ -152,7 +152,7 @@ if config_env() == :prod do stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, show_sensitive_data_on_connection_error: System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || false, - pool_size: System.get_env("SMF_REPO_POOL_SIZE") |> String.to_integer() || 10 + pool_size: System.get_env("SMF_REPO_POOL_SIZE") || "10" |> String.to_integer() # Configure Guardian for Runtime config :epochtalk_server, EpochtalkServer.Auth.Guardian, From 3b4c551ef449bd1ed96f50e45d9fd5ab663a0169 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 25 Jan 2024 14:23:26 -1000 Subject: [PATCH 032/231] refactor(config/runtime): configure port and pool size inside of String.to_integer call --- config/runtime.exs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 4a3f56b5..49fdcb33 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -137,22 +137,16 @@ if config_env() == :prod do environment variable SMF_REPO_DATABASE is missing. """ - smf_repo_port = - System.get_env("SMF_REPO_PORT") || "3306" |> String.to_integer() || - raise """ - environment variable SMF_REPO_PORT is missing. - """ - config :epochtalk_server, EpochtalkServer.SmfRepo, username: smf_repo_username, password: smf_repo_password, hostname: smf_repo_hostname, database: smf_repo_database, - port: smf_repo_port, + port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, show_sensitive_data_on_connection_error: System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || false, - pool_size: System.get_env("SMF_REPO_POOL_SIZE") || "10" |> String.to_integer() + pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") # Configure Guardian for Runtime config :epochtalk_server, EpochtalkServer.Auth.Guardian, From 413ada39fad16b2296cd9853968ee0652d082b02 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 25 Jan 2024 15:36:15 -1000 Subject: [PATCH 033/231] feat(helpers/proxy_conversion): blacklist boards --- config/runtime.exs | 14 +++++++++++- .../helpers/proxy_conversion.ex | 22 ++++++++++++------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 49fdcb33..b8051ecd 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -166,11 +166,23 @@ if config_env() == :prod do # Configure Redis for Session Storage config :epochtalk_server, :redix, host: System.get_env("REDIS_HOST") || "127.0.0.1" + id_board_blacklist = + System.get_env("ID_BOARD_BLACKLIST") || + raise """ + environment variable ID_BOARD_BLACKLIST is missing. + """ + + id_board_blacklist = + id_board_blacklist + |> String.split() + |> Enum.map(&(String.to_integer(&1))) + # Configure proxy config :epochtalk_server, proxy_config: %{ threads_seq: System.get_env("THREADS_SEQ") || "6000000", - boards_seq: System.get_env("BOARDS_SEQ") || "500" + boards_seq: System.get_env("BOARDS_SEQ") || "500", + id_board_blacklist: id_board_blacklist } # Configure frontend diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 7032badc..36bbffa2 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -68,9 +68,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_boards(ids) do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", limit: ^length(ids), - where: b.id_board in ^ids, + where: b.id_board in ^ids and b.id_board not in ^id_board_blacklist, select: %{ id: b.id_board, slug: b.id_board, @@ -101,11 +102,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_threads_by_board(id, page, per_page) do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) count_query = - from t in "smf_topics", where: t.id_board == ^id, select: %{count: count(t.id_topic)} + from t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, select: %{count: count(t.id_topic)} from(t in "smf_topics", - where: t.id_board == ^id, + where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, order_by: [desc: t.id_last_msg], select: %{ id: t.id_topic, @@ -157,9 +159,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_threads(ids) do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(t in "smf_topics", limit: ^length(ids), - where: t.id_topic in ^ids, + where: t.id_topic in ^ids and t.id_board not in ^id_board_blacklist, select: %{ id: t.id_topic, slug: t.id_topic, @@ -209,9 +212,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_posts(ids) do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(m in "smf_messages", limit: ^length(ids), - where: m.id_msg in ^ids, + where: m.id_msg in ^ids and m.id_board not in ^id_board_blacklist, select: %{ id: m.id_msg, thread_id: m.id_topic, @@ -243,12 +247,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_posts_by_thread(id, page, per_page) do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) count_query = - from m in "smf_messages", where: m.id_topic == ^id, select: %{count: count(m.id_topic)} + from m in "smf_messages", where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, select: %{count: count(m.id_topic)} from(m in "smf_messages", limit: 15, - where: m.id_topic == ^id, + where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, order_by: [asc: m.posterTime], select: %{ id: m.id_msg, @@ -302,9 +307,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end defp get_post(id) do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(m in "smf_messages", limit: 1, - where: m.id_msg == ^id, + where: m.id_msg == ^id and m.id_board not in ^id_board_blacklist, select: %{ id: m.id_msg, thread_id: m.id_topic, From 93d48c14849a90cb3d8781412e0e50b361173744 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 2 Feb 2024 13:53:52 -1000 Subject: [PATCH 034/231] refactor(helpers/proxy_conversion): omit posterEmail from query --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 36bbffa2..04212d01 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -264,7 +264,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do body: m.body, updated_at: m.modifiedTime, username: m.posterName, - user_email: m.posterEmail, poster_time: m.posterTime, poster_name: m.posterName, modified_time: m.modifiedTime @@ -319,7 +318,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do title: m.subject, body: m.body, username: m.posterName, - user_email: m.posterEmail, created_at: m.posterTime, modified_time: m.modifiedTime } From 7c9c702675f7a68a5eab486b1b1aa6e468128680 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 20 Feb 2024 14:12:54 -0800 Subject: [PATCH 035/231] feat(proxy_conversion.build_threads): integrate first/last post query into thread query --- .../helpers/proxy_conversion.ex | 74 ++++++++----------- 1 file changed, 29 insertions(+), 45 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 04212d01..d0e29904 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -162,53 +162,37 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(t in "smf_topics", limit: ^length(ids), - where: t.id_topic in ^ids and t.id_board not in ^id_board_blacklist, - select: %{ - id: t.id_topic, - slug: t.id_topic, - board_id: t.id_board, - sticky: t.isSticky, - locked: t.locked, - view_count: t.numViews, - first_post_id: t.id_first_msg, - last_post_id: t.id_last_msg, - started_user_id: t.id_member_started, - updated_user_id: t.id_member_updated, - moderated: t.selfModerated, - post_count: t.numReplies - } + where: t.id_topic in ^ids and t.id_board not in ^id_board_blacklist ) + # get first and last message for thread + |> join(:left, [t], m in "smf_messages", on: t.id_first_msg == m.id_msg) + |> join(:left, [t], m in "smf_messages", on: t.id_last_msg == m.id_msg) + |> select([t, f, l], %{ + id: t.id_topic, + slug: t.id_topic, + board_id: t.id_board, + sticky: t.isSticky, + locked: t.locked, + view_count: t.numViews, + first_post_id: t.id_first_msg, + last_post_id: t.id_last_msg, + started_user_id: t.id_member_started, + updated_user_id: t.id_member_updated, + moderated: t.selfModerated, + post_count: t.numReplies, + title: f.subject, + user_id: f.id_member, + username: f.posterName, + created_at: f.posterTime * 1000, + user_deleted: false, + last_post_created_at: l.posterTime * 1000, + last_post_deleted: false, + last_post_user_id: l.id_member, + last_post_username: l.posterName, + last_post_user_deleted: false, + last_viewed: nil + }) |> SmfRepo.all() - |> case do - [] -> - {:error, "Threads not found for ids: #{ids}"} - - threads -> - threads = - Enum.reduce(threads, [], fn thread, acc -> - first_post = get_post(thread.first_post_id) - last_post = get_post(thread.last_post_id) - - thread = - thread - |> Map.put(:title, first_post.title) - |> Map.put(:user_id, first_post.user_id) - |> Map.put(:username, first_post.username) - |> Map.put(:user_deleted, false) - |> Map.put(:last_post_id, last_post.id) - |> Map.put(:last_post_created_at, last_post.created_at * 1000) - |> Map.put(:last_post_deleted, false) - |> Map.put(:last_post_user_id, last_post.user_id) - |> Map.put(:last_post_username, last_post.username) - |> Map.put(:last_post_user_deleted, false) - |> Map.put(:last_viewed, nil) - |> Map.put(:created_at, first_post.created_at * 1000) - - [thread | acc] - end) - - return_tuple(Enum.reverse(threads)) - end end def build_posts(ids) do From 135ff860270491f5d117e14e3023096fe7a9ae7c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 20 Feb 2024 14:28:19 -0800 Subject: [PATCH 036/231] feat(proxy_conversion.build_threads_by_board): integrate first/last post query into thread query --- .../helpers/proxy_conversion.ex | 74 +++++++------------ 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index d0e29904..f72dc2e4 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -108,54 +108,36 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do from(t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, - order_by: [desc: t.id_last_msg], - select: %{ - id: t.id_topic, - slug: t.id_topic, - board_id: t.id_board, - sticky: t.isSticky, - locked: t.locked, - view_count: t.numViews, - first_post_id: t.id_first_msg, - last_post_id: t.id_last_msg, - started_user_id: t.id_member_started, - updated_user_id: t.id_member_updated, - moderated: t.selfModerated, - post_count: t.numReplies - } + order_by: [desc: t.id_last_msg] ) + |> join(:left, [t], m in "smf_messages", on: t.id_first_msg == m.id_msg) + |> join(:left, [t], m in "smf_messages", on: t.id_last_msg == m.id_msg) + |> select([t, f, l], %{ + id: t.id_topic, + slug: t.id_topic, + board_id: t.id_board, + sticky: t.isSticky, + locked: t.locked, + view_count: t.numViews, + first_post_id: t.id_first_msg, + last_post_id: t.id_last_msg, + started_user_id: t.id_member_started, + updated_user_id: t.id_member_updated, + moderated: t.selfModerated, + post_count: t.numReplies, + title: f.subject, + user_id: f.id_member, + username: f.posterName, + created_at: f.posterTime * 1000, + user_deleted: false, + last_post_created_at: l.posterTime * 1000, + last_post_deleted: false, + last_post_user_id: l.id_member, + last_post_username: l.posterName, + last_post_user_deleted: false, + last_viewed: nil + }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) - |> case do - {:ok, [], _} -> - {:error, "Threads not found for board_id: #{id}"} - - {:ok, threads, data} -> - threads = - Enum.reduce(threads, [], fn thread, acc -> - first_post = get_post(thread.first_post_id) - last_post = get_post(thread.last_post_id) - - thread = - thread - |> Map.put(:title, first_post.title) - |> Map.put(:user_id, first_post.user_id) - |> Map.put(:username, first_post.username) - |> Map.put(:user_deleted, false) - |> Map.put(:last_post_id, last_post.id) - |> Map.put(:last_post_created_at, last_post.created_at * 1000) - |> Map.put(:last_post_deleted, false) - |> Map.put(:last_post_user_id, last_post.user_id) - |> Map.put(:last_post_username, last_post.username) - |> Map.put(:last_post_user_deleted, false) - |> Map.put(:last_viewed, nil) - |> Map.put(:created_at, first_post.created_at * 1000) - |> Map.put(:is_proxy, true) - - [thread | acc] - end) - - return_tuple(Enum.reverse(threads), data) - end end def build_threads(ids) do From 39fd2f6067e92275f841def3e36812067d7d87f1 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 20 Feb 2024 14:33:10 -0800 Subject: [PATCH 037/231] fix(proxy_conversion): re-add is_proxy to build_threads_by_board --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index f72dc2e4..1fb3a4a2 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -135,7 +135,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_post_user_id: l.id_member, last_post_username: l.posterName, last_post_user_deleted: false, - last_viewed: nil + last_viewed: nil, + is_proxy: true }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) end From 4cf4024b7bfe5b001184a5e81a4a05920557e13d Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 21 Feb 2024 09:03:10 -0800 Subject: [PATCH 038/231] ci(config/runtime): allow logger level definition in env --- config/runtime.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/runtime.exs b/config/runtime.exs index b8051ecd..d98f284c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -28,6 +28,15 @@ if config_env() == :prod do For example: ecto://USER:PASS@HOST/DATABASE """ + logger_level = + System.get_env("LOGGER_LEVEL") + |> case do + "DEBUG" -> :debug + _ -> :info + end + + config :logger, level: logger_level + maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: [] config :epochtalk_server, EpochtalkServer.Repo, From e11dc8bf41e711e51df4f3a0a75d617455f3846e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 23 Feb 2024 13:07:24 -0800 Subject: [PATCH 039/231] fix(helpers/proxy_conversion): use return_tuple in by_thread --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 1fb3a4a2..be1a411b 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -176,6 +176,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_viewed: nil }) |> SmfRepo.all() + |> return_tuple() end def build_posts(ids) do From 395041ed535def2d9142aa2d8ea890d033cfc6fb Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 16 May 2024 12:47:36 -1000 Subject: [PATCH 040/231] feat(porcelain): add porcelain as dependency to run external command line applications (php script) --- mix.exs | 1 + mix.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/mix.exs b/mix.exs index 8df02883..fa1b0c22 100644 --- a/mix.exs +++ b/mix.exs @@ -54,6 +54,7 @@ defmodule EpochtalkServer.MixProject do {:phoenix_ecto, "~> 4.4"}, {:phoenix_html, "~> 3.0"}, {:plug_cowboy, "~> 2.5"}, + {:porcelain, "~> 2.0"}, {:postgrex, "~> 0.17.1"}, {:redix, "~> 1.2.2"}, {:remote_ip, "~> 1.1.0"}, diff --git a/mix.lock b/mix.lock index 1651dae0..1ae3d503 100644 --- a/mix.lock +++ b/mix.lock @@ -48,6 +48,7 @@ "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"}, "plug_cowboy": {:hex, :plug_cowboy, "2.6.2", "753611b23b29231fb916b0cdd96028084b12aff57bfd7b71781bd04b1dbeb5c9", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "951ed2433df22f4c97b85fdb145d4cee561f36b74854d64c06d896d7cd2921a7"}, "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, + "porcelain": {:hex, :porcelain, "2.0.3", "2d77b17d1f21fed875b8c5ecba72a01533db2013bd2e5e62c6d286c029150fdc", [:mix], [], "hexpm", "dc996ab8fadbc09912c787c7ab8673065e50ea1a6245177b0c24569013d23620"}, "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "redix": {:hex, :redix, "1.2.4", "8d980da0800262d5e8dd718af09d549bd788b1b08651d03cbc0854f5fb35f0e6", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fb6d7327b0d4190e88e35cf68551130b2c24f6662ccc08007bee64cbd312e1c5"}, From 3c830a7e8f8b9e12910e8595af9e6833e8b747a5 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 16 May 2024 14:44:38 -1000 Subject: [PATCH 041/231] feat(bbcode-parse-route): add skeleton route for bbcode parser --- config/config.exs | 3 +++ lib/epochtalk_server_web/controllers/post.ex | 20 +++++++++++++++++++- lib/epochtalk_server_web/json/post_json.ex | 14 ++++++++++++++ lib/epochtalk_server_web/router.ex | 1 + 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index b00e0781..2058374a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -37,6 +37,9 @@ config :epochtalk_server, rate_limiting: %{} } +# Configure Porcelain +config :porcelain, driver: Porcelain.Driver.Basic + # Configure Guardian config :epochtalk_server, EpochtalkServer.Auth.Guardian, issuer: "EpochtalkServer", diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 20b78153..4298d8c4 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -363,7 +363,8 @@ defmodule EpochtalkServerWeb.Controllers.Post do """ def preview(conn, attrs) do with post_max_length <- - Map.get(Application.get_env(:epochtalk_server, :frontend_config), :post_max_length), + Map.get(Application.get_env(:epochtalk_server, :frontend_config), :post_max_length) || + Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"], body <- Validate.cast(attrs, "body", :string, required: true, max: post_max_length, min: 1), parsed_body <- Parse.markdown(body) do @@ -374,6 +375,23 @@ defmodule EpochtalkServerWeb.Controllers.Post do end end + @doc """ + Parse legacy `Post` containing bbcode using Porcelain + """ + def parse_legacy(conn, attrs) do + with post_max_length <- + Map.get(Application.get_env(:epochtalk_server, :frontend_config), :post_max_length) || + Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"], + body <- + Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1), + parsed_body <- body do + render(conn, :parse_legacy, %{parsed_body: parsed_body}) + else + _ -> + ErrorHelpers.render_json_error(conn, 400, "Error, cannot parse") + end + end + ## === Private Authorization Helper Functions === defp can_authed_user_view_deleted_posts(nil, _thread_id), do: false diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 66f99ad3..7da69472 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -39,6 +39,20 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do %{parsed_body: parsed_body} end + @doc """ + Renders `Post` data for parsed legacy `Post` data + + ## Example + iex> parsed_body = %{parsed_body: "

Hello World

"} + iex> EpochtalkServerWeb.Controllers.PostJSON.preview(parsed_body) + parsed_body + """ + def parse_legacy(%{ + parsed_body: parsed_body + }) do + %{parsed_body: parsed_body} + end + @doc """ Renders all `Post` for a particular `Thread`. """ diff --git a/lib/epochtalk_server_web/router.ex b/lib/epochtalk_server_web/router.ex index 4318a6e8..838ccb56 100644 --- a/lib/epochtalk_server_web/router.ex +++ b/lib/epochtalk_server_web/router.ex @@ -56,6 +56,7 @@ defmodule EpochtalkServerWeb.Router do post "/posts", Post, :create post "/posts/:id", Post, :update post "/preview", Post, :preview + post "/bbcode", Post, :parse_legacy get "/admin/modlog", ModerationLog, :page get "/boards/movelist", Board, :movelist end From 9b000f9b2bb600cd0abf591500b5d34c9459a515 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 17 May 2024 12:56:50 -1000 Subject: [PATCH 042/231] refactor(bbcode-parser): wip test porcelain shell attempting to run php script --- lib/epochtalk_server_web/controllers/post.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 4298d8c4..fb37f173 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -385,6 +385,10 @@ defmodule EpochtalkServerWeb.Controllers.Post do body <- Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1), parsed_body <- body do + + %Porcelain.Result{out: output, status: status} = Porcelain.shell("pwd") + IO.inspect status + IO.inspect output render(conn, :parse_legacy, %{parsed_body: parsed_body}) else _ -> From 5f647fabfcbdcf3d750229b679f978d6d43eab90 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 20 May 2024 14:42:39 -1000 Subject: [PATCH 043/231] feat(bbcode-parser): add skeleton to run php script for bbcode parser, add php to tool versions --- .tool-versions | 1 + bbcode.php | 4 ++++ lib/epochtalk_server_web/controllers/post.ex | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 bbcode.php diff --git a/.tool-versions b/.tool-versions index 4c2c3458..0f420050 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ elixir 1.14.4-otp-25 erlang 25.3.1 +php 8.3.7 diff --git a/bbcode.php b/bbcode.php new file mode 100644 index 00000000..93bbd313 --- /dev/null +++ b/bbcode.php @@ -0,0 +1,4 @@ + diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index fb37f173..4318906d 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -386,7 +386,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1), parsed_body <- body do - %Porcelain.Result{out: output, status: status} = Porcelain.shell("pwd") + %Porcelain.Result{out: output, status: status} = Porcelain.shell("php bbcode.php") IO.inspect status IO.inspect output render(conn, :parse_legacy, %{parsed_body: parsed_body}) From 4ec59278de1f1a7d06d8622d43b17409dc2e61dc Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 21 May 2024 15:57:58 -1000 Subject: [PATCH 044/231] refactor(bbcode-parser): wip port bbcode parser --- lib/epochtalk_server_web/controllers/post.ex | 5 +- parsing.php | 1832 ++++++++++++++++++ 2 files changed, 1835 insertions(+), 2 deletions(-) create mode 100644 parsing.php diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 4318906d..4fafef5f 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -386,10 +386,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1), parsed_body <- body do - %Porcelain.Result{out: output, status: status} = Porcelain.shell("php bbcode.php") + %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('Hello [br] [color=red]Hello[/color]', false);\"") IO.inspect status IO.inspect output - render(conn, :parse_legacy, %{parsed_body: parsed_body}) + IO.inspect parsed_body + render(conn, :parse_legacy, %{parsed_body: output}) else _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot parse") diff --git a/parsing.php b/parsing.php new file mode 100644 index 00000000..8cbd08ff --- /dev/null +++ b/parsing.php @@ -0,0 +1,1832 @@ + 29 && strlen($message)>500 && php_sapi_name() != 'cli') { + die(); + } + + // Never show smileys for wireless clients. More bytes, can't see it anyway :P. + if (WIRELESS) + $smileys = false; + elseif ($smileys !== null && ($smileys == '1' || $smileys == '0')) + $smileys = (bool) $smileys; + + ECHO "a"; + if (empty($modSettings['enableBBC']) && $message !== false) + { + if ($smileys === true) + parsesmileys($message); + ECHO "b"; + + return $message; + } + ECHO "c"; + + // Just in case it wasn't determined yet whether UTF-8 is enabled. + if (!isset($context['utf8'])) + $context['utf8'] = (empty($modSettings['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']) === 'UTF-8'; + + //theymos - disable links and images on pages where we don't want to send a referer to random people + $disabledsecurity=''; + if(isset($_GET['sesc'])) { + $cache_id = ''; + $disabled['img']=true; + $disabled['iurl']=true; + $disabled['url']=true; + $disabled['ftp']=true; + $disabledsecurity=' (FORUM: disabled on this page for security.)'; + } + if(isset($_GET['patrol'])) { + $cache_id = ''; + $disabled['black']=true; + $disabled['color']=true; + } + + //theymos - these tags are aways disabled + $disabled['flash'] = true; + $disabled['move'] = true; + + // Sift out the bbc for a performance improvement. + if (empty($bbc_codes) || $message === false) + { + /*if (!empty($modSettings['disabledBBC'])) + { + $temp = explode(',', strtolower($modSettings['disabledBBC'])); + + foreach ($temp as $tag) + $disabled[trim($tag)] = true; + } + + if (empty($modSettings['enableEmbeddedFlash'])) + $disabled['flash'] = true;*/ + + /* The following bbc are formatted as an array, with keys as follows: + + tag: the tag's name - should be lowercase! + + type: one of... + - (missing): [tag]parsed content[/tag] + - unparsed_equals: [tag=xyz]parsed content[/tag] + - parsed_equals: [tag=parsed data]parsed content[/tag] + - unparsed_content: [tag]unparsed content[/tag] + - closed: [tag], [tag/], [tag /] + - unparsed_commas: [tag=1,2,3]parsed content[/tag] + - unparsed_commas_content: [tag=1,2,3]unparsed content[/tag] + - unparsed_equals_content: [tag=...]unparsed content[/tag] + + parameters: an optional array of parameters, for the form + [tag abc=123]content[/tag]. The array is an associative array + where the keys are the parameter names, and the values are an + array which may contain the following: + - match: a regular expression to validate and match the value. + - quoted: true if the value should be quoted. + - validate: callback to evaluate on the data, which is $data. + - value: a string in which to replace $1 with the data. + either it or validate may be used, not both. + - optional: true if the parameter is optional. + + test: a regular expression to test immediately after the tag's + '=', ' ' or ']'. Typically, should have a \] at the end. + Optional. + + content: only available for unparsed_content, closed, + unparsed_commas_content, and unparsed_equals_content. + $1 is replaced with the content of the tag. Parameters + are repalced in the form {param}. For unparsed_commas_content, + $2, $3, ..., $n are replaced. + + before: only when content is not used, to go before any + content. For unparsed_equals, $1 is replaced with the value. + For unparsed_commas, $1, $2, ..., $n are replaced. + + after: similar to before in every way, except that it is used + when the tag is closed. + + disabled_content: used in place of content when the tag is + disabled. For closed, default is '', otherwise it is '$1' if + block_level is false, '

$1
' elsewise. + + disabled_before: used in place of before when disabled. Defaults + to '
' if block_level, '' if not. + + disabled_after: used in place of after when disabled. Defaults + to '
' if block_level, '' if not. + + block_level: set to true the tag is a "block level" tag, similar + to HTML. Block level tags cannot be nested inside tags that are + not block level, and will not be implicitly closed as easily. + One break following a block level tag may also be removed. + + trim: if set, and 'inside' whitespace after the begin tag will be + removed. If set to 'outside', whitespace after the end tag will + meet the same fate. + + validate: except when type is missing or 'closed', a callback to + validate the data as $data. Depending on the tag's type, $data + may be a string or an array of strings (corresponding to the + replacement.) + + quoted: when type is 'unparsed_equals' or 'parsed_equals' only, + may be not set, 'optional', or 'required' corresponding to if + the content may be quoted. This allows the parser to read + [tag="abc]def[esdf]"] properly. + + require_parents: an array of tag names, or not set. If set, the + enclosing tag *must* be one of the listed tags, or parsing won't + occur. + + require_children: similar to require_parents, if set children + won't be parsed if they are not in the list. + + disallow_children: similar to, but very different from, + require_children, if it is set the listed tags will not be + parsed inside the tag. + */ + + $codes = array( + array( + 'tag' => 'abbr', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'quoted' => 'optional', + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'acronym', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'quoted' => 'optional', + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'anchor', + 'type' => 'unparsed_equals', + 'test' => '[#]?([A-Za-z][A-Za-z0-9_\-]*)\]', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'b', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'black', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'blue', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'br', + 'type' => 'closed', + 'content' => '
', + ), + array( + 'tag' => 'btc', + 'type' => 'closed', + 'content' => 'BTC', + ), + array( + 'tag' => 'code', + 'type' => 'unparsed_content', + 'content' => '
' . $txt['smf238'] . ':
' . ($context['browser']['is_gecko'] ? '
$1
' : '$1') . '
', + // !!! Maybe this can be simplified? + 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { + global $context; + + if (!isset($disabled['code'])) + { + $php_parts = preg_split('~(<\?php|\?>)~', $data, -1, PREG_SPLIT_DELIM_CAPTURE); + + for ($php_i = 0, $php_n = count($php_parts); $php_i < $php_n; $php_i++) + { + // Do PHP code coloring? + if ($php_parts[$php_i] != '<?php') + continue; + + $php_string = ''; + while ($php_i + 1 < count($php_parts) && $php_parts[$php_i] != '?>') + { + $php_string .= $php_parts[$php_i]; + $php_parts[$php_i++] = ''; + } + $php_parts[$php_i] = highlight_php_code($php_string . $php_parts[$php_i]); + } + + // Fix the PHP code stuff... + $data = str_replace("
\t
", "\t", implode('', $php_parts)); + + // Older browsers are annoying, aren't they? + if ($context['browser']['is_ie4'] || $context['browser']['is_ie5'] || $context['browser']['is_ie5.5']) + $data = str_replace("\t", "
\t
", $data); + elseif (!$context['browser']['is_gecko']) + $data = str_replace("\t", "\t", $data); + }}, + 'block_level' => true, + ), + array( + 'tag' => 'code', + 'type' => 'unparsed_equals_content', + 'content' => '
' . $txt['smf238'] . ': ($2)
' . ($context['browser']['is_gecko'] ? '
$1
' : '$1') . '
', + // !!! Maybe this can be simplified? + 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { + global $context; + + if (!isset($disabled['code'])) + { + $php_parts = preg_split('~(<\?php|\?>)~', $data[0], -1, PREG_SPLIT_DELIM_CAPTURE); + + for ($php_i = 0, $php_n = count($php_parts); $php_i < $php_n; $php_i++) + { + // Do PHP code coloring? + if ($php_parts[$php_i] != '<?php') + continue; + + $php_string = ''; + while ($php_i + 1 < count($php_parts) && $php_parts[$php_i] != '?>') + { + $php_string .= $php_parts[$php_i]; + $php_parts[$php_i++] = ''; + } + $php_parts[$php_i] = highlight_php_code($php_string . $php_parts[$php_i]); + } + + // Fix the PHP code stuff... + $data[0] = str_replace("
\t
", "\t", implode('', $php_parts)); + + // Older browsers are annoying, aren't they? + if ($context['browser']['is_ie4'] || $context['browser']['is_ie5'] || $context['browser']['is_ie5.5']) + $data = str_replace("\t", "
\t
", $data); + elseif (!$context['browser']['is_gecko']) + $data = str_replace("\t", "\t", $data); + }}, + 'block_level' => true, + ), + array( + 'tag' => 'center', + 'before' => '
', + 'after' => '
', + 'block_level' => true, + ), + array( + 'tag' => 'color', + 'type' => 'unparsed_equals', + 'test' => '(#[\da-fA-F]{3}|#[\da-fA-F]{6}|[A-Za-z]{1,12})\]', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'email', + 'type' => 'unparsed_content', + 'content' => '$1', + // !!! Should this respect guest_hideContacts? + 'validate' => function(&$tag, &$data, $disabled) {$data = strtr($data, array('
' => ''));}, + ), + array( + 'tag' => 'email', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + // !!! Should this respect guest_hideContacts? + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'ftp', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
' => '')); + if (strpos($data, 'ftp://') !== 0 && strpos($data, 'ftps://') !== 0) + $data = 'ftp://' . $data; + }, + ), + array( + 'tag' => 'ftp', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'validate' => function(&$tag, &$data, $disabled) { + if (strpos($data, 'ftp://') !== 0 && strpos($data, 'ftps://') !== 0) + $data = 'ftp://' . $data; + }, + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'font', + 'type' => 'unparsed_equals', + 'test' => '[A-Za-z0-9_,\-\s]+?\]', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'flash', + 'type' => 'unparsed_commas_content', + 'test' => '\d+,\d+\]', + 'content' => ($context['browser']['is_ie'] && !$context['browser']['is_mac_ie'] ? '<a href="$1">$1</a>' : '<a href="$1">$1</a>'), + 'validate' => function(&$tag, &$data, $disabled) { + if (isset($disabled['url'])) + $tag['content'] = '$1'; + elseif (strpos($data[0], 'http://') !== 0 && strpos($data[0], 'https://') !== 0) + $data[0] = 'http://' . $data[0]; + }, + 'disabled_content' => $disabledsecurity ? '$1': '$1', + ), + array( + 'tag' => 'green', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'glow', + 'type' => 'unparsed_commas', + 'test' => '[#0-9a-zA-Z\-]{3,12},([012]\d{1,2}|\d{1,2})(,[^]]+)?\]', + 'before' => $context['browser']['is_ie'] ? '
' : '', + 'after' => $context['browser']['is_ie'] ? '
' : '', + ), + array( + 'tag' => 'hr', + 'type' => 'closed', + 'content' => '
', + 'block_level' => true, + ), + array( + 'tag' => 'html', + 'type' => 'unparsed_content', + 'content' => '$1', + 'block_level' => true, + 'disabled_content' => '$1', + ), + array( + 'tag' => 'img', + 'type' => 'unparsed_content', + 'parameters' => array( + 'alt' => array('optional' => true), + 'width' => array('optional' => true, 'value' => ' width="$1"', 'match' => '(\d{1,4})'), + 'height' => array('optional' => true, 'value' => ' height="$1"', 'match' => '(\d{1,4})'), + ), + 'content' => '{alt}', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0) + $data = 'http://' . $data; + if(!isset($disabled['img'])) + $data = proxyurl($data); + }, + 'disabled_content' => $disabledsecurity ? ('($1)'.$disabledsecurity) : '$1', + ), + array( + 'tag' => 'img', + 'type' => 'unparsed_content', + 'content' => '', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0) + $data = 'http://' . $data; + if(!isset($disabled['img'])) + $data = proxyurl($data); + }, + 'disabled_content' => $disabledsecurity ? ('($1)'.$disabledsecurity) : '$1', + ), + array( + 'tag' => 'i', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'iurl', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + ), + array( + 'tag' => 'iurl', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'validate' => function(&$tag, &$data, $disabled) { + if (substr($data, 0, 1) == '#') + $data = '#post_' . substr($data, 1); + elseif (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'li', + 'before' => '
  • ', + 'after' => '
    • ', + 'trim' => 'outside', + 'require_parents' => array('list'), + 'block_level' => true, + 'disabled_before' => '', + 'disabled_after' => '
    ', + ), + array( + 'tag' => 'list', + 'before' => '
      ', + 'after' => '
    ', + 'trim' => 'inside', + 'require_children' => array('li'), + 'block_level' => true, + ), + array( + 'tag' => 'list', + 'parameters' => array( + 'type' => array('match' => '(none|disc|circle|square|decimal|decimal-leading-zero|lower-roman|upper-roman|lower-alpha|upper-alpha|lower-greek|lower-latin|upper-latin|hebrew|armenian|georgian|cjk-ideographic|hiragana|katakana|hiragana-iroha|katakana-iroha)'), + ), + 'before' => '
      ', + 'after' => '
    ', + 'trim' => 'inside', + 'require_children' => array('li'), + 'block_level' => true, + ), + array( + 'tag' => 'left', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'ltr', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'me', + 'type' => 'unparsed_equals', + 'before' => '
    * $1 ', + 'after' => '
    ', + 'quoted' => 'optional', + 'block_level' => true, + 'disabled_before' => '/me ', + 'disabled_after' => '
    ', + ), + array( + 'tag' => 'move', + 'before' => '', + 'after' => '', + 'block_level' => true, + ), + array( + 'tag' => 'nbsp', + 'type' => 'closed', + 'content' => ' ', + ), + array( + 'tag' => 'nobbc', + 'type' => 'unparsed_content', + 'content' => '$1', + ), + array( + 'tag' => 'pre', + 'before' => '
    ',
+				'after' => '
    ', + ), + array( + 'tag' => 'php', + 'type' => 'unparsed_content', + 'content' => '
    $1
    ', + 'validate' => isset($disabled['php']) ? null : function(&$tag, &$data, $disabled) { + if (!isset($disabled['php'])) + { + $add_begin = substr(trim($data), 0, 5) != '<?'; + $data = highlight_php_code($add_begin ? '<?php ' . $data . '?>' : $data); + if ($add_begin) + $data = preg_replace(array('~^(.+?)<\?.{0,40}?php( |\s)~', '~\?>((?:)*)$~'), '$1', $data, 2); + }}, + 'block_level' => true, + 'disabled_content' => '$1', + ), + array( + 'tag' => 'quote', + 'before' => '
    ' . $txt['smf240'] . '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'parameters' => array( + 'author' => array('match' => '(.{1,192}?)', 'quoted' => true, 'validate' => 'parse_bbc'), + ), + 'before' => '
    ' . $txt['smf239'] . ': {author}
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'type' => 'parsed_equals', + 'before' => '
    ' . $txt['smf239'] . ': $1
    ', + 'after' => '
    ', + 'quoted' => 'optional', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'parameters' => array( + 'author' => array('match' => '([^<>]{1,192}?)'), + 'link' => array('match' => '(?:board=\d+;)?((?:topic|threadid)=[\dmsg#\./]{1,40}(?:;start=[\dmsg#\./]{1,40})?|action=profile;u=\d+)'), + 'date' => array('match' => '(\d+)', 'validate' => 'timeformat'), + ), + 'before' => '
    ' . $txt['smf239'] . ': {author} ' . $txt[176] . ' {date}
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'parameters' => array( + 'author' => array('match' => '(.{1,192}?)', 'validate' => 'parse_bbc'), + ), + 'before' => '
    ' . $txt['smf239'] . ': {author}
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'right', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'red', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'rtl', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 's', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'size', + 'type' => 'unparsed_equals', + 'test' => '([1-9][\d]?p[xt]|(?:x-)?small(?:er)?|(?:x-)?large[r]?)\]', + // !!! line-height + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'size', + 'type' => 'unparsed_equals', + 'test' => '[1-9]\]', + // !!! line-height + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'sub', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'sup', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'shadow', + 'type' => 'unparsed_commas', + 'test' => '[#0-9a-zA-Z\-]{3,12},(left|right|top|bottom|[0123]\d{0,2})\]', + 'before' => $context['browser']['is_ie'] ? '' : '', + 'after' => '', + 'validate' => $context['browser']['is_ie'] ? function(&$tag, &$data, $disabled) { + if ($data[1] == 'left') + $data[1] = 270; + elseif ($data[1] == 'right') + $data[1] = 90; + elseif ($data[1] == 'top') + $data[1] = 0; + elseif ($data[1] == 'bottom') + $data[1] = 180; + else + $data[1] = (int) $data[1];} : function(&$tag, &$data, $disabled) { + if ($data[1] == 'top' || (is_numeric($data[1]) && $data[1] < 50)) + return '0 -2px'; + elseif ($data[1] == 'right' || (is_numeric($data[1]) && $data[1] < 100)) + return '2px 0'; + elseif ($data[1] == 'bottom' || (is_numeric($data[1]) && $data[1] < 190)) + return '0 2px'; + elseif ($data[1] == 'left' || (is_numeric($data[1]) && $data[1] < 280)) + return '-2px 0'; + else + return '0 0';}, + ), + array( + 'tag' => 'time', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + if (is_numeric($data)) + $data = timeformat($data); + else + $tag['content'] = '[time]$1[/time]';}, + ), + array( + 'tag' => 'tt', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'table', + 'before' => '', + 'after' => '
    ', + 'trim' => 'inside', + 'require_children' => array('tr'), + 'block_level' => true, + ), + array( + 'tag' => 'tr', + 'before' => '', + 'after' => '', + 'require_parents' => array('table'), + 'require_children' => array('td'), + 'trim' => 'both', + 'block_level' => true, + 'disabled_before' => '', + 'disabled_after' => '', + ), + array( + 'tag' => 'td', + 'before' => '', + 'after' => '', + 'require_parents' => array('tr'), + 'trim' => 'outside', + 'block_level' => true, + 'disabled_before' => '', + 'disabled_after' => '', + ), + array( + 'tag' => 'url', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
    ' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + ), + array( + 'tag' => 'url', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'validate' => function(&$tag, &$data, $disabled) { + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'u', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'white', + 'before' => '', + 'after' => '', + ), + ); + + // This is mainly for the bbc manager, so it's easy to add tags above. Custom BBC should be added above this line. + if ($message === false) + return $codes; + + // So the parser won't skip them. + $itemcodes = array( + '*' => '', + '@' => 'disc', + '+' => 'square', + 'x' => 'square', + '#' => 'square', + 'o' => 'circle', + 'O' => 'circle', + '0' => 'circle', + ); + if (!isset($disabled['li']) && !isset($disabled['list'])) + { + foreach ($itemcodes as $c => $dummy) + $bbc_codes[$c] = array(); + } + + // Inside these tags autolink is not recommendable. + $no_autolink_tags = array( + 'url', + 'iurl', + 'ftp', + 'email', + ); + + // Shhhh! + if (!isset($disabled['color'])) + { + $codes[] = array( + 'tag' => 'chrissy', + 'before' => '', + 'after' => ' :-*', + ); + $codes[] = array( + 'tag' => 'kissy', + 'before' => '', + 'after' => ' :-*', + ); + } + + foreach ($codes as $c) + $bbc_codes[substr($c['tag'], 0, 1)][] = $c; + $codes = null; + } + + // Shall we take the time to cache this? + if ($cache_id != '' && !empty($modSettings['cache_enable']) && (($modSettings['cache_enable'] >= 2 && strlen($message) > 1000) || strlen($message) > 2400)) + { + // It's likely this will change if the message is modified. + $cache_key = 'parse:' . $cache_id . '-' . md5(md5($message) . '-' . $smileys . (empty($disabled) ? '' : implode(',', array_keys($disabled))) . safe_serialize($context['browser']) . $txt['lang_locale'] . $user_info['time_offset'] . $user_info['time_format']); + + if (($temp = cache_get_data($cache_key, 600)) != null) + return $temp; + + $cache_t = microtime(); + } + + if ($smileys === 'print') + { + // [glow], [shadow], and [move] can't really be printed. + $disabled['glow'] = true; + $disabled['shadow'] = true; + $disabled['move'] = true; + + // Colors can't well be displayed... supposed to be black and white. + $disabled['color'] = true; + $disabled['black'] = true; + $disabled['blue'] = true; + $disabled['white'] = true; + $disabled['red'] = true; + $disabled['green'] = true; + $disabled['me'] = true; + + // Color coding doesn't make sense. + $disabled['php'] = true; + + // Links are useless on paper... just show the link. + $disabled['ftp'] = true; + $disabled['url'] = true; + $disabled['iurl'] = true; + $disabled['email'] = true; + $disabled['flash'] = true; + + // !!! Change maybe? + if (!isset($_GET['images'])) + $disabled['img'] = true; + + // !!! Interface/setting to add more? + } + + if($local_disable) + foreach($local_disable as $d) + $disabled[$d] = true; + + $open_tags = array(); + $message = strtr($message, array("\n" => '
    ')); + + // The non-breaking-space looks a bit different each time. + $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{C2A0}' : chr(0xC2) . chr(0xA0)) : '\xA0'; + + $pos = -1; + while ($pos !== false) + { + // theymos - prevent various infinite loops + if($pos>90000) { + if(!isset($loopcount)) + $loopcount=0; + $loopcount++; + if($loopcount > 500) + return 'INVALID BBCODE: loop, probably unclosed tags'; + } + + $last_pos = isset($last_pos) ? max($pos, $last_pos) : $pos; + $pos = strpos($message, '[', $pos + 1); + + // Failsafe. + if ($pos === false || $last_pos > $pos) + $pos = strlen($message) + 1; + + // Can't have a one letter smiley, URL, or email! (sorry.) + if ($last_pos < $pos - 1) + { + // We want to eat one less, and one more, character (for smileys.) + $last_pos = max($last_pos - 1, 0); + $data = substr($message, $last_pos, $pos - $last_pos + 1); + + // Take care of some HTML! + if (!empty($modSettings['enablePostHTML']) && strpos($data, '<') !== false) + { + $data = preg_replace('~<a\s+href=((?:")?)((?:https?://|ftps?://|mailto:|bitcoin:)\S+?)\\1>~i', '[url=$2]', $data); + $data = preg_replace('~</a>~i', '[/url]', $data); + + //
    should be empty. + $empty_tags = array('br', 'hr'); + foreach ($empty_tags as $tag) + $data = str_replace(array('<' . $tag . '>', '<' . $tag . '/>', '<' . $tag . ' />'), '[' . $tag . ' /]', $data); + + // b, u, i, s, pre... basic tags. + $closable_tags = array('b', 'u', 'i', 's', 'em', 'ins', 'del', 'pre', 'blockquote'); + foreach ($closable_tags as $tag) + { + $diff = substr_count($data, '<' . $tag . '>') - substr_count($data, '</' . $tag . '>'); + $data = strtr($data, array('<' . $tag . '>' => '<' . $tag . '>', '</' . $tag . '>' => '')); + + if ($diff > 0) + $data .= str_repeat('', $diff); + } + + // Do - with security... action= -> action-. + preg_match_all('~<img\s+src=((?:")?)((?:https?://|ftps?://)\S+?)\\1(?:\s+alt=(".*?"|\S*?))?(?:\s?/)?>~i', $data, $matches, PREG_PATTERN_ORDER); + if (!empty($matches[0])) + { + $replaces = array(); + foreach ($matches[2] as $match => $imgtag) + { + $alt = empty($matches[3][$match]) ? '' : ' alt=' . preg_replace('~^"|"$~', '', $matches[3][$match]); + + // Remove action= from the URL - no funny business, now. + if (preg_match('~action(=|%3d)(?!dlattach)~i', $imgtag) != 0) + $imgtag = preg_replace('~action(=|%3d)(?!dlattach)~i', 'action-', $imgtag); + + // Check if the image is larger than allowed. + if (!empty($modSettings['max_image_width']) && !empty($modSettings['max_image_height'])) + { + list ($width, $height) = url_image_size($imgtag); + + if (!empty($modSettings['max_image_width']) && $width > $modSettings['max_image_width']) + { + $height = (int) (($modSettings['max_image_width'] * $height) / $width); + $width = $modSettings['max_image_width']; + } + + if (!empty($modSettings['max_image_height']) && $height > $modSettings['max_image_height']) + { + $width = (int) (($modSettings['max_image_height'] * $width) / $height); + $height = $modSettings['max_image_height']; + } + + // Set the new image tag. + $replaces[$matches[0][$match]] = '[img width=' . $width . ' height=' . $height . $alt . ']' . $imgtag . '[/img]'; + } + else + $replaces[$matches[0][$match]] = '[img' . $alt . ']' . $imgtag . '[/img]'; + } + + $data = strtr($data, $replaces); + } + } + + if (!empty($modSettings['autoLinkUrls'])) + { + // Are we inside tags that should be auto linked? + $no_autolink_area = false; + if (!empty($open_tags)) + { + foreach ($open_tags as $open_tag) + if (in_array($open_tag['tag'], $no_autolink_tags)) + $no_autolink_area = true; + } + + // Don't go backwards. + //!!! Don't think is the real solution.... + $lastAutoPos = isset($lastAutoPos) ? $lastAutoPos : 0; + if ($pos < $lastAutoPos) + $no_autolink_area = true; + $lastAutoPos = $pos; + + if (!$no_autolink_area) + { + // Parse any URLs.... have to get rid of the @ problems some things cause... stupid email addresses. + if (!isset($disabled['url']) && (strpos($data, '://') !== false || strpos($data, 'www.') !== false || strpos($data,'bitcoin:') !==false)) + { + // Switch out quotes really quick because they can cause problems. + $data = strtr($data, array(''' => '\'', ' ' => $context['utf8'] ? "\xC2\xA0" : "\xA0", '"' => '>">', '"' => '<"<', '<' => '\.(;\'"]|' . $nbsp . '|^)((?:http|https|ftp|ftps)://[\w\-_%@:|]+(?:\.[\w\-_%]+)*(?::\d+)?(?:/[\w\-_\~%\.@,\?&;=#(){}+:\'\\\\]*)*[/\w\-_\~%@\?;=#}\\\\])~i', + '~(?<=[\s>(;\'<]|' . $nbsp . '|^)(www(?:\.[\w\-_]+)+(?::\d+)?(?:/[\w\-_\~%\.@,\?&;=#(){}+:\'\\\\]*)*[/\w\-_\~%@\?;=#}\\\\])~i', + '~bitcoin:([-A-Za-z0-9._:/?#!%@$()*+,;=]{25,})~i' + ), array( + '[url]$1[/url]', + '[url=http://$1]$1[/url]', + '[url]bitcoin:$1[/url]' + ), $data))) + $data = $result; + + $data = strtr($data, array('\'' => ''', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ', '>">' => '"', '<"<' => '"', ' '<')); + } + + // Next, emails... + if (!isset($disabled['email']) && strpos($data, '@') !== false) + { + $data = preg_replace('~(?<=[\?\s' . $non_breaking_space . '\[\]()*\\\;>]|^)([\w\-\.]{1,80}@[\w\-]+\.[\w\-\.]+[\w\-])(?=[?,\s' . $non_breaking_space . '\[\]()*\\\]|$|
    | |>|<|"|'|\.(?:\.|;| |\s|$|
    ))~' . ($context['utf8'] ? 'u' : ''), '[email]$1[/email]', $data); + $data = preg_replace('~(?<=
    )([\w\-\.]{1,80}@[\w\-]+\.[\w\-\.]+[\w\-])(?=[?\.,;\s' . $non_breaking_space . '\[\]()*\\\]|$|
    | |>|<|"|')~' . ($context['utf8'] ? 'u' : ''), '[email]$1[/email]', $data); + // theymos - infinite loop + if($pos > 1000 && strpos(substr($message, $pos-100), '[email][email][email][email]') !== false) + return 'INVALID BBCODE: loop, probably unclosed tags (2)'; + } + } + } + + $data = strtr($data, array("\t" => '   ')); + + if (!empty($modSettings['fixLongWords']) && $modSettings['fixLongWords'] > 5) + { + // This is SADLY and INCREDIBLY browser dependent. + if ($context['browser']['is_gecko'] || $context['browser']['is_konqueror']) + $breaker = ' '; + // Opera... + elseif ($context['browser']['is_opera']) + $breaker = ' '; + // Internet Explorer... + else + $breaker = ' '; + + // PCRE will not be happy if we don't give it a short. + $modSettings['fixLongWords'] = (int) min(65535, $modSettings['fixLongWords']); + + // The idea is, find words xx long, and then replace them with xx + space + more. + if (strlen($data) > $modSettings['fixLongWords']) + { + // This is done in a roundabout way because $breaker has "long words" :P. + $data = strtr($data, array($breaker => '< >', ' ' => $context['utf8'] ? "\xC2\xA0" : "\xA0")); + $data = preg_replace_callback( + '~(?<=[>;:!? ' . $non_breaking_space . '\]()]|^)([\w' . ($context['utf8'] ? '\pL' : '') . '\.]{' . $modSettings['fixLongWords'] . ',})~' . ($context['utf8'] ? 'u' : ''), + 'word_break__preg_callback', + $data); + $data = strtr($data, array('< >' => $breaker, $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); + } + } + + // Do any smileys! + if ($smileys === true) + parsesmileys($data); + + // If it wasn't changed, no copying or other boring stuff has to happen! + if ($data != substr($message, $last_pos, $pos - $last_pos + 1)) + { + $message = substr($message, 0, $last_pos) . $data . substr($message, $pos + 1); + + // Since we changed it, look again incase we added or removed a tag. But we don't want to skip any. + $old_pos = strlen($data) + $last_pos - 1; + $pos = strpos($message, '[', $last_pos); + $pos = $pos === false ? $old_pos : min($pos, $old_pos); + } + } + + // Are we there yet? Are we there yet? + if ($pos >= strlen($message) - 1) + break; + + $tags = strtolower(substr($message, $pos + 1, 1)); + + if ($tags == '/' && !empty($open_tags)) + { + $pos2 = strpos($message, ']', $pos + 1); + if ($pos2 == $pos + 2) + continue; + $look_for = strtolower(substr($message, $pos + 2, $pos2 - $pos - 2)); + + $to_close = array(); + $block_level = null; + do + { + $tag = array_pop($open_tags); + if (!$tag) + break; + + if (!empty($tag['block_level'])) + { + // Only find out if we need to. + if ($block_level === false) + { + array_push($open_tags, $tag); + break; + } + + // The idea is, if we are LOOKING for a block level tag, we can close them on the way. + if (strlen($look_for) > 0 && isset($bbc_codes[$look_for[0]])) + { + foreach ($bbc_codes[$look_for[0]] as $temp) + if ($temp['tag'] == $look_for) + { + $block_level = !empty($temp['block_level']); + break; + } + } + + if ($block_level !== true) + { + $block_level = false; + array_push($open_tags, $tag); + break; + } + } + + $to_close[] = $tag; + } + while ($tag['tag'] != $look_for); + + // Did we just eat through everything and not find it? + if ((empty($open_tags) && (empty($tag) || $tag['tag'] != $look_for))) + { + $tablewarn = false; + for($cc=count($to_close)-1;$cc>=0;$cc--) { + if($to_close[$cc]['tag'] == 'table') + $tablewarn = true; + if($tablewarn && (in_array($to_close[$cc]['tag'], array('td', 'tr')))) + return 'INVALID BBCODE: close of unopened tag in table (2)'; + } + unset($cc, $tablewarn); + $open_tags = $to_close; + continue; + } + elseif (!empty($to_close) && $tag['tag'] != $look_for) + { + if ($block_level === null && isset($look_for[0], $bbc_codes[$look_for[0]])) + { + foreach ($bbc_codes[$look_for[0]] as $temp) + if ($temp['tag'] == $look_for) + { + $block_level = !empty($temp['block_level']); + break; + } + } + + // We're not looking for a block level tag (or maybe even a tag that exists...) + if (!$block_level) + { + foreach ($to_close as $tag) + array_push($open_tags, $tag); + continue; + } + } + + foreach ($to_close as $tag) + { + $message = substr($message, 0, $pos) . $tag['after'] . substr($message, $pos2 + 1); + $pos += strlen($tag['after']); + $pos2 = $pos - 1; + + // See the comment at the end of the big loop - just eating whitespace ;). + if (!empty($tag['block_level']) && substr($message, $pos, 6) == '
    ') + $message = substr($message, 0, $pos) . substr($message, $pos + 6); + if (!empty($tag['trim']) && $tag['trim'] != 'inside' && preg_match('~(
    | |\s)*~', substr($message, $pos), $matches) != 0) + $message = substr($message, 0, $pos) . substr($message, $pos + strlen($matches[0])); + } + + if (!empty($to_close)) + { + $to_close = array(); + $pos--; + } + + continue; + } + + // No tags for this character, so just keep going (fastest possible course.) + if (!isset($bbc_codes[$tags])) + continue; + + $inside = empty($open_tags) ? null : $open_tags[count($open_tags) - 1]; + $tag = null; + foreach ($bbc_codes[$tags] as $possible) + { + // Not a match? + if (strtolower(substr($message, $pos + 1, strlen($possible['tag']))) != $possible['tag']) + continue; + + $next_c = substr($message, $pos + 1 + strlen($possible['tag']), 1); + + // A test validation? + if (isset($possible['test']) && preg_match('~^' . $possible['test'] . '~', substr($message, $pos + 1 + strlen($possible['tag']) + 1)) == 0) + continue; + // Do we want parameters? + elseif (!empty($possible['parameters'])) + { + if ($next_c != ' ') + continue; + } + elseif (isset($possible['type'])) + { + // Do we need an equal sign? + if (in_array($possible['type'], array('unparsed_equals', 'unparsed_commas', 'unparsed_commas_content', 'unparsed_equals_content', 'parsed_equals')) && $next_c != '=') + continue; + // Maybe we just want a /... + if ($possible['type'] == 'closed' && $next_c != ']' && substr($message, $pos + 1 + strlen($possible['tag']), 2) != '/]' && substr($message, $pos + 1 + strlen($possible['tag']), 3) != ' /]') + continue; + // An immediate ]? + if ($possible['type'] == 'unparsed_content' && $next_c != ']') + continue; + } + // No type means 'parsed_content', which demands an immediate ] without parameters! + elseif ($next_c != ']') + continue; + + // Check allowed tree? + if (isset($possible['require_parents']) && ($inside === null || !in_array($inside['tag'], $possible['require_parents']))) + continue; + elseif (isset($inside['require_children']) && !in_array($possible['tag'], $inside['require_children'])) + continue; + // If this is in the list of disallowed child tags, don't parse it. + elseif (isset($inside['disallow_children']) && in_array($possible['tag'], $inside['disallow_children'])) + continue; + + $pos1 = $pos + 1 + strlen($possible['tag']) + 1; + + // This is long, but it makes things much easier and cleaner. + if (!empty($possible['parameters'])) + { + $preg = array(); + foreach ($possible['parameters'] as $p => $info) + $preg[] = '(\s+' . $p . '=' . (empty($info['quoted']) ? '' : '"') . (isset($info['match']) ? $info['match'] : '(.+?)') . (empty($info['quoted']) ? '' : '"') . ')' . (empty($info['optional']) ? '' : '?'); + + // Okay, this may look ugly and it is, but it's not going to happen much and it is the best way of allowing any order of parameters but still parsing them right. + $match = false; + $orders = permute($preg); + foreach ($orders as $p) + if (preg_match('~^' . implode('', $p) . '\]~i', substr($message, $pos1 - 1), $matches) != 0) + { + $match = true; + break; + } + + // Didn't match our parameter list, try the next possible. + if (!$match) + continue; + + $params = array(); + for ($i = 1, $n = count($matches); $i < $n; $i += 2) + { + $key = strtok(ltrim($matches[$i]), '='); + if (isset($possible['parameters'][$key]['value'])) + $params['{' . $key . '}'] = strtr($possible['parameters'][$key]['value'], array('$1' => $matches[$i + 1])); + elseif (isset($possible['parameters'][$key]['validate'])) + $params['{' . $key . '}'] = $possible['parameters'][$key]['validate']($matches[$i + 1]); + else + $params['{' . $key . '}'] = $matches[$i + 1]; + + // Just to make sure: replace any $ or { so they can't interpolate wrongly. + $params['{' . $key . '}'] = strtr($params['{' . $key . '}'], array('$' => '$', '{' => '{')); + } + + foreach ($possible['parameters'] as $p => $info) + { + if (!isset($params['{' . $p . '}'])) + $params['{' . $p . '}'] = ''; + } + + $tag = $possible; + + // Put the parameters into the string. + if (isset($tag['before'])) + $tag['before'] = strtr($tag['before'], $params); + if (isset($tag['after'])) + $tag['after'] = strtr($tag['after'], $params); + if (isset($tag['content'])) + $tag['content'] = strtr($tag['content'], $params); + + $pos1 += strlen($matches[0]) - 1; + } + else + $tag = $possible; + break; + } + + // Item codes are complicated buggers... they are implicit [li]s and can make [list]s! + if ($smileys !== false && $tag === null && isset($itemcodes[substr($message, $pos + 1, 1)]) && substr($message, $pos + 2, 1) == ']' && !isset($disabled['list']) && !isset($disabled['li'])) + { + if (substr($message, $pos + 1, 1) == '0' && !in_array(substr($message, $pos - 1, 1), array(';', ' ', "\t", '>'))) + continue; + $tag = $itemcodes[substr($message, $pos + 1, 1)]; + + // First let's set up the tree: it needs to be in a list, or after an li. + if ($inside === null || ($inside['tag'] != 'list' && $inside['tag'] != 'li')) + { + $open_tags[] = array( + 'tag' => 'list', + 'after' => '', + 'block_level' => true, + 'require_children' => array('li'), + 'disallow_children' => isset($inside['disallow_children']) ? $inside['disallow_children'] : null, + ); + $code = '
      '; + } + // We're in a list item already: another itemcode? Close it first. + elseif ($inside['tag'] == 'li') + { + array_pop($open_tags); + $code = ''; + } + else + $code = ''; + + // Now we open a new tag. + $open_tags[] = array( + 'tag' => 'li', + 'after' => '', + 'trim' => 'outside', + 'block_level' => true, + 'disallow_children' => isset($inside['disallow_children']) ? $inside['disallow_children'] : null, + ); + + // First, open the tag... + $code .= ''; + $message = substr($message, 0, $pos) . $code . substr($message, $pos + 3); + $pos += strlen($code) - 1; + + // Next, find the next break (if any.) If there's more itemcode after it, keep it going - otherwise close! + $pos2 = strpos($message, '
      ', $pos); + $pos3 = strpos($message, '[/', $pos); + if ($pos2 !== false && ($pos2 <= $pos3 || $pos3 === false)) + { + preg_match('~^(
      | |\s|\[)+~', substr($message, $pos2 + 6), $matches); + $message = substr($message, 0, $pos2) . (!empty($matches[0]) && substr($matches[0], -1) == '[' ? '[/li]' : '[/li][/list]') . substr($message, $pos2); + + $open_tags[count($open_tags) - 2]['after'] = '
    '; + } + // Tell the [list] that it needs to close specially. + else + { + if(count($open_tags)<2) { + return 'INVALID BBCODE: messed-up itemcodes'; + } + // Move the li over, because we're not sure what we'll hit. + $open_tags[count($open_tags) - 1]['after'] = ''; + $open_tags[count($open_tags) - 2]['after'] = ''; + } + + continue; + } + + // Implicitly close lists and tables if something other than what's required is in them. This is needed for itemcode. + if ($tag === null && $inside !== null && !empty($inside['require_children'])) + { + array_pop($open_tags); + + $message = substr($message, 0, $pos) . $inside['after'] . substr($message, $pos); + $pos += strlen($inside['after']) - 1; + } + + // No tag? Keep looking, then. Silly people using brackets without actual tags. + if ($tag === null) + continue; + + // Propagate the list to the child (so wrapping the disallowed tag won't work either.) + if (isset($inside['disallow_children'])) + $tag['disallow_children'] = isset($tag['disallow_children']) ? array_unique(array_merge($tag['disallow_children'], $inside['disallow_children'])) : $inside['disallow_children']; + + // Is this tag disabled? + if (isset($disabled[$tag['tag']])) + { + if (!isset($tag['disabled_before']) && !isset($tag['disabled_after']) && !isset($tag['disabled_content'])) + { + $tag['before'] = !empty($tag['block_level']) ? '
    ' : ''; + $tag['after'] = !empty($tag['block_level']) ? '
    ' : ''; + $tag['content'] = isset($tag['type']) && $tag['type'] == 'closed' ? '' : (!empty($tag['block_level']) ? '
    $1
    ' : '$1'); + } + elseif (isset($tag['disabled_before']) || isset($tag['disabled_after'])) + { + $tag['before'] = isset($tag['disabled_before']) ? $tag['disabled_before'] : (!empty($tag['block_level']) ? '
    ' : ''); + $tag['after'] = isset($tag['disabled_after']) ? $tag['disabled_after'] : (!empty($tag['block_level']) ? '
    ' : ''); + } + else + $tag['content'] = $tag['disabled_content']; + } + + // The only special case is 'html', which doesn't need to close things. + if (!empty($tag['block_level']) && $tag['tag'] != 'html' && empty($inside['block_level'])) + { + $n = count($open_tags) - 1; + while (empty($open_tags[$n]['block_level']) && $n >= 0) + $n--; + + // Close all the non block level tags so this tag isn't surrounded by them. + for ($i = count($open_tags) - 1; $i > $n; $i--) + { + $message = substr($message, 0, $pos) . $open_tags[$i]['after'] . substr($message, $pos); + $pos += strlen($open_tags[$i]['after']); + $pos1 += strlen($open_tags[$i]['after']); + + // Trim or eat trailing stuff... see comment at the end of the big loop. + if (!empty($open_tags[$i]['block_level']) && substr($message, $pos, 6) == '
    ') + $message = substr($message, 0, $pos) . substr($message, $pos + 6); + if (!empty($open_tags[$i]['trim']) && $tag['trim'] != 'inside' && preg_match('~(
    | |\s)*~', substr($message, $pos), $matches) != 0) + $message = substr($message, 0, $pos) . substr($message, $pos + strlen($matches[0])); + + array_pop($open_tags); + } + } + + // No type means 'parsed_content'. + if (!isset($tag['type'])) + { + // !!! Check for end tag first, so people can say "I like that [i] tag"? + $open_tags[] = $tag; + $message = substr($message, 0, $pos) . $tag['before'] . substr($message, $pos1); + $pos += strlen($tag['before']) - 1; + } + // Don't parse the content, just skip it. + elseif ($tag['type'] == 'unparsed_content') + { + $pos2 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos1); + if ($pos2 === false) + continue; + + $data = substr($message, $pos1, $pos2 - $pos1); + + if (!empty($tag['block_level']) && substr($data, 0, 6) == '
    ') + $data = substr($data, 6); + + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + $code = strtr($tag['content'], array('$1' => $data)); + $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + 3 + strlen($tag['tag'])); + $pos += strlen($code) - 1; + } + // Don't parse the content, just skip it. + elseif ($tag['type'] == 'unparsed_equals_content') + { + // The value may be quoted for some tags - check. + if (isset($tag['quoted'])) + { + $quoted = substr($message, $pos1, 6) == '"'; + if ($tag['quoted'] != 'optional' && !$quoted) + continue; + + if ($quoted) + $pos1 += 6; + } + else + $quoted = false; + + $pos2 = strpos($message, $quoted == false ? ']' : '"]', $pos1); + if ($pos2 === false) + continue; + $pos3 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos2); + if ($pos3 === false) + continue; + + $data = array( + substr($message, $pos2 + ($quoted == false ? 1 : 7), $pos3 - ($pos2 + ($quoted == false ? 1 : 7))), + substr($message, $pos1, $pos2 - $pos1) + ); + + if (!empty($tag['block_level']) && substr($data[0], 0, 6) == '
    ') + $data[0] = substr($data[0], 6); + + // Validation for my parking, please! + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + $code = strtr($tag['content'], array('$1' => $data[0], '$2' => $data[1])); + $message = substr($message, 0, $pos) . $code . substr($message, $pos3 + 3 + strlen($tag['tag'])); + $pos += strlen($code) - 1; + } + // A closed tag, with no content or value. + elseif ($tag['type'] == 'closed') + { + $pos2 = strpos($message, ']', $pos); + $message = substr($message, 0, $pos) . $tag['content'] . substr($message, $pos2 + 1); + $pos += strlen($tag['content']) - 1; + } + // This one is sorta ugly... :/. Unforunately, it's needed for flash. + elseif ($tag['type'] == 'unparsed_commas_content') + { + $pos2 = strpos($message, ']', $pos1); + if ($pos2 === false) + continue; + $pos3 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos2); + if ($pos3 === false) + continue; + + // We want $1 to be the content, and the rest to be csv. + $data = explode(',', ',' . substr($message, $pos1, $pos2 - $pos1)); + $data[0] = substr($message, $pos2 + 1, $pos3 - $pos2 - 1); + + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + $code = $tag['content']; + foreach ($data as $k => $d) + $code = strtr($code, array('$' . ($k + 1) => trim($d))); + $message = substr($message, 0, $pos) . $code . substr($message, $pos3 + 3 + strlen($tag['tag'])); + $pos += strlen($code) - 1; + } + // This has parsed content, and a csv value which is unparsed. + elseif ($tag['type'] == 'unparsed_commas') + { + $pos2 = strpos($message, ']', $pos1); + if ($pos2 === false) + continue; + + $data = explode(',', substr($message, $pos1, $pos2 - $pos1)); + + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + // Fix after, for disabled code mainly. + foreach ($data as $k => $d) + $tag['after'] = strtr($tag['after'], array('$' . ($k + 1) => trim($d))); + + $open_tags[] = $tag; + + // Replace them out, $1, $2, $3, $4, etc. + $code = $tag['before']; + foreach ($data as $k => $d) + $code = strtr($code, array('$' . ($k + 1) => trim($d))); + $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + 1); + $pos += strlen($code) - 1; + } + // A tag set to a value, parsed or not. + elseif ($tag['type'] == 'unparsed_equals' || $tag['type'] == 'parsed_equals') + { + // The value may be quoted for some tags - check. + if (isset($tag['quoted'])) + { + $quoted = substr($message, $pos1, 6) == '"'; + if ($tag['quoted'] != 'optional' && !$quoted) + continue; + + if ($quoted) + $pos1 += 6; + } + else + $quoted = false; + + $pos2 = strpos($message, $quoted == false ? ']' : '"]', $pos1); + if ($pos2 === false) + continue; + + $data = substr($message, $pos1, $pos2 - $pos1); + + // Validation for my parking, please! + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + // For parsed content, we must recurse to avoid security problems. + if ($tag['type'] != 'unparsed_equals') + $data = parse_bbc($data); + + $tag['after'] = strtr($tag['after'], array('$1' => $data)); + + $open_tags[] = $tag; + + $code = strtr($tag['before'], array('$1' => $data)); + $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + ($quoted == false ? 1 : 7)); + $pos += strlen($code) - 1; + } + + // If this is block level, eat any breaks after it. + if (!empty($tag['block_level']) && substr($message, $pos + 1, 6) == '
    ') + $message = substr($message, 0, $pos + 1) . substr($message, $pos + 7); + + // Are we trimming outside this tag? + if (!empty($tag['trim']) && $tag['trim'] != 'outside' && preg_match('~(
    | |\s)*~', substr($message, $pos + 1), $matches) != 0) + $message = substr($message, 0, $pos + 1) . substr($message, $pos + 1 + strlen($matches[0])); + } + + // Close any remaining tags. + while ($tag = array_pop($open_tags)) { + if(in_array($tag['tag'], array('table','td','tr','th'))) + return 'INVALID BBCODE: close of unopened tag in table (1)'; + $message .= $tag['after']; + } + + if (substr($message, 0, 1) == ' ') + $message = ' ' . substr($message, 1); + + // Cleanup whitespace. + $message = strtr($message, array(' ' => '  ', "\r" => '', "\n" => '
    ', '
    ' => '
     ', ' ' => "\n")); + + if(in_array('ugc', $local_disable)) + $message = str_replace(' 0.05) + cache_put_data($cache_key, $message, 600); + + ECHO "d"; + + return $message; +} + +// Parse smileys in the passed message. +function parsesmileys(&$message) +{ + global $modSettings, $db_prefix, $txt, $user_info, $context; + static $smileyfromcache = array(), $smileytocache = array(); + + // No smiley set at all?! + if ($user_info['smiley_set'] == 'none') + return; + + // If the smiley array hasn't been set, do it now. + if (empty($smileyfromcache)) + { + // Use the default smileys if it is disabled. (better for "portability" of smileys.) + if (empty($modSettings['smiley_enable'])) + { + $smileysfrom = array('>:D', ':D', '::)', '>:(', ':)', ';)', ';D', ':(', ':o', '8)', ':P', '???', ':-[', ':-X', ':-*', ':\'(', ':-\\', '^-^', 'O0', 'C:-)', '0:)'); + $smileysto = array('evil.gif', 'cheesy.gif', 'rolleyes.gif', 'angry.gif', 'smiley.gif', 'wink.gif', 'grin.gif', 'sad.gif', 'shocked.gif', 'cool.gif', 'tongue.gif', 'huh.gif', 'embarrassed.gif', 'lipsrsealed.gif', 'kiss.gif', 'cry.gif', 'undecided.gif', 'azn.gif', 'afro.gif', 'police.gif', 'angel.gif'); + $smileysdescs = array('', $txt[289], $txt[450], $txt[288], $txt[287], $txt[292], $txt[293], $txt[291], $txt[294], $txt[295], $txt[451], $txt[296], $txt[526], $txt[527], $txt[529], $txt[530], $txt[528], '', '', '', ''); + } + else + { + // Load the smileys in reverse order by length so they don't get parsed wrong. + if (($temp = cache_get_data('parsing_smileys', 480)) == null) + { + $result = db_query(" + SELECT code, filename, description + FROM {$db_prefix}smileys", __FILE__, __LINE__); + $smileysfrom = array(); + $smileysto = array(); + $smileysdescs = array(); + while ($row = mysql_fetch_assoc($result)) + { + $smileysfrom[] = $row['code']; + $smileysto[] = $row['filename']; + $smileysdescs[] = $row['description']; + } + mysql_free_result($result); + + cache_put_data('parsing_smileys', array($smileysfrom, $smileysto, $smileysdescs), 480); + } + else + list ($smileysfrom, $smileysto, $smileysdescs) = $temp; + } + + // The non-breaking-space is a complex thing... + $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{A0}' : pack('C*', 0xC2, 0xA0)) : '\xA0'; + + // This smiley regex makes sure it doesn't parse smileys within code tags (so [url=mailto:David@bla.com] doesn't parse the :D smiley) + for ($i = 0, $n = count($smileysfrom); $i < $n; $i++) + { + $smileyfromcache[] = '/(?<=[>:\?\.\s' . $non_breaking_space . '[\]()*\\\;]|^)(' . preg_quote($smileysfrom[$i], '/') . '|' . preg_quote(htmlspecialchars($smileysfrom[$i], ENT_QUOTES), '/') . ')(?=[^[:alpha:]0-9]|$)/' . ($context['utf8'] ? 'u' : ''); + // Escape a bunch of smiley-related characters in the description so it doesn't get a double dose :P. + $smileytocache[] = '' . strtr(htmlspecialchars($smileysdescs[$i]), array(':' => ':', '(' => '(', ')' => ')', '$' => '$', '[' => '[')) . ''; + } + } + + // Replace away! + // !!! There must be a way to speed this up. + $message = preg_replace($smileyfromcache, $smileytocache, $message); +} + +// Parses some bbc before sending into the database... +function preparsecode(&$message, $previewing = false) +{ + global $user_info, $modSettings, $context; + + + // Clean up after nobbc ;). + $message = preg_replace_callback('~\[nobbc\](.+?)\[/nobbc\]~is', 'nobbc__preg_callback', $message); + + //$message = preg_replace('~\[([^\]=\s]+)[^\]]*\](?' . '>\s|(?R))*?\[/\1\]\s?~i', '', $message); + + // Remove \r's... they're evil! + $message = strtr($message, array("\r" => '')); + + // You won't believe this - but too many periods upsets apache it seems! + $message = preg_replace('~\.{100,}~', '...', $message); + + // Trim off trailing quotes - these often happen by accident. + while (substr($message, -7) == '[quote]') + $message = substr($message, 0, -7); + while (substr($message, 0, 8) == '[/quote]') + $message = substr($message, 8); + + // Check if all code tags are closed. + $codeopen = preg_match_all('~(\[code(?:=[^\]]+)?\])~is', $message, $dummy); + $codeclose = preg_match_all('~(\[/code\])~is', $message, $dummy); + + // Close/open all code tags... + if ($codeopen > $codeclose) + $message .= str_repeat('[/code]', $codeopen - $codeclose); + elseif ($codeclose > $codeopen) + $message = str_repeat('[code]', $codeclose - $codeopen) . $message; + + // Now that we've fixed all the code tags, let's fix the img and url tags... + $parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $message, -1, PREG_SPLIT_DELIM_CAPTURE); + + // The regular expression non breaking space has many versions. + $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{A0}' : pack('C*', 0xC2, 0xA0)) : '\xA0'; + + // Only mess with stuff outside [code] tags. + for ($i = 0, $n = count($parts); $i < $n; $i++) + { + // It goes 0 = outside, 1 = begin tag, 2 = inside, 3 = close tag, repeat. + if ($i % 4 == 0) + { + fixTags($parts[$i]); + + // Replace /me.+?\n with [me=name]dsf[/me]\n. + if (strpos($user_info['name'], '[') !== false || strpos($user_info['name'], ']') !== false || strpos($user_info['name'], '\'') !== false || strpos($user_info['name'], '"') !== false) + $parts[$i] = preg_replace('~(?:\A|\n)/me(?: | )([^\n]*)(?:\z)?~i', '[me="' . $user_info['name'] . '"]$1[/me]', $parts[$i]); + else + $parts[$i] = preg_replace('~(?:\A|\n)/me(?: | )([^\n]*)(?:\z)?~i', '[me=' . $user_info['name'] . ']$1[/me]', $parts[$i]); + + if (!$previewing && strpos($parts[$i], '[html]') !== false) + { + //if (false && allowedTo('admin_forum')) + //$parts[$i] = preg_replace('~\[html\](.+?)\[/html\]~ise', '\'[html]\' . strtr(un_htmlspecialchars(\'$1\'), array("\n" => \' \', \' \' => \' \')) . \'[/html]\'', $parts[$i]); + // We should edit them out, or else if an admin edits the message they will get shown... + //else + //{ + while (strpos($parts[$i], '[html]') !== false) + $parts[$i] = preg_replace('~\[[/]?html\]~i', '', $parts[$i]); + //} + } + + // Let's look at the time tags... + $parts[$i] = preg_replace_callback('~\[time(?:=(absolute))*\](.+?)\[/time\]~i', 'time_fix__preg_callback', $parts[$i]); + + $list_open = substr_count($parts[$i], '[list]') + substr_count($parts[$i], '[list '); + $list_close = substr_count($parts[$i], '[/list]'); + if ($list_close - $list_open > 0) + $parts[$i] = str_repeat('[list]', $list_close - $list_open) . $parts[$i]; + if ($list_open - $list_close > 0) + $parts[$i] = $parts[$i] . str_repeat('[/list]', $list_open - $list_close); + + // Make sure all tags are lowercase. + $parts[$i] = preg_replace_callback('~\[([/]?)(list|li|table|tr|td)((\s[^\]]+)*)\]~i', 'lowercase_tags__preg_callback', $parts[$i]); + + $mistake_fixes = array( + // Find [table]s not followed by [tr]. + '~\[table\](?![\s' . $non_breaking_space . ']*\[tr\])~s' . ($context['utf8'] ? 'u' : '') => '[table][tr]', + // Find [tr]s not followed by [td]. + '~\[tr\](?![\s' . $non_breaking_space . ']*\[td\])~s' . ($context['utf8'] ? 'u' : '') => '[tr][td]', + // Find [/td]s not followed by something valid. + '~\[/td\](?![\s' . $non_breaking_space . ']*(?:\[td\]|\[/tr\]|\[/table\]))~s' . ($context['utf8'] ? 'u' : '') => '[/td][/tr]', + // Find [/tr]s not followed by something valid. + '~\[/tr\](?![\s' . $non_breaking_space . ']*(?:\[tr\]|\[/table\]))~s' . ($context['utf8'] ? 'u' : '') => '[/tr][/table]', + // Find [/td]s incorrectly followed by [/table]. + '~\[/td\][\s' . $non_breaking_space . ']*\[/table\]~s' . ($context['utf8'] ? 'u' : '') => '[/td][/tr][/table]', + // Find [table]s, [tr]s, and [/td]s (possibly correctly) followed by [td]. + '~\[(table|tr|/td)\]([\s' . $non_breaking_space . ']*)\[td\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_td_]', + // Now, any [td]s left should have a [tr] before them. + '~\[td\]~s' => '[tr][td]', + // Look for [tr]s which are correctly placed. + '~\[(table|/tr)\]([\s' . $non_breaking_space . ']*)\[tr\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_tr_]', + // Any remaining [tr]s should have a [table] before them. + '~\[tr\]~s' => '[table][tr]', + // Look for [/td]s followed by [/tr]. + '~\[/td\]([\s' . $non_breaking_space . ']*)\[/tr\]~s' . ($context['utf8'] ? 'u' : '') => '[/td]$1[_/tr_]', + // Any remaining [/tr]s should have a [/td]. + '~\[/tr\]~s' => '[/td][/tr]', + // Look for properly opened [li]s which aren't closed. + '~\[li\]([^\[\]]+?)\[li\]~s' => '[li]$1[_/li_][_li_]', + '~\[li\]([^\[\]]+?)$~s' => '[li]$1[/li]', + // Lists - find correctly closed items/lists. + '~\[/li\]([\s' . $non_breaking_space . ']*)\[/list\]~s' . ($context['utf8'] ? 'u' : '') => '[_/li_]$1[/list]', + // Find list items closed and then opened. + '~\[/li\]([\s' . $non_breaking_space . ']*)\[li\]~s' . ($context['utf8'] ? 'u' : '') => '[_/li_]$1[_li_]', + // Now, find any [list]s or [/li]s followed by [li]. + '~\[(list(?: [^\]]*?)?|/li)\]([\s' . $non_breaking_space . ']*)\[li\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_li_]', + // Any remaining [li]s weren't inside a [list]. + '~\[li\]~' => '[list][li]', + // Any remaining [/li]s weren't before a [/list]. + '~\[/li\]~' => '[/li][/list]', + // Put the correct ones back how we found them. + '~\[_(li|/li|td|tr|/tr)_\]~' => '[$1]', + ); + + // Fix up some use of tables without [tr]s, etc. (it has to be done more than once to catch it all.) + for ($j = 0; $j < 3; $j++) + $parts[$i] = preg_replace(array_keys($mistake_fixes), $mistake_fixes, $parts[$i]); + } + } + + // Put it back together! + if (!$previewing) + $message = strtr(implode('', $parts), array(' ' => '  ', "\n" => '
    ', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); + else + $message = strtr(implode('', $parts), array(' ' => '  ', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); + + // Now let's quickly clean up things that will slow our parser (which are common in posted code.) + $message = strtr($message, array('[]' => '[]', '['' => '['')); +} + +// This is very simple, and just removes things done by preparsecode. +function un_preparsecode($message) +{ + $parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $message, -1, PREG_SPLIT_DELIM_CAPTURE); + + // We're going to unparse only the stuff outside [code]... + for ($i = 0, $n = count($parts); $i < $n; $i++) + { + // If $i is a multiple of four (0, 4, 8, ...) then it's not a code section... + if ($i % 4 == 0) + { + $parts[$i] = preg_replace_callback('~\[html\](.+?)\[/html\]~i', function($m){return '[html]' . strtr(htmlspecialchars(stripslashes($m[1]), ENT_QUOTES), array('&#13;' => '
    ', '&#32;' => ' ')) . '[/html]';}, $parts[$i]); + + // Attempt to un-parse the time to something less awful. + $parts[$i] = preg_replace_callback('~\[time\](\d{0,10})\[/time\]~i', 'time_format__preg_callback', $parts[$i]); + } + } + + // Change breaks back to \n's and &nsbp; back to spaces. + return preg_replace('~~', "\n", str_replace(' ', ' ', implode('', $parts))); +} + +function action_fix__preg_callback($matches) +{ + return $matches[1] . preg_replace('~action(=|%3d)(?!dlattach)~i', 'action-', $matches[2]) . '[/img]'; +} + +function mime_convert__preg_callback($matches) +{ + $c = $matches[1]; + if (strlen($c) === 1 && ord($c[0]) <= 0x7F) + return $c; + elseif (strlen($c) === 2 && ord($c[0]) >= 0xC0 && ord($c[0]) <= 0xDF) + return '&#' . (((ord($c[0]) ^ 0xC0) << 6) + (ord($c[1]) ^ 0x80)) . ';'; + elseif (strlen($c) === 3 && ord($c[0]) >= 0xE0 && ord($c[0]) <= 0xEF) + return '&#' . (((ord($c[0]) ^ 0xE0) << 12) + ((ord($c[1]) ^ 0x80) << 6) + (ord($c[2]) ^ 0x80)) . ';'; + elseif (strlen($c) === 4 && ord($c[0]) >= 0xF0 && ord($c[0]) <= 0xF7) + return '&#' . (((ord($c[0]) ^ 0xF0) << 18) + ((ord($c[1]) ^ 0x80) << 12) + ((ord($c[2]) ^ 0x80) << 6) + (ord($c[3]) ^ 0x80)) . ';'; + else + return ''; +} + +function time_fix__preg_callback($matches) +{ + global $modSettings, $user_info; + return '[time]' . (is_numeric($matches[2]) || @strtotime($matches[2]) == 0 ? $matches[2] : strtotime($matches[2]) - ($matches[1] == 'absolute' ? 0 : (($modSettings['time_offset'] + $user_info['time_offset']) * 3600))) . '[/time]'; +} + +function nobbc__preg_callback($matches) +{ + return '[nobbc]' . strtr($matches[1], array('[' => '[', ']' => ']', ':' => ':', '@' => '@')) . '[/nobbc]'; +} + +function lowercase_tags__preg_callback($matches) +{ + return '[' . $matches[1] . strtolower($matches[2]) . $matches[3] . ']'; +} + +function htmlspecial_html__preg_callback($matches) +{ + global $modSettings, $txt; + static $charset = null; + if ($charset === null) + $charset = empty($modSettings['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']; + + return '[html]' . strtr(htmlspecialchars($matches[1], ENT_QUOTES, $charset), array('\\"' => '"', '&#13;' => '
    ', '&#32;' => ' ', '&#91;' => '[', '&#93;' => ']')) . '[/html]'; +} + +function time_format__preg_callback($matches) +{ + return '[time]' . timeformat($matches[1], false) . '[/time]'; +} +function word_break__preg_callback($matches) +{ + global $modSettings, $context; + return preg_replace('~(.{' . ($modSettings['fixLongWords'] - 1) . '})~' . ($context['utf8'] ? 'u' : ''), '$1< >', $matches[1]); +} + +//this would normally transform the URL into a proxied URL, but here it does nothing +function proxyurl($url) { + return $url; +} + +?> From 5088ca013a9e3903527a0271cf35c97ae149830e Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 23 May 2024 15:25:50 -1000 Subject: [PATCH 045/231] refactor(bbcode-parser): wip comment out code until parser runs --- parsing.php | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/parsing.php b/parsing.php index 8cbd08ff..8f751e65 100644 --- a/parsing.php +++ b/parsing.php @@ -7,6 +7,14 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a { global $txt, $scripturl, $context, $modSettings, $user_info; static $bbc_codes = array(), $itemcodes = array(), $no_autolink_tags = array(); + $context['browser']['is_gecko'] = false; + $context['browser']['is_ie5'] = false; + $context['browser']['is_ie5'] = false; + $context['browser']['is_ie4'] = false; + $context['browser']['is_mac_ie'] = false; + $context['browser']['is_konqueror'] = false; + $context['browser']['is_opera'] = false; + $context['browser']['is_ie'] = false; $disabled = array(); //theymos - die if it's taking way too long @@ -21,19 +29,20 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a $smileys = (bool) $smileys; ECHO "a"; - if (empty($modSettings['enableBBC']) && $message !== false) - { - if ($smileys === true) - parsesmileys($message); - ECHO "b"; - - return $message; - } + // if (empty($modSettings['enableBBC']) && $message !== false) + // { + // if ($smileys === true) + // parsesmileys($message); + // ECHO "b"; + + // return $message; + // } ECHO "c"; // Just in case it wasn't determined yet whether UTF-8 is enabled. if (!isset($context['utf8'])) - $context['utf8'] = (empty($modSettings['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']) === 'UTF-8'; + // $context['utf8'] = (empty(ç['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']) === 'UTF-8'; + $context['utf8'] = true; //theymos - disable links and images on pages where we don't want to send a referer to random people $disabledsecurity=''; @@ -204,7 +213,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a array( 'tag' => 'code', 'type' => 'unparsed_content', - 'content' => '
    ' . $txt['smf238'] . ':
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', + 'content' => '
    ' . '$txt[\'smf238\']' . ':
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', // !!! Maybe this can be simplified? 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { global $context; @@ -242,7 +251,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a array( 'tag' => 'code', 'type' => 'unparsed_equals_content', - 'content' => '
    ' . $txt['smf238'] . ': ($2)
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', + 'content' => '
    ' . '$txt[\'smf238\']' . ': ($2)
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', // !!! Maybe this can be simplified? 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { global $context; @@ -522,7 +531,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a ), array( 'tag' => 'quote', - 'before' => '
    ' . $txt['smf240'] . '
    ', + 'before' => '
    ' . '$txt[\'smf240\']' . '
    ', 'after' => '
    ', 'block_level' => true, ), @@ -531,14 +540,14 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a 'parameters' => array( 'author' => array('match' => '(.{1,192}?)', 'quoted' => true, 'validate' => 'parse_bbc'), ), - 'before' => '
    ' . $txt['smf239'] . ': {author}
    ', + 'before' => '
    ' . '$txt[\'smf239\']' . ': {author}
    ', 'after' => '
    ', 'block_level' => true, ), array( 'tag' => 'quote', 'type' => 'parsed_equals', - 'before' => '
    ' . $txt['smf239'] . ': $1
    ', + 'before' => '
    ' . '$txt[\'smf239\']' . ': $1
    ', 'after' => '
    ', 'quoted' => 'optional', 'block_level' => true, @@ -550,7 +559,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a 'link' => array('match' => '(?:board=\d+;)?((?:topic|threadid)=[\dmsg#\./]{1,40}(?:;start=[\dmsg#\./]{1,40})?|action=profile;u=\d+)'), 'date' => array('match' => '(\d+)', 'validate' => 'timeformat'), ), - 'before' => '
    ' . $txt['smf239'] . ': {author} ' . $txt[176] . ' {date}
    ', + 'before' => '
    ' . '$txt[\'smf239\']' . ': {author} ' . '$txt[\'176\']' . ' {date}
    ', 'after' => '
    ', 'block_level' => true, ), @@ -559,7 +568,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a 'parameters' => array( 'author' => array('match' => '(.{1,192}?)', 'validate' => 'parse_bbc'), ), - 'before' => '
    ' . $txt['smf239'] . ': {author}
    ', + 'before' => '
    ' . '$txt[\'smf239\']' . ': {author}
    ', 'after' => '
    ', 'block_level' => true, ), @@ -819,6 +828,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a $message = strtr($message, array("\n" => '
    ')); // The non-breaking-space looks a bit different each time. + $context['server']['complex_preg_chars'] = true; $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{C2A0}' : chr(0xC2) . chr(0xA0)) : '\xA0'; $pos = -1; From 65a9ecbcc49c5f5c655f6d9393276a2b598ef32a Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 28 May 2024 11:40:30 -1000 Subject: [PATCH 046/231] refactor(wip-bbcode): partially working bbcode parser --- lib/epochtalk_server_web/controllers/post.ex | 6 ++---- parsing.php | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 4fafef5f..9228eeaa 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -383,13 +383,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do Map.get(Application.get_env(:epochtalk_server, :frontend_config), :post_max_length) || Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"], body <- - Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1), - parsed_body <- body do + Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1) do - %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('Hello [br] [color=red]Hello[/color]', false);\"") + %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "', false);\"") IO.inspect status IO.inspect output - IO.inspect parsed_body render(conn, :parse_legacy, %{parsed_body: output}) else _ -> diff --git a/parsing.php b/parsing.php index 8f751e65..a3b0d70e 100644 --- a/parsing.php +++ b/parsing.php @@ -28,7 +28,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a elseif ($smileys !== null && ($smileys == '1' || $smileys == '0')) $smileys = (bool) $smileys; - ECHO "a"; + // ECHO "a"; // if (empty($modSettings['enableBBC']) && $message !== false) // { // if ($smileys === true) @@ -37,7 +37,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a // return $message; // } - ECHO "c"; + // ECHO "c"; // Just in case it wasn't determined yet whether UTF-8 is enabled. if (!isset($context['utf8'])) @@ -1555,7 +1555,7 @@ function parse_bbc($message, $smileys = true, $cache_id = '', $local_disable = a if (isset($cache_key, $cache_t) && array_sum(explode(' ', microtime())) - array_sum(explode(' ', $cache_t)) > 0.05) cache_put_data($cache_key, $message, 600); - ECHO "d"; + // ECHO "d"; return $message; } From f6ef7c9f142e113695ecfb87eec2ec3050ea7472 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 28 May 2024 13:28:38 -1000 Subject: [PATCH 047/231] refactor(wip-bbcode): escape single quote for php bbcode script input --- lib/epochtalk_server_web/controllers/post.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 9228eeaa..698d0634 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -384,10 +384,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"], body <- Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1) do - + body = String.replace(body, "'", "\'") %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "', false);\"") - IO.inspect status - IO.inspect output + IO.inspect "'#{body}'" + IO.inspect output + IO.inspect status render(conn, :parse_legacy, %{parsed_body: output}) else _ -> From 9eb8e30dd8c4ab652e28774dae3467e26112fc92 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 28 May 2024 14:06:10 -1000 Subject: [PATCH 048/231] fix(bbcode): no auth for bbcode parser --- lib/epochtalk_server_web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/router.ex b/lib/epochtalk_server_web/router.ex index 3ba52053..12292b0e 100644 --- a/lib/epochtalk_server_web/router.ex +++ b/lib/epochtalk_server_web/router.ex @@ -56,7 +56,6 @@ defmodule EpochtalkServerWeb.Router do post "/posts", Post, :create post "/posts/:id", Post, :update post "/preview", Post, :preview - post "/bbcode", Post, :parse_legacy get "/admin/modlog", ModerationLog, :page get "/boards/movelist", Board, :movelist post "/images/s3/upload", ImageReference, :s3_request_upload @@ -78,6 +77,7 @@ defmodule EpochtalkServerWeb.Router do post "/register", User, :register post "/login", User, :login post "/confirm", User, :confirm + post "/bbcode", Post, :parse_legacy delete "/logout", User, :logout end From af54a8e88e37edddda3666cac6fd5842c8f0c01a Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 11 Jun 2024 10:21:33 -1000 Subject: [PATCH 049/231] refactor(wip-parser): wip implement parser for legacy bbcode --- parsing.php | 3554 ++++++++++++++++++++++----------------------- parsing_extra.php | 649 +++++++++ 2 files changed, 2416 insertions(+), 1787 deletions(-) create mode 100644 parsing_extra.php diff --git a/parsing.php b/parsing.php index a3b0d70e..99bb44df 100644 --- a/parsing.php +++ b/parsing.php @@ -1,1842 +1,1822 @@ 29 && strlen($message)>500 && php_sapi_name() != 'cli') { - die(); - } - - // Never show smileys for wireless clients. More bytes, can't see it anyway :P. - if (WIRELESS) - $smileys = false; - elseif ($smileys !== null && ($smileys == '1' || $smileys == '0')) - $smileys = (bool) $smileys; - - // ECHO "a"; - // if (empty($modSettings['enableBBC']) && $message !== false) - // { - // if ($smileys === true) - // parsesmileys($message); - // ECHO "b"; - - // return $message; - // } - // ECHO "c"; - - // Just in case it wasn't determined yet whether UTF-8 is enabled. - if (!isset($context['utf8'])) - // $context['utf8'] = (empty(ç['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']) === 'UTF-8'; - $context['utf8'] = true; - - //theymos - disable links and images on pages where we don't want to send a referer to random people - $disabledsecurity=''; - if(isset($_GET['sesc'])) { - $cache_id = ''; - $disabled['img']=true; - $disabled['iurl']=true; - $disabled['url']=true; - $disabled['ftp']=true; - $disabledsecurity=' (FORUM: disabled on this page for security.)'; - } - if(isset($_GET['patrol'])) { - $cache_id = ''; - $disabled['black']=true; - $disabled['color']=true; - } - - //theymos - these tags are aways disabled - $disabled['flash'] = true; - $disabled['move'] = true; - - // Sift out the bbc for a performance improvement. - if (empty($bbc_codes) || $message === false) - { - /*if (!empty($modSettings['disabledBBC'])) - { - $temp = explode(',', strtolower($modSettings['disabledBBC'])); - - foreach ($temp as $tag) - $disabled[trim($tag)] = true; - } - - if (empty($modSettings['enableEmbeddedFlash'])) - $disabled['flash'] = true;*/ - - /* The following bbc are formatted as an array, with keys as follows: - - tag: the tag's name - should be lowercase! - - type: one of... - - (missing): [tag]parsed content[/tag] - - unparsed_equals: [tag=xyz]parsed content[/tag] - - parsed_equals: [tag=parsed data]parsed content[/tag] - - unparsed_content: [tag]unparsed content[/tag] - - closed: [tag], [tag/], [tag /] - - unparsed_commas: [tag=1,2,3]parsed content[/tag] - - unparsed_commas_content: [tag=1,2,3]unparsed content[/tag] - - unparsed_equals_content: [tag=...]unparsed content[/tag] - - parameters: an optional array of parameters, for the form - [tag abc=123]content[/tag]. The array is an associative array - where the keys are the parameter names, and the values are an - array which may contain the following: - - match: a regular expression to validate and match the value. - - quoted: true if the value should be quoted. - - validate: callback to evaluate on the data, which is $data. - - value: a string in which to replace $1 with the data. - either it or validate may be used, not both. - - optional: true if the parameter is optional. - - test: a regular expression to test immediately after the tag's - '=', ' ' or ']'. Typically, should have a \] at the end. - Optional. - - content: only available for unparsed_content, closed, - unparsed_commas_content, and unparsed_equals_content. - $1 is replaced with the content of the tag. Parameters - are repalced in the form {param}. For unparsed_commas_content, - $2, $3, ..., $n are replaced. - - before: only when content is not used, to go before any - content. For unparsed_equals, $1 is replaced with the value. - For unparsed_commas, $1, $2, ..., $n are replaced. - - after: similar to before in every way, except that it is used - when the tag is closed. - - disabled_content: used in place of content when the tag is - disabled. For closed, default is '', otherwise it is '$1' if - block_level is false, '
    $1
    ' elsewise. - - disabled_before: used in place of before when disabled. Defaults - to '
    ' if block_level, '' if not. - - disabled_after: used in place of after when disabled. Defaults - to '
    ' if block_level, '' if not. - - block_level: set to true the tag is a "block level" tag, similar - to HTML. Block level tags cannot be nested inside tags that are - not block level, and will not be implicitly closed as easily. - One break following a block level tag may also be removed. - - trim: if set, and 'inside' whitespace after the begin tag will be - removed. If set to 'outside', whitespace after the end tag will - meet the same fate. - - validate: except when type is missing or 'closed', a callback to - validate the data as $data. Depending on the tag's type, $data - may be a string or an array of strings (corresponding to the - replacement.) - - quoted: when type is 'unparsed_equals' or 'parsed_equals' only, - may be not set, 'optional', or 'required' corresponding to if - the content may be quoted. This allows the parser to read - [tag="abc]def[esdf]"] properly. - - require_parents: an array of tag names, or not set. If set, the - enclosing tag *must* be one of the listed tags, or parsing won't - occur. - - require_children: similar to require_parents, if set children - won't be parsed if they are not in the list. - - disallow_children: similar to, but very different from, - require_children, if it is set the listed tags will not be - parsed inside the tag. - */ - - $codes = array( - array( - 'tag' => 'abbr', - 'type' => 'unparsed_equals', - 'before' => '', - 'after' => '', - 'quoted' => 'optional', - 'disabled_after' => ' ($1)', - ), - array( - 'tag' => 'acronym', - 'type' => 'unparsed_equals', - 'before' => '', - 'after' => '', - 'quoted' => 'optional', - 'disabled_after' => ' ($1)', - ), - array( - 'tag' => 'anchor', - 'type' => 'unparsed_equals', - 'test' => '[#]?([A-Za-z][A-Za-z0-9_\-]*)\]', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'b', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'black', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'blue', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'br', - 'type' => 'closed', - 'content' => '
    ', - ), - array( - 'tag' => 'btc', - 'type' => 'closed', - 'content' => 'BTC', - ), - array( - 'tag' => 'code', - 'type' => 'unparsed_content', - 'content' => '
    ' . '$txt[\'smf238\']' . ':
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', - // !!! Maybe this can be simplified? - 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { - global $context; - - if (!isset($disabled['code'])) - { - $php_parts = preg_split('~(<\?php|\?>)~', $data, -1, PREG_SPLIT_DELIM_CAPTURE); - - for ($php_i = 0, $php_n = count($php_parts); $php_i < $php_n; $php_i++) - { - // Do PHP code coloring? - if ($php_parts[$php_i] != '<?php') - continue; - - $php_string = ''; - while ($php_i + 1 < count($php_parts) && $php_parts[$php_i] != '?>') - { - $php_string .= $php_parts[$php_i]; - $php_parts[$php_i++] = ''; - } - $php_parts[$php_i] = highlight_php_code($php_string . $php_parts[$php_i]); - } - - // Fix the PHP code stuff... - $data = str_replace("
    \t
    ", "\t", implode('', $php_parts)); - - // Older browsers are annoying, aren't they? - if ($context['browser']['is_ie4'] || $context['browser']['is_ie5'] || $context['browser']['is_ie5.5']) - $data = str_replace("\t", "
    \t
    ", $data); - elseif (!$context['browser']['is_gecko']) - $data = str_replace("\t", "\t", $data); - }}, - 'block_level' => true, - ), - array( - 'tag' => 'code', - 'type' => 'unparsed_equals_content', - 'content' => '
    ' . '$txt[\'smf238\']' . ': ($2)
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', - // !!! Maybe this can be simplified? - 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { - global $context; - - if (!isset($disabled['code'])) - { - $php_parts = preg_split('~(<\?php|\?>)~', $data[0], -1, PREG_SPLIT_DELIM_CAPTURE); - - for ($php_i = 0, $php_n = count($php_parts); $php_i < $php_n; $php_i++) - { - // Do PHP code coloring? - if ($php_parts[$php_i] != '<?php') - continue; - - $php_string = ''; - while ($php_i + 1 < count($php_parts) && $php_parts[$php_i] != '?>') - { - $php_string .= $php_parts[$php_i]; - $php_parts[$php_i++] = ''; - } - $php_parts[$php_i] = highlight_php_code($php_string . $php_parts[$php_i]); - } - - // Fix the PHP code stuff... - $data[0] = str_replace("
    \t
    ", "\t", implode('', $php_parts)); - - // Older browsers are annoying, aren't they? - if ($context['browser']['is_ie4'] || $context['browser']['is_ie5'] || $context['browser']['is_ie5.5']) - $data = str_replace("\t", "
    \t
    ", $data); - elseif (!$context['browser']['is_gecko']) - $data = str_replace("\t", "\t", $data); - }}, - 'block_level' => true, - ), - array( - 'tag' => 'center', - 'before' => '
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'color', - 'type' => 'unparsed_equals', - 'test' => '(#[\da-fA-F]{3}|#[\da-fA-F]{6}|[A-Za-z]{1,12})\]', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'email', - 'type' => 'unparsed_content', - 'content' => '$1', - // !!! Should this respect guest_hideContacts? - 'validate' => function(&$tag, &$data, $disabled) {$data = strtr($data, array('
    ' => ''));}, - ), - array( - 'tag' => 'email', - 'type' => 'unparsed_equals', - 'before' => '', - 'after' => '', - // !!! Should this respect guest_hideContacts? - 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), - 'disabled_after' => ' ($1)', - ), - array( - 'tag' => 'ftp', - 'type' => 'unparsed_content', - 'content' => '$1', - 'validate' => function(&$tag, &$data, $disabled) { - $data = strtr($data, array('
    ' => '')); - if (strpos($data, 'ftp://') !== 0 && strpos($data, 'ftps://') !== 0) - $data = 'ftp://' . $data; - }, - ), - array( - 'tag' => 'ftp', - 'type' => 'unparsed_equals', - 'before' => '', - 'after' => '', - 'validate' => function(&$tag, &$data, $disabled) { - if (strpos($data, 'ftp://') !== 0 && strpos($data, 'ftps://') !== 0) - $data = 'ftp://' . $data; - }, - 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), - 'disabled_after' => ' ($1)', - ), - array( - 'tag' => 'font', - 'type' => 'unparsed_equals', - 'test' => '[A-Za-z0-9_,\-\s]+?\]', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'flash', - 'type' => 'unparsed_commas_content', - 'test' => '\d+,\d+\]', - 'content' => ($context['browser']['is_ie'] && !$context['browser']['is_mac_ie'] ? '<a href="$1">$1</a>' : '<a href="$1">$1</a>'), - 'validate' => function(&$tag, &$data, $disabled) { - if (isset($disabled['url'])) - $tag['content'] = '$1'; - elseif (strpos($data[0], 'http://') !== 0 && strpos($data[0], 'https://') !== 0) - $data[0] = 'http://' . $data[0]; - }, - 'disabled_content' => $disabledsecurity ? '$1': '$1', - ), - array( - 'tag' => 'green', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'glow', - 'type' => 'unparsed_commas', - 'test' => '[#0-9a-zA-Z\-]{3,12},([012]\d{1,2}|\d{1,2})(,[^]]+)?\]', - 'before' => $context['browser']['is_ie'] ? '
    ' : '', - 'after' => $context['browser']['is_ie'] ? '
    ' : '    ', - ), - array( - 'tag' => 'hr', - 'type' => 'closed', - 'content' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'html', - 'type' => 'unparsed_content', - 'content' => '$1', - 'block_level' => true, - 'disabled_content' => '$1', - ), - array( - 'tag' => 'img', - 'type' => 'unparsed_content', - 'parameters' => array( - 'alt' => array('optional' => true), - 'width' => array('optional' => true, 'value' => ' width="$1"', 'match' => '(\d{1,4})'), - 'height' => array('optional' => true, 'value' => ' height="$1"', 'match' => '(\d{1,4})'), - ), - 'content' => '{alt}', - 'validate' => function(&$tag, &$data, $disabled) { - $data = strtr($data, array('
    ' => '')); - if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0) - $data = 'http://' . $data; - if(!isset($disabled['img'])) - $data = proxyurl($data); - }, - 'disabled_content' => $disabledsecurity ? ('($1)'.$disabledsecurity) : '$1', - ), - array( - 'tag' => 'img', - 'type' => 'unparsed_content', - 'content' => '', - 'validate' => function(&$tag, &$data, $disabled) { - $data = strtr($data, array('
    ' => '')); - if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0) - $data = 'http://' . $data; - if(!isset($disabled['img'])) - $data = proxyurl($data); - }, - 'disabled_content' => $disabledsecurity ? ('($1)'.$disabledsecurity) : '$1', - ), - array( - 'tag' => 'i', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'iurl', - 'type' => 'unparsed_content', - 'content' => '$1', - 'validate' => function(&$tag, &$data, $disabled) { - $data = strtr($data, array('
    ' => '')); - if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) - $data = 'http://' . $data; - }, - ), - array( - 'tag' => 'iurl', - 'type' => 'unparsed_equals', - 'before' => '', - 'after' => '', - 'validate' => function(&$tag, &$data, $disabled) { - if (substr($data, 0, 1) == '#') - $data = '#post_' . substr($data, 1); - elseif (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) - $data = 'http://' . $data; - }, - 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), - 'disabled_after' => ' ($1)', - ), - array( - 'tag' => 'li', - 'before' => '
  • ', - 'after' => '
    • ', - 'trim' => 'outside', - 'require_parents' => array('list'), - 'block_level' => true, - 'disabled_before' => '', - 'disabled_after' => '
    ', - ), - array( - 'tag' => 'list', - 'before' => '
      ', - 'after' => '
    ', - 'trim' => 'inside', - 'require_children' => array('li'), - 'block_level' => true, - ), - array( - 'tag' => 'list', - 'parameters' => array( - 'type' => array('match' => '(none|disc|circle|square|decimal|decimal-leading-zero|lower-roman|upper-roman|lower-alpha|upper-alpha|lower-greek|lower-latin|upper-latin|hebrew|armenian|georgian|cjk-ideographic|hiragana|katakana|hiragana-iroha|katakana-iroha)'), - ), - 'before' => '
      ', - 'after' => '
    ', - 'trim' => 'inside', - 'require_children' => array('li'), - 'block_level' => true, - ), - array( - 'tag' => 'left', - 'before' => '
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'ltr', - 'before' => '
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'me', - 'type' => 'unparsed_equals', - 'before' => '
    * $1 ', - 'after' => '
    ', - 'quoted' => 'optional', - 'block_level' => true, - 'disabled_before' => '/me ', - 'disabled_after' => '
    ', - ), - array( - 'tag' => 'move', - 'before' => '', - 'after' => '', - 'block_level' => true, - ), - array( - 'tag' => 'nbsp', - 'type' => 'closed', - 'content' => ' ', - ), - array( - 'tag' => 'nobbc', - 'type' => 'unparsed_content', - 'content' => '$1', - ), - array( - 'tag' => 'pre', - 'before' => '
    ',
-				'after' => '
    ', - ), - array( - 'tag' => 'php', - 'type' => 'unparsed_content', - 'content' => '
    $1
    ', - 'validate' => isset($disabled['php']) ? null : function(&$tag, &$data, $disabled) { - if (!isset($disabled['php'])) - { - $add_begin = substr(trim($data), 0, 5) != '<?'; - $data = highlight_php_code($add_begin ? '<?php ' . $data . '?>' : $data); - if ($add_begin) - $data = preg_replace(array('~^(.+?)<\?.{0,40}?php( |\s)~', '~\?>((?:)*)$~'), '$1', $data, 2); - }}, - 'block_level' => true, - 'disabled_content' => '$1', - ), - array( - 'tag' => 'quote', - 'before' => '
    ' . '$txt[\'smf240\']' . '
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'quote', - 'parameters' => array( - 'author' => array('match' => '(.{1,192}?)', 'quoted' => true, 'validate' => 'parse_bbc'), - ), - 'before' => '
    ' . '$txt[\'smf239\']' . ': {author}
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'quote', - 'type' => 'parsed_equals', - 'before' => '
    ' . '$txt[\'smf239\']' . ': $1
    ', - 'after' => '
    ', - 'quoted' => 'optional', - 'block_level' => true, - ), - array( - 'tag' => 'quote', - 'parameters' => array( - 'author' => array('match' => '([^<>]{1,192}?)'), - 'link' => array('match' => '(?:board=\d+;)?((?:topic|threadid)=[\dmsg#\./]{1,40}(?:;start=[\dmsg#\./]{1,40})?|action=profile;u=\d+)'), - 'date' => array('match' => '(\d+)', 'validate' => 'timeformat'), - ), - 'before' => '
    ' . '$txt[\'smf239\']' . ': {author} ' . '$txt[\'176\']' . ' {date}
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'quote', - 'parameters' => array( - 'author' => array('match' => '(.{1,192}?)', 'validate' => 'parse_bbc'), - ), - 'before' => '
    ' . '$txt[\'smf239\']' . ': {author}
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'right', - 'before' => '
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 'red', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'rtl', - 'before' => '
    ', - 'after' => '
    ', - 'block_level' => true, - ), - array( - 'tag' => 's', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'size', - 'type' => 'unparsed_equals', - 'test' => '([1-9][\d]?p[xt]|(?:x-)?small(?:er)?|(?:x-)?large[r]?)\]', - // !!! line-height - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'size', - 'type' => 'unparsed_equals', - 'test' => '[1-9]\]', - // !!! line-height - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'sub', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'sup', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'shadow', - 'type' => 'unparsed_commas', - 'test' => '[#0-9a-zA-Z\-]{3,12},(left|right|top|bottom|[0123]\d{0,2})\]', - 'before' => $context['browser']['is_ie'] ? '' : '', - 'after' => '', - 'validate' => $context['browser']['is_ie'] ? function(&$tag, &$data, $disabled) { - if ($data[1] == 'left') - $data[1] = 270; - elseif ($data[1] == 'right') - $data[1] = 90; - elseif ($data[1] == 'top') - $data[1] = 0; - elseif ($data[1] == 'bottom') - $data[1] = 180; - else - $data[1] = (int) $data[1];} : function(&$tag, &$data, $disabled) { - if ($data[1] == 'top' || (is_numeric($data[1]) && $data[1] < 50)) - return '0 -2px'; - elseif ($data[1] == 'right' || (is_numeric($data[1]) && $data[1] < 100)) - return '2px 0'; - elseif ($data[1] == 'bottom' || (is_numeric($data[1]) && $data[1] < 190)) - return '0 2px'; - elseif ($data[1] == 'left' || (is_numeric($data[1]) && $data[1] < 280)) - return '-2px 0'; - else - return '0 0';}, - ), - array( - 'tag' => 'time', - 'type' => 'unparsed_content', - 'content' => '$1', - 'validate' => function(&$tag, &$data, $disabled) { - if (is_numeric($data)) - $data = timeformat($data); - else - $tag['content'] = '[time]$1[/time]';}, - ), - array( - 'tag' => 'tt', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'table', - 'before' => '', - 'after' => '
    ', - 'trim' => 'inside', - 'require_children' => array('tr'), - 'block_level' => true, - ), - array( - 'tag' => 'tr', - 'before' => '', - 'after' => '', - 'require_parents' => array('table'), - 'require_children' => array('td'), - 'trim' => 'both', - 'block_level' => true, - 'disabled_before' => '', - 'disabled_after' => '', - ), - array( - 'tag' => 'td', - 'before' => '', - 'after' => '', - 'require_parents' => array('tr'), - 'trim' => 'outside', - 'block_level' => true, - 'disabled_before' => '', - 'disabled_after' => '', - ), - array( - 'tag' => 'url', - 'type' => 'unparsed_content', - 'content' => '$1', - 'validate' => function(&$tag, &$data, $disabled) { - $data = strtr($data, array('
    ' => '')); - if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) - $data = 'http://' . $data; - }, - ), - array( - 'tag' => 'url', - 'type' => 'unparsed_equals', - 'before' => '', - 'after' => '', - 'validate' => function(&$tag, &$data, $disabled) { - if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) - $data = 'http://' . $data; - }, - 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), - 'disabled_after' => ' ($1)', - ), - array( - 'tag' => 'u', - 'before' => '', - 'after' => '', - ), - array( - 'tag' => 'white', - 'before' => '', - 'after' => '', - ), - ); - - // This is mainly for the bbc manager, so it's easy to add tags above. Custom BBC should be added above this line. - if ($message === false) - return $codes; - - // So the parser won't skip them. - $itemcodes = array( - '*' => '', - '@' => 'disc', - '+' => 'square', - 'x' => 'square', - '#' => 'square', - 'o' => 'circle', - 'O' => 'circle', - '0' => 'circle', - ); - if (!isset($disabled['li']) && !isset($disabled['list'])) - { - foreach ($itemcodes as $c => $dummy) - $bbc_codes[$c] = array(); - } - - // Inside these tags autolink is not recommendable. - $no_autolink_tags = array( - 'url', - 'iurl', - 'ftp', - 'email', - ); - - // Shhhh! - if (!isset($disabled['color'])) - { - $codes[] = array( - 'tag' => 'chrissy', - 'before' => '', - 'after' => ' :-*', - ); - $codes[] = array( - 'tag' => 'kissy', - 'before' => '', - 'after' => ' :-*', - ); - } - - foreach ($codes as $c) - $bbc_codes[substr($c['tag'], 0, 1)][] = $c; - $codes = null; - } - - // Shall we take the time to cache this? - if ($cache_id != '' && !empty($modSettings['cache_enable']) && (($modSettings['cache_enable'] >= 2 && strlen($message) > 1000) || strlen($message) > 2400)) - { - // It's likely this will change if the message is modified. - $cache_key = 'parse:' . $cache_id . '-' . md5(md5($message) . '-' . $smileys . (empty($disabled) ? '' : implode(',', array_keys($disabled))) . safe_serialize($context['browser']) . $txt['lang_locale'] . $user_info['time_offset'] . $user_info['time_format']); - - if (($temp = cache_get_data($cache_key, 600)) != null) - return $temp; - - $cache_t = microtime(); - } - - if ($smileys === 'print') - { - // [glow], [shadow], and [move] can't really be printed. - $disabled['glow'] = true; - $disabled['shadow'] = true; - $disabled['move'] = true; - - // Colors can't well be displayed... supposed to be black and white. - $disabled['color'] = true; - $disabled['black'] = true; - $disabled['blue'] = true; - $disabled['white'] = true; - $disabled['red'] = true; - $disabled['green'] = true; - $disabled['me'] = true; - - // Color coding doesn't make sense. - $disabled['php'] = true; - - // Links are useless on paper... just show the link. - $disabled['ftp'] = true; - $disabled['url'] = true; - $disabled['iurl'] = true; - $disabled['email'] = true; - $disabled['flash'] = true; - - // !!! Change maybe? - if (!isset($_GET['images'])) - $disabled['img'] = true; - - // !!! Interface/setting to add more? - } - - if($local_disable) - foreach($local_disable as $d) - $disabled[$d] = true; - - $open_tags = array(); - $message = strtr($message, array("\n" => '
    ')); - - // The non-breaking-space looks a bit different each time. - $context['server']['complex_preg_chars'] = true; - $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{C2A0}' : chr(0xC2) . chr(0xA0)) : '\xA0'; - - $pos = -1; - while ($pos !== false) - { - // theymos - prevent various infinite loops - if($pos>90000) { - if(!isset($loopcount)) - $loopcount=0; - $loopcount++; - if($loopcount > 500) - return 'INVALID BBCODE: loop, probably unclosed tags'; - } - - $last_pos = isset($last_pos) ? max($pos, $last_pos) : $pos; - $pos = strpos($message, '[', $pos + 1); - - // Failsafe. - if ($pos === false || $last_pos > $pos) - $pos = strlen($message) + 1; - - // Can't have a one letter smiley, URL, or email! (sorry.) - if ($last_pos < $pos - 1) - { - // We want to eat one less, and one more, character (for smileys.) - $last_pos = max($last_pos - 1, 0); - $data = substr($message, $last_pos, $pos - $last_pos + 1); - - // Take care of some HTML! - if (!empty($modSettings['enablePostHTML']) && strpos($data, '<') !== false) - { - $data = preg_replace('~<a\s+href=((?:")?)((?:https?://|ftps?://|mailto:|bitcoin:)\S+?)\\1>~i', '[url=$2]', $data); - $data = preg_replace('~</a>~i', '[/url]', $data); - - //
    should be empty. - $empty_tags = array('br', 'hr'); - foreach ($empty_tags as $tag) - $data = str_replace(array('<' . $tag . '>', '<' . $tag . '/>', '<' . $tag . ' />'), '[' . $tag . ' /]', $data); - - // b, u, i, s, pre... basic tags. - $closable_tags = array('b', 'u', 'i', 's', 'em', 'ins', 'del', 'pre', 'blockquote'); - foreach ($closable_tags as $tag) - { - $diff = substr_count($data, '<' . $tag . '>') - substr_count($data, '</' . $tag . '>'); - $data = strtr($data, array('<' . $tag . '>' => '<' . $tag . '>', '</' . $tag . '>' => '')); - - if ($diff > 0) - $data .= str_repeat('', $diff); - } - - // Do - with security... action= -> action-. - preg_match_all('~<img\s+src=((?:")?)((?:https?://|ftps?://)\S+?)\\1(?:\s+alt=(".*?"|\S*?))?(?:\s?/)?>~i', $data, $matches, PREG_PATTERN_ORDER); - if (!empty($matches[0])) - { - $replaces = array(); - foreach ($matches[2] as $match => $imgtag) - { - $alt = empty($matches[3][$match]) ? '' : ' alt=' . preg_replace('~^"|"$~', '', $matches[3][$match]); - - // Remove action= from the URL - no funny business, now. - if (preg_match('~action(=|%3d)(?!dlattach)~i', $imgtag) != 0) - $imgtag = preg_replace('~action(=|%3d)(?!dlattach)~i', 'action-', $imgtag); - - // Check if the image is larger than allowed. - if (!empty($modSettings['max_image_width']) && !empty($modSettings['max_image_height'])) - { - list ($width, $height) = url_image_size($imgtag); - - if (!empty($modSettings['max_image_width']) && $width > $modSettings['max_image_width']) - { - $height = (int) (($modSettings['max_image_width'] * $height) / $width); - $width = $modSettings['max_image_width']; - } - - if (!empty($modSettings['max_image_height']) && $height > $modSettings['max_image_height']) - { - $width = (int) (($modSettings['max_image_height'] * $width) / $height); - $height = $modSettings['max_image_height']; - } - - // Set the new image tag. - $replaces[$matches[0][$match]] = '[img width=' . $width . ' height=' . $height . $alt . ']' . $imgtag . '[/img]'; - } - else - $replaces[$matches[0][$match]] = '[img' . $alt . ']' . $imgtag . '[/img]'; - } - - $data = strtr($data, $replaces); - } - } - - if (!empty($modSettings['autoLinkUrls'])) - { - // Are we inside tags that should be auto linked? - $no_autolink_area = false; - if (!empty($open_tags)) - { - foreach ($open_tags as $open_tag) - if (in_array($open_tag['tag'], $no_autolink_tags)) - $no_autolink_area = true; - } - - // Don't go backwards. - //!!! Don't think is the real solution.... - $lastAutoPos = isset($lastAutoPos) ? $lastAutoPos : 0; - if ($pos < $lastAutoPos) - $no_autolink_area = true; - $lastAutoPos = $pos; - - if (!$no_autolink_area) - { - // Parse any URLs.... have to get rid of the @ problems some things cause... stupid email addresses. - if (!isset($disabled['url']) && (strpos($data, '://') !== false || strpos($data, 'www.') !== false || strpos($data,'bitcoin:') !==false)) - { - // Switch out quotes really quick because they can cause problems. - $data = strtr($data, array(''' => '\'', ' ' => $context['utf8'] ? "\xC2\xA0" : "\xA0", '"' => '>">', '"' => '<"<', '<' => '\.(;\'"]|' . $nbsp . '|^)((?:http|https|ftp|ftps)://[\w\-_%@:|]+(?:\.[\w\-_%]+)*(?::\d+)?(?:/[\w\-_\~%\.@,\?&;=#(){}+:\'\\\\]*)*[/\w\-_\~%@\?;=#}\\\\])~i', - '~(?<=[\s>(;\'<]|' . $nbsp . '|^)(www(?:\.[\w\-_]+)+(?::\d+)?(?:/[\w\-_\~%\.@,\?&;=#(){}+:\'\\\\]*)*[/\w\-_\~%@\?;=#}\\\\])~i', - '~bitcoin:([-A-Za-z0-9._:/?#!%@$()*+,;=]{25,})~i' - ), array( - '[url]$1[/url]', - '[url=http://$1]$1[/url]', - '[url]bitcoin:$1[/url]' - ), $data))) - $data = $result; - - $data = strtr($data, array('\'' => ''', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ', '>">' => '"', '<"<' => '"', ' '<')); - } - - // Next, emails... - if (!isset($disabled['email']) && strpos($data, '@') !== false) - { - $data = preg_replace('~(?<=[\?\s' . $non_breaking_space . '\[\]()*\\\;>]|^)([\w\-\.]{1,80}@[\w\-]+\.[\w\-\.]+[\w\-])(?=[?,\s' . $non_breaking_space . '\[\]()*\\\]|$|
    | |>|<|"|'|\.(?:\.|;| |\s|$|
    ))~' . ($context['utf8'] ? 'u' : ''), '[email]$1[/email]', $data); - $data = preg_replace('~(?<=
    )([\w\-\.]{1,80}@[\w\-]+\.[\w\-\.]+[\w\-])(?=[?\.,;\s' . $non_breaking_space . '\[\]()*\\\]|$|
    | |>|<|"|')~' . ($context['utf8'] ? 'u' : ''), '[email]$1[/email]', $data); - // theymos - infinite loop - if($pos > 1000 && strpos(substr($message, $pos-100), '[email][email][email][email]') !== false) - return 'INVALID BBCODE: loop, probably unclosed tags (2)'; - } - } - } - - $data = strtr($data, array("\t" => '   ')); - - if (!empty($modSettings['fixLongWords']) && $modSettings['fixLongWords'] > 5) - { - // This is SADLY and INCREDIBLY browser dependent. - if ($context['browser']['is_gecko'] || $context['browser']['is_konqueror']) - $breaker = ' '; - // Opera... - elseif ($context['browser']['is_opera']) - $breaker = ' '; - // Internet Explorer... - else - $breaker = ' '; - - // PCRE will not be happy if we don't give it a short. - $modSettings['fixLongWords'] = (int) min(65535, $modSettings['fixLongWords']); - - // The idea is, find words xx long, and then replace them with xx + space + more. - if (strlen($data) > $modSettings['fixLongWords']) - { - // This is done in a roundabout way because $breaker has "long words" :P. - $data = strtr($data, array($breaker => '< >', ' ' => $context['utf8'] ? "\xC2\xA0" : "\xA0")); - $data = preg_replace_callback( - '~(?<=[>;:!? ' . $non_breaking_space . '\]()]|^)([\w' . ($context['utf8'] ? '\pL' : '') . '\.]{' . $modSettings['fixLongWords'] . ',})~' . ($context['utf8'] ? 'u' : ''), - 'word_break__preg_callback', - $data); - $data = strtr($data, array('< >' => $breaker, $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); - } - } - - // Do any smileys! - if ($smileys === true) - parsesmileys($data); - - // If it wasn't changed, no copying or other boring stuff has to happen! - if ($data != substr($message, $last_pos, $pos - $last_pos + 1)) - { - $message = substr($message, 0, $last_pos) . $data . substr($message, $pos + 1); - - // Since we changed it, look again incase we added or removed a tag. But we don't want to skip any. - $old_pos = strlen($data) + $last_pos - 1; - $pos = strpos($message, '[', $last_pos); - $pos = $pos === false ? $old_pos : min($pos, $old_pos); - } - } - - // Are we there yet? Are we there yet? - if ($pos >= strlen($message) - 1) - break; - - $tags = strtolower(substr($message, $pos + 1, 1)); - - if ($tags == '/' && !empty($open_tags)) - { - $pos2 = strpos($message, ']', $pos + 1); - if ($pos2 == $pos + 2) - continue; - $look_for = strtolower(substr($message, $pos + 2, $pos2 - $pos - 2)); - - $to_close = array(); - $block_level = null; - do - { - $tag = array_pop($open_tags); - if (!$tag) - break; - - if (!empty($tag['block_level'])) - { - // Only find out if we need to. - if ($block_level === false) - { - array_push($open_tags, $tag); - break; - } - - // The idea is, if we are LOOKING for a block level tag, we can close them on the way. - if (strlen($look_for) > 0 && isset($bbc_codes[$look_for[0]])) - { - foreach ($bbc_codes[$look_for[0]] as $temp) - if ($temp['tag'] == $look_for) - { - $block_level = !empty($temp['block_level']); - break; - } - } - - if ($block_level !== true) - { - $block_level = false; - array_push($open_tags, $tag); - break; - } - } - - $to_close[] = $tag; - } - while ($tag['tag'] != $look_for); - - // Did we just eat through everything and not find it? - if ((empty($open_tags) && (empty($tag) || $tag['tag'] != $look_for))) - { - $tablewarn = false; - for($cc=count($to_close)-1;$cc>=0;$cc--) { - if($to_close[$cc]['tag'] == 'table') - $tablewarn = true; - if($tablewarn && (in_array($to_close[$cc]['tag'], array('td', 'tr')))) - return 'INVALID BBCODE: close of unopened tag in table (2)'; - } - unset($cc, $tablewarn); - $open_tags = $to_close; - continue; - } - elseif (!empty($to_close) && $tag['tag'] != $look_for) - { - if ($block_level === null && isset($look_for[0], $bbc_codes[$look_for[0]])) - { - foreach ($bbc_codes[$look_for[0]] as $temp) - if ($temp['tag'] == $look_for) - { - $block_level = !empty($temp['block_level']); - break; - } - } - - // We're not looking for a block level tag (or maybe even a tag that exists...) - if (!$block_level) - { - foreach ($to_close as $tag) - array_push($open_tags, $tag); - continue; - } - } - - foreach ($to_close as $tag) - { - $message = substr($message, 0, $pos) . $tag['after'] . substr($message, $pos2 + 1); - $pos += strlen($tag['after']); - $pos2 = $pos - 1; - - // See the comment at the end of the big loop - just eating whitespace ;). - if (!empty($tag['block_level']) && substr($message, $pos, 6) == '
    ') - $message = substr($message, 0, $pos) . substr($message, $pos + 6); - if (!empty($tag['trim']) && $tag['trim'] != 'inside' && preg_match('~(
    | |\s)*~', substr($message, $pos), $matches) != 0) - $message = substr($message, 0, $pos) . substr($message, $pos + strlen($matches[0])); - } - - if (!empty($to_close)) - { - $to_close = array(); - $pos--; - } - - continue; - } - - // No tags for this character, so just keep going (fastest possible course.) - if (!isset($bbc_codes[$tags])) - continue; - - $inside = empty($open_tags) ? null : $open_tags[count($open_tags) - 1]; - $tag = null; - foreach ($bbc_codes[$tags] as $possible) - { - // Not a match? - if (strtolower(substr($message, $pos + 1, strlen($possible['tag']))) != $possible['tag']) - continue; - - $next_c = substr($message, $pos + 1 + strlen($possible['tag']), 1); - - // A test validation? - if (isset($possible['test']) && preg_match('~^' . $possible['test'] . '~', substr($message, $pos + 1 + strlen($possible['tag']) + 1)) == 0) - continue; - // Do we want parameters? - elseif (!empty($possible['parameters'])) - { - if ($next_c != ' ') - continue; - } - elseif (isset($possible['type'])) - { - // Do we need an equal sign? - if (in_array($possible['type'], array('unparsed_equals', 'unparsed_commas', 'unparsed_commas_content', 'unparsed_equals_content', 'parsed_equals')) && $next_c != '=') - continue; - // Maybe we just want a /... - if ($possible['type'] == 'closed' && $next_c != ']' && substr($message, $pos + 1 + strlen($possible['tag']), 2) != '/]' && substr($message, $pos + 1 + strlen($possible['tag']), 3) != ' /]') - continue; - // An immediate ]? - if ($possible['type'] == 'unparsed_content' && $next_c != ']') - continue; - } - // No type means 'parsed_content', which demands an immediate ] without parameters! - elseif ($next_c != ']') - continue; - - // Check allowed tree? - if (isset($possible['require_parents']) && ($inside === null || !in_array($inside['tag'], $possible['require_parents']))) - continue; - elseif (isset($inside['require_children']) && !in_array($possible['tag'], $inside['require_children'])) - continue; - // If this is in the list of disallowed child tags, don't parse it. - elseif (isset($inside['disallow_children']) && in_array($possible['tag'], $inside['disallow_children'])) - continue; - - $pos1 = $pos + 1 + strlen($possible['tag']) + 1; - - // This is long, but it makes things much easier and cleaner. - if (!empty($possible['parameters'])) - { - $preg = array(); - foreach ($possible['parameters'] as $p => $info) - $preg[] = '(\s+' . $p . '=' . (empty($info['quoted']) ? '' : '"') . (isset($info['match']) ? $info['match'] : '(.+?)') . (empty($info['quoted']) ? '' : '"') . ')' . (empty($info['optional']) ? '' : '?'); - - // Okay, this may look ugly and it is, but it's not going to happen much and it is the best way of allowing any order of parameters but still parsing them right. - $match = false; - $orders = permute($preg); - foreach ($orders as $p) - if (preg_match('~^' . implode('', $p) . '\]~i', substr($message, $pos1 - 1), $matches) != 0) - { - $match = true; - break; - } - - // Didn't match our parameter list, try the next possible. - if (!$match) - continue; - - $params = array(); - for ($i = 1, $n = count($matches); $i < $n; $i += 2) - { - $key = strtok(ltrim($matches[$i]), '='); - if (isset($possible['parameters'][$key]['value'])) - $params['{' . $key . '}'] = strtr($possible['parameters'][$key]['value'], array('$1' => $matches[$i + 1])); - elseif (isset($possible['parameters'][$key]['validate'])) - $params['{' . $key . '}'] = $possible['parameters'][$key]['validate']($matches[$i + 1]); - else - $params['{' . $key . '}'] = $matches[$i + 1]; - - // Just to make sure: replace any $ or { so they can't interpolate wrongly. - $params['{' . $key . '}'] = strtr($params['{' . $key . '}'], array('$' => '$', '{' => '{')); - } - - foreach ($possible['parameters'] as $p => $info) - { - if (!isset($params['{' . $p . '}'])) - $params['{' . $p . '}'] = ''; - } - - $tag = $possible; - - // Put the parameters into the string. - if (isset($tag['before'])) - $tag['before'] = strtr($tag['before'], $params); - if (isset($tag['after'])) - $tag['after'] = strtr($tag['after'], $params); - if (isset($tag['content'])) - $tag['content'] = strtr($tag['content'], $params); - - $pos1 += strlen($matches[0]) - 1; - } - else - $tag = $possible; - break; - } - - // Item codes are complicated buggers... they are implicit [li]s and can make [list]s! - if ($smileys !== false && $tag === null && isset($itemcodes[substr($message, $pos + 1, 1)]) && substr($message, $pos + 2, 1) == ']' && !isset($disabled['list']) && !isset($disabled['li'])) - { - if (substr($message, $pos + 1, 1) == '0' && !in_array(substr($message, $pos - 1, 1), array(';', ' ', "\t", '>'))) - continue; - $tag = $itemcodes[substr($message, $pos + 1, 1)]; - - // First let's set up the tree: it needs to be in a list, or after an li. - if ($inside === null || ($inside['tag'] != 'list' && $inside['tag'] != 'li')) - { - $open_tags[] = array( - 'tag' => 'list', - 'after' => '', - 'block_level' => true, - 'require_children' => array('li'), - 'disallow_children' => isset($inside['disallow_children']) ? $inside['disallow_children'] : null, - ); - $code = '
      '; - } - // We're in a list item already: another itemcode? Close it first. - elseif ($inside['tag'] == 'li') - { - array_pop($open_tags); - $code = ''; - } - else - $code = ''; - - // Now we open a new tag. - $open_tags[] = array( - 'tag' => 'li', - 'after' => '', - 'trim' => 'outside', - 'block_level' => true, - 'disallow_children' => isset($inside['disallow_children']) ? $inside['disallow_children'] : null, - ); - - // First, open the tag... - $code .= ''; - $message = substr($message, 0, $pos) . $code . substr($message, $pos + 3); - $pos += strlen($code) - 1; - - // Next, find the next break (if any.) If there's more itemcode after it, keep it going - otherwise close! - $pos2 = strpos($message, '
      ', $pos); - $pos3 = strpos($message, '[/', $pos); - if ($pos2 !== false && ($pos2 <= $pos3 || $pos3 === false)) - { - preg_match('~^(
      | |\s|\[)+~', substr($message, $pos2 + 6), $matches); - $message = substr($message, 0, $pos2) . (!empty($matches[0]) && substr($matches[0], -1) == '[' ? '[/li]' : '[/li][/list]') . substr($message, $pos2); - - $open_tags[count($open_tags) - 2]['after'] = '
    '; - } - // Tell the [list] that it needs to close specially. - else - { - if(count($open_tags)<2) { - return 'INVALID BBCODE: messed-up itemcodes'; - } - // Move the li over, because we're not sure what we'll hit. - $open_tags[count($open_tags) - 1]['after'] = ''; - $open_tags[count($open_tags) - 2]['after'] = ''; - } - - continue; - } - - // Implicitly close lists and tables if something other than what's required is in them. This is needed for itemcode. - if ($tag === null && $inside !== null && !empty($inside['require_children'])) - { - array_pop($open_tags); - - $message = substr($message, 0, $pos) . $inside['after'] . substr($message, $pos); - $pos += strlen($inside['after']) - 1; - } - - // No tag? Keep looking, then. Silly people using brackets without actual tags. - if ($tag === null) - continue; - - // Propagate the list to the child (so wrapping the disallowed tag won't work either.) - if (isset($inside['disallow_children'])) - $tag['disallow_children'] = isset($tag['disallow_children']) ? array_unique(array_merge($tag['disallow_children'], $inside['disallow_children'])) : $inside['disallow_children']; - - // Is this tag disabled? - if (isset($disabled[$tag['tag']])) - { - if (!isset($tag['disabled_before']) && !isset($tag['disabled_after']) && !isset($tag['disabled_content'])) - { - $tag['before'] = !empty($tag['block_level']) ? '
    ' : ''; - $tag['after'] = !empty($tag['block_level']) ? '
    ' : ''; - $tag['content'] = isset($tag['type']) && $tag['type'] == 'closed' ? '' : (!empty($tag['block_level']) ? '
    $1
    ' : '$1'); - } - elseif (isset($tag['disabled_before']) || isset($tag['disabled_after'])) - { - $tag['before'] = isset($tag['disabled_before']) ? $tag['disabled_before'] : (!empty($tag['block_level']) ? '
    ' : ''); - $tag['after'] = isset($tag['disabled_after']) ? $tag['disabled_after'] : (!empty($tag['block_level']) ? '
    ' : ''); - } - else - $tag['content'] = $tag['disabled_content']; - } - - // The only special case is 'html', which doesn't need to close things. - if (!empty($tag['block_level']) && $tag['tag'] != 'html' && empty($inside['block_level'])) - { - $n = count($open_tags) - 1; - while (empty($open_tags[$n]['block_level']) && $n >= 0) - $n--; - - // Close all the non block level tags so this tag isn't surrounded by them. - for ($i = count($open_tags) - 1; $i > $n; $i--) - { - $message = substr($message, 0, $pos) . $open_tags[$i]['after'] . substr($message, $pos); - $pos += strlen($open_tags[$i]['after']); - $pos1 += strlen($open_tags[$i]['after']); - - // Trim or eat trailing stuff... see comment at the end of the big loop. - if (!empty($open_tags[$i]['block_level']) && substr($message, $pos, 6) == '
    ') - $message = substr($message, 0, $pos) . substr($message, $pos + 6); - if (!empty($open_tags[$i]['trim']) && $tag['trim'] != 'inside' && preg_match('~(
    | |\s)*~', substr($message, $pos), $matches) != 0) - $message = substr($message, 0, $pos) . substr($message, $pos + strlen($matches[0])); - - array_pop($open_tags); - } - } - - // No type means 'parsed_content'. - if (!isset($tag['type'])) - { - // !!! Check for end tag first, so people can say "I like that [i] tag"? - $open_tags[] = $tag; - $message = substr($message, 0, $pos) . $tag['before'] . substr($message, $pos1); - $pos += strlen($tag['before']) - 1; - } - // Don't parse the content, just skip it. - elseif ($tag['type'] == 'unparsed_content') - { - $pos2 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos1); - if ($pos2 === false) - continue; - - $data = substr($message, $pos1, $pos2 - $pos1); - - if (!empty($tag['block_level']) && substr($data, 0, 6) == '
    ') - $data = substr($data, 6); - - if (isset($tag['validate'])) - $tag['validate']($tag, $data, $disabled); - - $code = strtr($tag['content'], array('$1' => $data)); - $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + 3 + strlen($tag['tag'])); - $pos += strlen($code) - 1; - } - // Don't parse the content, just skip it. - elseif ($tag['type'] == 'unparsed_equals_content') - { - // The value may be quoted for some tags - check. - if (isset($tag['quoted'])) - { - $quoted = substr($message, $pos1, 6) == '"'; - if ($tag['quoted'] != 'optional' && !$quoted) - continue; - - if ($quoted) - $pos1 += 6; - } - else - $quoted = false; - - $pos2 = strpos($message, $quoted == false ? ']' : '"]', $pos1); - if ($pos2 === false) - continue; - $pos3 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos2); - if ($pos3 === false) - continue; - - $data = array( - substr($message, $pos2 + ($quoted == false ? 1 : 7), $pos3 - ($pos2 + ($quoted == false ? 1 : 7))), - substr($message, $pos1, $pos2 - $pos1) - ); - - if (!empty($tag['block_level']) && substr($data[0], 0, 6) == '
    ') - $data[0] = substr($data[0], 6); - - // Validation for my parking, please! - if (isset($tag['validate'])) - $tag['validate']($tag, $data, $disabled); - - $code = strtr($tag['content'], array('$1' => $data[0], '$2' => $data[1])); - $message = substr($message, 0, $pos) . $code . substr($message, $pos3 + 3 + strlen($tag['tag'])); - $pos += strlen($code) - 1; - } - // A closed tag, with no content or value. - elseif ($tag['type'] == 'closed') - { - $pos2 = strpos($message, ']', $pos); - $message = substr($message, 0, $pos) . $tag['content'] . substr($message, $pos2 + 1); - $pos += strlen($tag['content']) - 1; - } - // This one is sorta ugly... :/. Unforunately, it's needed for flash. - elseif ($tag['type'] == 'unparsed_commas_content') - { - $pos2 = strpos($message, ']', $pos1); - if ($pos2 === false) - continue; - $pos3 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos2); - if ($pos3 === false) - continue; - - // We want $1 to be the content, and the rest to be csv. - $data = explode(',', ',' . substr($message, $pos1, $pos2 - $pos1)); - $data[0] = substr($message, $pos2 + 1, $pos3 - $pos2 - 1); - - if (isset($tag['validate'])) - $tag['validate']($tag, $data, $disabled); - - $code = $tag['content']; - foreach ($data as $k => $d) - $code = strtr($code, array('$' . ($k + 1) => trim($d))); - $message = substr($message, 0, $pos) . $code . substr($message, $pos3 + 3 + strlen($tag['tag'])); - $pos += strlen($code) - 1; - } - // This has parsed content, and a csv value which is unparsed. - elseif ($tag['type'] == 'unparsed_commas') - { - $pos2 = strpos($message, ']', $pos1); - if ($pos2 === false) - continue; - - $data = explode(',', substr($message, $pos1, $pos2 - $pos1)); - - if (isset($tag['validate'])) - $tag['validate']($tag, $data, $disabled); - - // Fix after, for disabled code mainly. - foreach ($data as $k => $d) - $tag['after'] = strtr($tag['after'], array('$' . ($k + 1) => trim($d))); - - $open_tags[] = $tag; - - // Replace them out, $1, $2, $3, $4, etc. - $code = $tag['before']; - foreach ($data as $k => $d) - $code = strtr($code, array('$' . ($k + 1) => trim($d))); - $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + 1); - $pos += strlen($code) - 1; - } - // A tag set to a value, parsed or not. - elseif ($tag['type'] == 'unparsed_equals' || $tag['type'] == 'parsed_equals') - { - // The value may be quoted for some tags - check. - if (isset($tag['quoted'])) - { - $quoted = substr($message, $pos1, 6) == '"'; - if ($tag['quoted'] != 'optional' && !$quoted) - continue; - - if ($quoted) - $pos1 += 6; - } - else - $quoted = false; - - $pos2 = strpos($message, $quoted == false ? ']' : '"]', $pos1); - if ($pos2 === false) - continue; - - $data = substr($message, $pos1, $pos2 - $pos1); - - // Validation for my parking, please! - if (isset($tag['validate'])) - $tag['validate']($tag, $data, $disabled); - - // For parsed content, we must recurse to avoid security problems. - if ($tag['type'] != 'unparsed_equals') - $data = parse_bbc($data); - - $tag['after'] = strtr($tag['after'], array('$1' => $data)); - - $open_tags[] = $tag; - - $code = strtr($tag['before'], array('$1' => $data)); - $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + ($quoted == false ? 1 : 7)); - $pos += strlen($code) - 1; - } - - // If this is block level, eat any breaks after it. - if (!empty($tag['block_level']) && substr($message, $pos + 1, 6) == '
    ') - $message = substr($message, 0, $pos + 1) . substr($message, $pos + 7); - - // Are we trimming outside this tag? - if (!empty($tag['trim']) && $tag['trim'] != 'outside' && preg_match('~(
    | |\s)*~', substr($message, $pos + 1), $matches) != 0) - $message = substr($message, 0, $pos + 1) . substr($message, $pos + 1 + strlen($matches[0])); - } - - // Close any remaining tags. - while ($tag = array_pop($open_tags)) { - if(in_array($tag['tag'], array('table','td','tr','th'))) - return 'INVALID BBCODE: close of unopened tag in table (1)'; - $message .= $tag['after']; - } - - if (substr($message, 0, 1) == ' ') - $message = ' ' . substr($message, 1); - - // Cleanup whitespace. - $message = strtr($message, array(' ' => '  ', "\r" => '', "\n" => '
    ', '
    ' => '
     ', ' ' => "\n")); - - if(in_array('ugc', $local_disable)) - $message = str_replace(' 0.05) - cache_put_data($cache_key, $message, 600); - - // ECHO "d"; - - return $message; + global $txt, $scripturl, $context, $modSettings, $user_info; + static $bbc_codes = array(), $itemcodes = array(), $no_autolink_tags = array(); + $disabled = array(); + + //theymos - die if it's taking way too long + if(isset($_SERVER["REQUEST_TIME_FLOAT"]) && microtime(true) - $_SERVER["REQUEST_TIME_FLOAT"] > 29 && strlen($message)>500 && php_sapi_name() != 'cli') { + die(); + } + + // Never show smileys for wireless clients. More bytes, can't see it anyway :P. + if (WIRELESS) + $smileys = false; + elseif ($smileys !== null && ($smileys == '1' || $smileys == '0')) + $smileys = (bool) $smileys; + + if (empty($modSettings['enableBBC']) && $message !== false) + { + if ($smileys === true) + parsesmileys($message); + + return $message; + } + + // Just in case it wasn't determined yet whether UTF-8 is enabled. + if (!isset($context['utf8'])) + $context['utf8'] = (empty($modSettings['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']) === 'UTF-8'; + + //theymos - disable links and images on pages where we don't want to send a referer to random people + $disabledsecurity=''; + if(isset($_GET['sesc'])) { + $cache_id = ''; + $disabled['img']=true; + $disabled['iurl']=true; + $disabled['url']=true; + $disabled['ftp']=true; + $disabledsecurity=' (FORUM: disabled on this page for security.)'; + } + if(isset($_GET['patrol'])) { + $cache_id = ''; + $disabled['black']=true; + $disabled['color']=true; + } + + //theymos - these tags are aways disabled + $disabled['flash'] = true; + $disabled['move'] = true; + + // Sift out the bbc for a performance improvement. + if (empty($bbc_codes) || $message === false) + { + /*if (!empty($modSettings['disabledBBC'])) + { + $temp = explode(',', strtolower($modSettings['disabledBBC'])); + + foreach ($temp as $tag) + $disabled[trim($tag)] = true; + } + + if (empty($modSettings['enableEmbeddedFlash'])) + $disabled['flash'] = true;*/ + + /* The following bbc are formatted as an array, with keys as follows: + + tag: the tag's name - should be lowercase! + + type: one of... + - (missing): [tag]parsed content[/tag] + - unparsed_equals: [tag=xyz]parsed content[/tag] + - parsed_equals: [tag=parsed data]parsed content[/tag] + - unparsed_content: [tag]unparsed content[/tag] + - closed: [tag], [tag/], [tag /] + - unparsed_commas: [tag=1,2,3]parsed content[/tag] + - unparsed_commas_content: [tag=1,2,3]unparsed content[/tag] + - unparsed_equals_content: [tag=...]unparsed content[/tag] + + parameters: an optional array of parameters, for the form + [tag abc=123]content[/tag]. The array is an associative array + where the keys are the parameter names, and the values are an + array which may contain the following: + - match: a regular expression to validate and match the value. + - quoted: true if the value should be quoted. + - validate: callback to evaluate on the data, which is $data. + - value: a string in which to replace $1 with the data. + either it or validate may be used, not both. + - optional: true if the parameter is optional. + + test: a regular expression to test immediately after the tag's + '=', ' ' or ']'. Typically, should have a \] at the end. + Optional. + + content: only available for unparsed_content, closed, + unparsed_commas_content, and unparsed_equals_content. + $1 is replaced with the content of the tag. Parameters + are repalced in the form {param}. For unparsed_commas_content, + $2, $3, ..., $n are replaced. + + before: only when content is not used, to go before any + content. For unparsed_equals, $1 is replaced with the value. + For unparsed_commas, $1, $2, ..., $n are replaced. + + after: similar to before in every way, except that it is used + when the tag is closed. + + disabled_content: used in place of content when the tag is + disabled. For closed, default is '', otherwise it is '$1' if + block_level is false, '
    $1
    ' elsewise. + + disabled_before: used in place of before when disabled. Defaults + to '
    ' if block_level, '' if not. + + disabled_after: used in place of after when disabled. Defaults + to '
    ' if block_level, '' if not. + + block_level: set to true the tag is a "block level" tag, similar + to HTML. Block level tags cannot be nested inside tags that are + not block level, and will not be implicitly closed as easily. + One break following a block level tag may also be removed. + + trim: if set, and 'inside' whitespace after the begin tag will be + removed. If set to 'outside', whitespace after the end tag will + meet the same fate. + + validate: except when type is missing or 'closed', a callback to + validate the data as $data. Depending on the tag's type, $data + may be a string or an array of strings (corresponding to the + replacement.) + + quoted: when type is 'unparsed_equals' or 'parsed_equals' only, + may be not set, 'optional', or 'required' corresponding to if + the content may be quoted. This allows the parser to read + [tag="abc]def[esdf]"] properly. + + require_parents: an array of tag names, or not set. If set, the + enclosing tag *must* be one of the listed tags, or parsing won't + occur. + + require_children: similar to require_parents, if set children + won't be parsed if they are not in the list. + + disallow_children: similar to, but very different from, + require_children, if it is set the listed tags will not be + parsed inside the tag. + */ + + $codes = array( + array( + 'tag' => 'abbr', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'quoted' => 'optional', + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'acronym', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'quoted' => 'optional', + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'anchor', + 'type' => 'unparsed_equals', + 'test' => '[#]?([A-Za-z][A-Za-z0-9_\-]*)\]', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'b', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'black', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'blue', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'br', + 'type' => 'closed', + 'content' => '
    ', + ), + array( + 'tag' => 'btc', + 'type' => 'closed', + 'content' => 'BTC', + ), + array( + 'tag' => 'code', + 'type' => 'unparsed_content', + 'content' => '
    ' . $txt['smf238'] . ':
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', + // !!! Maybe this can be simplified? + 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { + global $context; + + if (!isset($disabled['code'])) + { + $php_parts = preg_split('~(<\?php|\?>)~', $data, -1, PREG_SPLIT_DELIM_CAPTURE); + + for ($php_i = 0, $php_n = count($php_parts); $php_i < $php_n; $php_i++) + { + // Do PHP code coloring? + if ($php_parts[$php_i] != '<?php') + continue; + + $php_string = ''; + while ($php_i + 1 < count($php_parts) && $php_parts[$php_i] != '?>') + { + $php_string .= $php_parts[$php_i]; + $php_parts[$php_i++] = ''; + } + $php_parts[$php_i] = highlight_php_code($php_string . $php_parts[$php_i]); + } + + // Fix the PHP code stuff... + $data = str_replace("
    \t
    ", "\t", implode('', $php_parts)); + + // Older browsers are annoying, aren't they? + if ($context['browser']['is_ie4'] || $context['browser']['is_ie5'] || $context['browser']['is_ie5.5']) + $data = str_replace("\t", "
    \t
    ", $data); + elseif (!$context['browser']['is_gecko']) + $data = str_replace("\t", "\t", $data); + }}, + 'block_level' => true, + ), + array( + 'tag' => 'code', + 'type' => 'unparsed_equals_content', + 'content' => '
    ' . $txt['smf238'] . ': ($2)
    ' . ($context['browser']['is_gecko'] ? '
    $1
    ' : '$1') . '
    ', + // !!! Maybe this can be simplified? + 'validate' => isset($disabled['code']) ? null : function(&$tag, &$data, $disabled) { + global $context; + + if (!isset($disabled['code'])) + { + $php_parts = preg_split('~(<\?php|\?>)~', $data[0], -1, PREG_SPLIT_DELIM_CAPTURE); + + for ($php_i = 0, $php_n = count($php_parts); $php_i < $php_n; $php_i++) + { + // Do PHP code coloring? + if ($php_parts[$php_i] != '<?php') + continue; + + $php_string = ''; + while ($php_i + 1 < count($php_parts) && $php_parts[$php_i] != '?>') + { + $php_string .= $php_parts[$php_i]; + $php_parts[$php_i++] = ''; + } + $php_parts[$php_i] = highlight_php_code($php_string . $php_parts[$php_i]); + } + + // Fix the PHP code stuff... + $data[0] = str_replace("
    \t
    ", "\t", implode('', $php_parts)); + + // Older browsers are annoying, aren't they? + if ($context['browser']['is_ie4'] || $context['browser']['is_ie5'] || $context['browser']['is_ie5.5']) + $data = str_replace("\t", "
    \t
    ", $data); + elseif (!$context['browser']['is_gecko']) + $data = str_replace("\t", "\t", $data); + }}, + 'block_level' => true, + ), + array( + 'tag' => 'center', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'color', + 'type' => 'unparsed_equals', + 'test' => '(#[\da-fA-F]{3}|#[\da-fA-F]{6}|[A-Za-z]{1,12})\]', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'email', + 'type' => 'unparsed_content', + 'content' => '    $1', + // !!! Should this respect guest_hideContacts? + 'validate' => function(&$tag, &$data, $disabled) {$data = strtr($data, array('
    ' => ''));}, + ), + array( + 'tag' => 'email', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + // !!! Should this respect guest_hideContacts? + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'ftp', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
    ' => '')); + if (strpos($data, 'ftp://') !== 0 && strpos($data, 'ftps://') !== 0) + $data = 'ftp://' . $data; + }, + ), + array( + 'tag' => 'ftp', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'validate' => function(&$tag, &$data, $disabled) { + if (strpos($data, 'ftp://') !== 0 && strpos($data, 'ftps://') !== 0) + $data = 'ftp://' . $data; + }, + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'font', + 'type' => 'unparsed_equals', + 'test' => '[A-Za-z0-9_,\-\s]+?\]', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'flash', + 'type' => 'unparsed_commas_content', + 'test' => '\d+,\d+\]', + 'content' => ($context['browser']['is_ie'] && !$context['browser']['is_mac_ie'] ? '<a href="$1">$1</a>' : '<a href="$1">$1</a>'), + 'validate' => function(&$tag, &$data, $disabled) { + if (isset($disabled['url'])) + $tag['content'] = '$1'; + elseif (strpos($data[0], 'http://') !== 0 && strpos($data[0], 'https://') !== 0) + $data[0] = 'http://' . $data[0]; + }, + 'disabled_content' => $disabledsecurity ? '$1': '$1', + ), + array( + 'tag' => 'green', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'glow', + 'type' => 'unparsed_commas', + 'test' => '[#0-9a-zA-Z\-]{3,12},([012]\d{1,2}|\d{1,2})(,[^]]+)?\]', + 'before' => $context['browser']['is_ie'] ? '
    ' : '', + 'after' => $context['browser']['is_ie'] ? '
    ' : '    ', + ), + array( + 'tag' => 'hr', + 'type' => 'closed', + 'content' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'html', + 'type' => 'unparsed_content', + 'content' => '$1', + 'block_level' => true, + 'disabled_content' => '$1', + ), + array( + 'tag' => 'img', + 'type' => 'unparsed_content', + 'parameters' => array( + 'alt' => array('optional' => true), + 'width' => array('optional' => true, 'value' => ' width="$1"', 'match' => '(\d{1,4})'), + 'height' => array('optional' => true, 'value' => ' height="$1"', 'match' => '(\d{1,4})'), + ), + 'content' => '{alt}', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
    ' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0) + $data = 'http://' . $data; + if(!isset($disabled['img'])) + $data = proxyurl($data); + }, + 'disabled_content' => $disabledsecurity ? ('($1)'.$disabledsecurity) : '$1', + ), + array( + 'tag' => 'img', + 'type' => 'unparsed_content', + 'content' => '', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
    ' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0) + $data = 'http://' . $data; + if(!isset($disabled['img'])) + $data = proxyurl($data); + }, + 'disabled_content' => $disabledsecurity ? ('($1)'.$disabledsecurity) : '$1', + ), + array( + 'tag' => 'i', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'iurl', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
    ' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + ), + array( + 'tag' => 'iurl', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'validate' => function(&$tag, &$data, $disabled) { + if (substr($data, 0, 1) == '#') + $data = '#post_' . substr($data, 1); + elseif (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'li', + 'before' => '
  • ', + 'after' => '
    • ', + 'trim' => 'outside', + 'require_parents' => array('list'), + 'block_level' => true, + 'disabled_before' => '', + 'disabled_after' => '
    ', + ), + array( + 'tag' => 'list', + 'before' => '
      ', + 'after' => '
    ', + 'trim' => 'inside', + 'require_children' => array('li'), + 'block_level' => true, + ), + array( + 'tag' => 'list', + 'parameters' => array( + 'type' => array('match' => '(none|disc|circle|square|decimal|decimal-leading-zero|lower-roman|upper-roman|lower-alpha|upper-alpha|lower-greek|lower-latin|upper-latin|hebrew|armenian|georgian|cjk-ideographic|hiragana|katakana|hiragana-iroha|katakana-iroha)'), + ), + 'before' => '
      ', + 'after' => '
    ', + 'trim' => 'inside', + 'require_children' => array('li'), + 'block_level' => true, + ), + array( + 'tag' => 'left', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'ltr', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'me', + 'type' => 'unparsed_equals', + 'before' => '
    * $1 ', + 'after' => '
    ', + 'quoted' => 'optional', + 'block_level' => true, + 'disabled_before' => '/me ', + 'disabled_after' => '
    ', + ), + array( + 'tag' => 'move', + 'before' => '', + 'after' => '', + 'block_level' => true, + ), + array( + 'tag' => 'nbsp', + 'type' => 'closed', + 'content' => ' ', + ), + array( + 'tag' => 'nobbc', + 'type' => 'unparsed_content', + 'content' => '$1', + ), + array( + 'tag' => 'pre', + 'before' => '
    ',
+        'after' => '
    ', + ), + array( + 'tag' => 'php', + 'type' => 'unparsed_content', + 'content' => '
    $1
    ', + 'validate' => isset($disabled['php']) ? null : function(&$tag, &$data, $disabled) { + if (!isset($disabled['php'])) + { + $add_begin = substr(trim($data), 0, 5) != '<?'; + $data = highlight_php_code($add_begin ? '<?php ' . $data . '?>' : $data); + if ($add_begin) + $data = preg_replace(array('~^(.+?)<\?.{0,40}?php( |\s)~', '~\?>((?:)*)$~'), '$1', $data, 2); + }}, + 'block_level' => true, + 'disabled_content' => '$1', + ), + array( + 'tag' => 'quote', + 'before' => '
    ' . $txt['smf240'] . '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'parameters' => array( + 'author' => array('match' => '(.{1,192}?)', 'quoted' => true, 'validate' => 'parse_bbc'), + ), + 'before' => '
    ' . $txt['smf239'] . ': {author}
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'type' => 'parsed_equals', + 'before' => '
    ' . $txt['smf239'] . ': $1
    ', + 'after' => '
    ', + 'quoted' => 'optional', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'parameters' => array( + 'author' => array('match' => '([^<>]{1,192}?)'), + 'link' => array('match' => '(?:board=\d+;)?((?:topic|threadid)=[\dmsg#\./]{1,40}(?:;start=[\dmsg#\./]{1,40})?|action=profile;u=\d+)'), + 'date' => array('match' => '(\d+)', 'validate' => 'timeformat'), + ), + 'before' => '
    ' . $txt['smf239'] . ': {author} ' . $txt[176] . ' {date}
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'quote', + 'parameters' => array( + 'author' => array('match' => '(.{1,192}?)', 'validate' => 'parse_bbc'), + ), + 'before' => '
    ' . $txt['smf239'] . ': {author}
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'right', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 'red', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'rtl', + 'before' => '
    ', + 'after' => '
    ', + 'block_level' => true, + ), + array( + 'tag' => 's', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'size', + 'type' => 'unparsed_equals', + 'test' => '([1-9][\d]?p[xt]|(?:x-)?small(?:er)?|(?:x-)?large[r]?)\]', + // !!! line-height + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'size', + 'type' => 'unparsed_equals', + 'test' => '[1-9]\]', + // !!! line-height + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'sub', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'sup', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'shadow', + 'type' => 'unparsed_commas', + 'test' => '[#0-9a-zA-Z\-]{3,12},(left|right|top|bottom|[0123]\d{0,2})\]', + 'before' => $context['browser']['is_ie'] ? '' : '', + 'after' => '', + 'validate' => $context['browser']['is_ie'] ? function(&$tag, &$data, $disabled) { + if ($data[1] == 'left') + $data[1] = 270; + elseif ($data[1] == 'right') + $data[1] = 90; + elseif ($data[1] == 'top') + $data[1] = 0; + elseif ($data[1] == 'bottom') + $data[1] = 180; + else + $data[1] = (int) $data[1];} : function(&$tag, &$data, $disabled) { + if ($data[1] == 'top' || (is_numeric($data[1]) && $data[1] < 50)) + return '0 -2px'; + elseif ($data[1] == 'right' || (is_numeric($data[1]) && $data[1] < 100)) + return '2px 0'; + elseif ($data[1] == 'bottom' || (is_numeric($data[1]) && $data[1] < 190)) + return '0 2px'; + elseif ($data[1] == 'left' || (is_numeric($data[1]) && $data[1] < 280)) + return '-2px 0'; + else + return '0 0';}, + ), + array( + 'tag' => 'time', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + if (is_numeric($data)) + $data = timeformat($data); + else + $tag['content'] = '[time]$1[/time]';}, + ), + array( + 'tag' => 'tt', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'table', + 'before' => '', + 'after' => '
    ', + 'trim' => 'inside', + 'require_children' => array('tr'), + 'block_level' => true, + ), + array( + 'tag' => 'tr', + 'before' => '', + 'after' => '', + 'require_parents' => array('table'), + 'require_children' => array('td'), + 'trim' => 'both', + 'block_level' => true, + 'disabled_before' => '', + 'disabled_after' => '', + ), + array( + 'tag' => 'td', + 'before' => '', + 'after' => '', + 'require_parents' => array('tr'), + 'trim' => 'outside', + 'block_level' => true, + 'disabled_before' => '', + 'disabled_after' => '', + ), + array( + 'tag' => 'url', + 'type' => 'unparsed_content', + 'content' => '$1', + 'validate' => function(&$tag, &$data, $disabled) { + $data = strtr($data, array('
    ' => '')); + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + ), + array( + 'tag' => 'url', + 'type' => 'unparsed_equals', + 'before' => '', + 'after' => '', + 'validate' => function(&$tag, &$data, $disabled) { + if (strpos($data, 'http://') !== 0 && strpos($data, 'https://') !== 0 && strpos($data, 'bitcoin:') !== 0 && strpos($data, 'magnet:') !== 0) + $data = 'http://' . $data; + }, + 'disallow_children' => array('email', 'ftp', 'url', 'iurl'), + 'disabled_after' => ' ($1)', + ), + array( + 'tag' => 'u', + 'before' => '', + 'after' => '', + ), + array( + 'tag' => 'white', + 'before' => '', + 'after' => '', + ), + ); + + // This is mainly for the bbc manager, so it's easy to add tags above. Custom BBC should be added above this line. + if ($message === false) + return $codes; + + // So the parser won't skip them. + $itemcodes = array( + '*' => '', + '@' => 'disc', + '+' => 'square', + 'x' => 'square', + '#' => 'square', + 'o' => 'circle', + 'O' => 'circle', + '0' => 'circle', + ); + if (!isset($disabled['li']) && !isset($disabled['list'])) + { + foreach ($itemcodes as $c => $dummy) + $bbc_codes[$c] = array(); + } + + // Inside these tags autolink is not recommendable. + $no_autolink_tags = array( + 'url', + 'iurl', + 'ftp', + 'email', + ); + + // Shhhh! + if (!isset($disabled['color'])) + { + $codes[] = array( + 'tag' => 'chrissy', + 'before' => '', + 'after' => ' :-*', + ); + $codes[] = array( + 'tag' => 'kissy', + 'before' => '', + 'after' => ' :-*', + ); + } + + foreach ($codes as $c) + $bbc_codes[substr($c['tag'], 0, 1)][] = $c; + $codes = null; + } + + // Shall we take the time to cache this? + if ($cache_id != '' && !empty($modSettings['cache_enable']) && (($modSettings['cache_enable'] >= 2 && strlen($message) > 1000) || strlen($message) > 2400)) + { + // It's likely this will change if the message is modified. + $cache_key = 'parse:' . $cache_id . '-' . md5(md5($message) . '-' . $smileys . (empty($disabled) ? '' : implode(',', array_keys($disabled))) . safe_serialize($context['browser']) . $txt['lang_locale'] . $user_info['time_offset'] . $user_info['time_format']); + + if (($temp = cache_get_data($cache_key, 600)) != null) + return $temp; + + $cache_t = microtime(); + } + + if ($smileys === 'print') + { + // [glow], [shadow], and [move] can't really be printed. + $disabled['glow'] = true; + $disabled['shadow'] = true; + $disabled['move'] = true; + + // Colors can't well be displayed... supposed to be black and white. + $disabled['color'] = true; + $disabled['black'] = true; + $disabled['blue'] = true; + $disabled['white'] = true; + $disabled['red'] = true; + $disabled['green'] = true; + $disabled['me'] = true; + + // Color coding doesn't make sense. + $disabled['php'] = true; + + // Links are useless on paper... just show the link. + $disabled['ftp'] = true; + $disabled['url'] = true; + $disabled['iurl'] = true; + $disabled['email'] = true; + $disabled['flash'] = true; + + // !!! Change maybe? + if (!isset($_GET['images'])) + $disabled['img'] = true; + + // !!! Interface/setting to add more? + } + + if($local_disable) + foreach($local_disable as $d) + $disabled[$d] = true; + + $open_tags = array(); + $message = strtr($message, array("\n" => '
    ')); + + // The non-breaking-space looks a bit different each time. + $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{C2A0}' : chr(0xC2) . chr(0xA0)) : '\xA0'; + + $pos = -1; + while ($pos !== false) + { + // theymos - prevent various infinite loops + if($pos>90000) { + if(!isset($loopcount)) + $loopcount=0; + $loopcount++; + if($loopcount > 500) + return 'INVALID BBCODE: loop, probably unclosed tags'; + } + + $last_pos = isset($last_pos) ? max($pos, $last_pos) : $pos; + $pos = strpos($message, '[', $pos + 1); + + // Failsafe. + if ($pos === false || $last_pos > $pos) + $pos = strlen($message) + 1; + + // Can't have a one letter smiley, URL, or email! (sorry.) + if ($last_pos < $pos - 1) + { + // We want to eat one less, and one more, character (for smileys.) + $last_pos = max($last_pos - 1, 0); + $data = substr($message, $last_pos, $pos - $last_pos + 1); + + // Take care of some HTML! + if (!empty($modSettings['enablePostHTML']) && strpos($data, '<') !== false) + { + $data = preg_replace('~<a\s+href=((?:")?)((?:https?://|ftps?://|mailto:|bitcoin:)\S+?)\\1>~i', '[url=$2]', $data); + $data = preg_replace('~</a>~i', '[/url]', $data); + + //
    should be empty. + $empty_tags = array('br', 'hr'); + foreach ($empty_tags as $tag) + $data = str_replace(array('<' . $tag . '>', '<' . $tag . '/>', '<' . $tag . ' />'), '[' . $tag . ' /]', $data); + + // b, u, i, s, pre... basic tags. + $closable_tags = array('b', 'u', 'i', 's', 'em', 'ins', 'del', 'pre', 'blockquote'); + foreach ($closable_tags as $tag) + { + $diff = substr_count($data, '<' . $tag . '>') - substr_count($data, '</' . $tag . '>'); + $data = strtr($data, array('<' . $tag . '>' => '<' . $tag . '>', '</' . $tag . '>' => '')); + + if ($diff > 0) + $data .= str_repeat('', $diff); + } + + // Do - with security... action= -> action-. + preg_match_all('~<img\s+src=((?:")?)((?:https?://|ftps?://)\S+?)\\1(?:\s+alt=(".*?"|\S*?))?(?:\s?/)?>~i', $data, $matches, PREG_PATTERN_ORDER); + if (!empty($matches[0])) + { + $replaces = array(); + foreach ($matches[2] as $match => $imgtag) + { + $alt = empty($matches[3][$match]) ? '' : ' alt=' . preg_replace('~^"|"$~', '', $matches[3][$match]); + + // Remove action= from the URL - no funny business, now. + if (preg_match('~action(=|%3d)(?!dlattach)~i', $imgtag) != 0) + $imgtag = preg_replace('~action(=|%3d)(?!dlattach)~i', 'action-', $imgtag); + + // Check if the image is larger than allowed. + if (!empty($modSettings['max_image_width']) && !empty($modSettings['max_image_height'])) + { + list ($width, $height) = url_image_size($imgtag); + + if (!empty($modSettings['max_image_width']) && $width > $modSettings['max_image_width']) + { + $height = (int) (($modSettings['max_image_width'] * $height) / $width); + $width = $modSettings['max_image_width']; + } + + if (!empty($modSettings['max_image_height']) && $height > $modSettings['max_image_height']) + { + $width = (int) (($modSettings['max_image_height'] * $width) / $height); + $height = $modSettings['max_image_height']; + } + + // Set the new image tag. + $replaces[$matches[0][$match]] = '[img width=' . $width . ' height=' . $height . $alt . ']' . $imgtag . '[/img]'; + } + else + $replaces[$matches[0][$match]] = '[img' . $alt . ']' . $imgtag . '[/img]'; + } + + $data = strtr($data, $replaces); + } + } + + if (!empty($modSettings['autoLinkUrls'])) + { + // Are we inside tags that should be auto linked? + $no_autolink_area = false; + if (!empty($open_tags)) + { + foreach ($open_tags as $open_tag) + if (in_array($open_tag['tag'], $no_autolink_tags)) + $no_autolink_area = true; + } + + // Don't go backwards. + //!!! Don't think is the real solution.... + $lastAutoPos = isset($lastAutoPos) ? $lastAutoPos : 0; + if ($pos < $lastAutoPos) + $no_autolink_area = true; + $lastAutoPos = $pos; + + if (!$no_autolink_area) + { + // Parse any URLs.... have to get rid of the @ problems some things cause... stupid email addresses. + if (!isset($disabled['url']) && (strpos($data, '://') !== false || strpos($data, 'www.') !== false || strpos($data,'bitcoin:') !==false)) + { + // Switch out quotes really quick because they can cause problems. + $data = strtr($data, array(''' => '\'', ' ' => $context['utf8'] ? "\xC2\xA0" : "\xA0", '"' => '>">', '"' => '<"<', '<' => '\.(;\'"]|' . $nbsp . '|^)((?:http|https|ftp|ftps)://[\w\-_%@:|]+(?:\.[\w\-_%]+)*(?::\d+)?(?:/[\w\-_\~%\.@,\?&;=#(){}+:\'\\\\]*)*[/\w\-_\~%@\?;=#}\\\\])~i', + '~(?<=[\s>(;\'<]|' . $nbsp . '|^)(www(?:\.[\w\-_]+)+(?::\d+)?(?:/[\w\-_\~%\.@,\?&;=#(){}+:\'\\\\]*)*[/\w\-_\~%@\?;=#}\\\\])~i', + '~bitcoin:([-A-Za-z0-9._:/?#!%@$()*+,;=]{25,})~i' + ), array( + '[url]$1[/url]', + '[url=http://$1]$1[/url]', + '[url]bitcoin:$1[/url]' + ), $data))) + $data = $result; + + $data = strtr($data, array('\'' => ''', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ', '>">' => '"', '<"<' => '"', ' '<')); + } + + // Next, emails... + if (!isset($disabled['email']) && strpos($data, '@') !== false) + { + $data = preg_replace('~(?<=[\?\s' . $non_breaking_space . '\[\]()*\\\;>]|^)([\w\-\.]{1,80}@[\w\-]+\.[\w\-\.]+[\w\-])(?=[?,\s' . $non_breaking_space . '\[\]()*\\\]|$|
    | |>|<|"|'|\.(?:\.|;| |\s|$|
    ))~' . ($context['utf8'] ? 'u' : ''), '[email]$1[/email]', $data); + $data = preg_replace('~(?<=
    )([\w\-\.]{1,80}@[\w\-]+\.[\w\-\.]+[\w\-])(?=[?\.,;\s' . $non_breaking_space . '\[\]()*\\\]|$|
    | |>|<|"|')~' . ($context['utf8'] ? 'u' : ''), '[email]$1[/email]', $data); + // theymos - infinite loop + if($pos > 1000 && strpos(substr($message, $pos-100), '[email][email][email][email]') !== false) + return 'INVALID BBCODE: loop, probably unclosed tags (2)'; + } + } + } + + $data = strtr($data, array("\t" => '   ')); + + if (!empty($modSettings['fixLongWords']) && $modSettings['fixLongWords'] > 5) + { + // This is SADLY and INCREDIBLY browser dependent. + if ($context['browser']['is_gecko'] || $context['browser']['is_konqueror']) + $breaker = ' '; + // Opera... + elseif ($context['browser']['is_opera']) + $breaker = ' '; + // Internet Explorer... + else + $breaker = ' '; + + // PCRE will not be happy if we don't give it a short. + $modSettings['fixLongWords'] = (int) min(65535, $modSettings['fixLongWords']); + + // The idea is, find words xx long, and then replace them with xx + space + more. + if (strlen($data) > $modSettings['fixLongWords']) + { + // This is done in a roundabout way because $breaker has "long words" :P. + $data = strtr($data, array($breaker => '< >', ' ' => $context['utf8'] ? "\xC2\xA0" : "\xA0")); + $data = preg_replace_callback( + '~(?<=[>;:!? ' . $non_breaking_space . '\]()]|^)([\w' . ($context['utf8'] ? '\pL' : '') . '\.]{' . $modSettings['fixLongWords'] . ',})~' . ($context['utf8'] ? 'u' : ''), + 'word_break__preg_callback', + $data); + $data = strtr($data, array('< >' => $breaker, $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); + } + } + + // Do any smileys! + if ($smileys === true) + parsesmileys($data); + + // If it wasn't changed, no copying or other boring stuff has to happen! + if ($data != substr($message, $last_pos, $pos - $last_pos + 1)) + { + $message = substr($message, 0, $last_pos) . $data . substr($message, $pos + 1); + + // Since we changed it, look again incase we added or removed a tag. But we don't want to skip any. + $old_pos = strlen($data) + $last_pos - 1; + $pos = strpos($message, '[', $last_pos); + $pos = $pos === false ? $old_pos : min($pos, $old_pos); + } + } + + // Are we there yet? Are we there yet? + if ($pos >= strlen($message) - 1) + break; + + $tags = strtolower(substr($message, $pos + 1, 1)); + + if ($tags == '/' && !empty($open_tags)) + { + $pos2 = strpos($message, ']', $pos + 1); + if ($pos2 == $pos + 2) + continue; + $look_for = strtolower(substr($message, $pos + 2, $pos2 - $pos - 2)); + + $to_close = array(); + $block_level = null; + do + { + $tag = array_pop($open_tags); + if (!$tag) + break; + + if (!empty($tag['block_level'])) + { + // Only find out if we need to. + if ($block_level === false) + { + array_push($open_tags, $tag); + break; + } + + // The idea is, if we are LOOKING for a block level tag, we can close them on the way. + if (strlen($look_for) > 0 && isset($bbc_codes[$look_for[0]])) + { + foreach ($bbc_codes[$look_for[0]] as $temp) + if ($temp['tag'] == $look_for) + { + $block_level = !empty($temp['block_level']); + break; + } + } + + if ($block_level !== true) + { + $block_level = false; + array_push($open_tags, $tag); + break; + } + } + + $to_close[] = $tag; + } + while ($tag['tag'] != $look_for); + + // Did we just eat through everything and not find it? + if ((empty($open_tags) && (empty($tag) || $tag['tag'] != $look_for))) + { + $tablewarn = false; + for($cc=count($to_close)-1;$cc>=0;$cc--) { + if($to_close[$cc]['tag'] == 'table') + $tablewarn = true; + if($tablewarn && (in_array($to_close[$cc]['tag'], array('td', 'tr')))) + return 'INVALID BBCODE: close of unopened tag in table (2)'; + } + unset($cc, $tablewarn); + $open_tags = $to_close; + continue; + } + elseif (!empty($to_close) && $tag['tag'] != $look_for) + { + if ($block_level === null && isset($look_for[0], $bbc_codes[$look_for[0]])) + { + foreach ($bbc_codes[$look_for[0]] as $temp) + if ($temp['tag'] == $look_for) + { + $block_level = !empty($temp['block_level']); + break; + } + } + + // We're not looking for a block level tag (or maybe even a tag that exists...) + if (!$block_level) + { + foreach ($to_close as $tag) + array_push($open_tags, $tag); + continue; + } + } + + foreach ($to_close as $tag) + { + $message = substr($message, 0, $pos) . $tag['after'] . substr($message, $pos2 + 1); + $pos += strlen($tag['after']); + $pos2 = $pos - 1; + + // See the comment at the end of the big loop - just eating whitespace ;). + if (!empty($tag['block_level']) && substr($message, $pos, 6) == '
    ') + $message = substr($message, 0, $pos) . substr($message, $pos + 6); + if (!empty($tag['trim']) && $tag['trim'] != 'inside' && preg_match('~(
    | |\s)*~', substr($message, $pos), $matches) != 0) + $message = substr($message, 0, $pos) . substr($message, $pos + strlen($matches[0])); + } + + if (!empty($to_close)) + { + $to_close = array(); + $pos--; + } + + continue; + } + + // No tags for this character, so just keep going (fastest possible course.) + if (!isset($bbc_codes[$tags])) + continue; + + $inside = empty($open_tags) ? null : $open_tags[count($open_tags) - 1]; + $tag = null; + foreach ($bbc_codes[$tags] as $possible) + { + // Not a match? + if (strtolower(substr($message, $pos + 1, strlen($possible['tag']))) != $possible['tag']) + continue; + + $next_c = substr($message, $pos + 1 + strlen($possible['tag']), 1); + + // A test validation? + if (isset($possible['test']) && preg_match('~^' . $possible['test'] . '~', substr($message, $pos + 1 + strlen($possible['tag']) + 1)) == 0) + continue; + // Do we want parameters? + elseif (!empty($possible['parameters'])) + { + if ($next_c != ' ') + continue; + } + elseif (isset($possible['type'])) + { + // Do we need an equal sign? + if (in_array($possible['type'], array('unparsed_equals', 'unparsed_commas', 'unparsed_commas_content', 'unparsed_equals_content', 'parsed_equals')) && $next_c != '=') + continue; + // Maybe we just want a /... + if ($possible['type'] == 'closed' && $next_c != ']' && substr($message, $pos + 1 + strlen($possible['tag']), 2) != '/]' && substr($message, $pos + 1 + strlen($possible['tag']), 3) != ' /]') + continue; + // An immediate ]? + if ($possible['type'] == 'unparsed_content' && $next_c != ']') + continue; + } + // No type means 'parsed_content', which demands an immediate ] without parameters! + elseif ($next_c != ']') + continue; + + // Check allowed tree? + if (isset($possible['require_parents']) && ($inside === null || !in_array($inside['tag'], $possible['require_parents']))) + continue; + elseif (isset($inside['require_children']) && !in_array($possible['tag'], $inside['require_children'])) + continue; + // If this is in the list of disallowed child tags, don't parse it. + elseif (isset($inside['disallow_children']) && in_array($possible['tag'], $inside['disallow_children'])) + continue; + + $pos1 = $pos + 1 + strlen($possible['tag']) + 1; + + // This is long, but it makes things much easier and cleaner. + if (!empty($possible['parameters'])) + { + $preg = array(); + foreach ($possible['parameters'] as $p => $info) + $preg[] = '(\s+' . $p . '=' . (empty($info['quoted']) ? '' : '"') . (isset($info['match']) ? $info['match'] : '(.+?)') . (empty($info['quoted']) ? '' : '"') . ')' . (empty($info['optional']) ? '' : '?'); + + // Okay, this may look ugly and it is, but it's not going to happen much and it is the best way of allowing any order of parameters but still parsing them right. + $match = false; + $orders = permute($preg); + foreach ($orders as $p) + if (preg_match('~^' . implode('', $p) . '\]~i', substr($message, $pos1 - 1), $matches) != 0) + { + $match = true; + break; + } + + // Didn't match our parameter list, try the next possible. + if (!$match) + continue; + + $params = array(); + for ($i = 1, $n = count($matches); $i < $n; $i += 2) + { + $key = strtok(ltrim($matches[$i]), '='); + if (isset($possible['parameters'][$key]['value'])) + $params['{' . $key . '}'] = strtr($possible['parameters'][$key]['value'], array('$1' => $matches[$i + 1])); + elseif (isset($possible['parameters'][$key]['validate'])) + $params['{' . $key . '}'] = $possible['parameters'][$key]['validate']($matches[$i + 1]); + else + $params['{' . $key . '}'] = $matches[$i + 1]; + + // Just to make sure: replace any $ or { so they can't interpolate wrongly. + $params['{' . $key . '}'] = strtr($params['{' . $key . '}'], array('$' => '$', '{' => '{')); + } + + foreach ($possible['parameters'] as $p => $info) + { + if (!isset($params['{' . $p . '}'])) + $params['{' . $p . '}'] = ''; + } + + $tag = $possible; + + // Put the parameters into the string. + if (isset($tag['before'])) + $tag['before'] = strtr($tag['before'], $params); + if (isset($tag['after'])) + $tag['after'] = strtr($tag['after'], $params); + if (isset($tag['content'])) + $tag['content'] = strtr($tag['content'], $params); + + $pos1 += strlen($matches[0]) - 1; + } + else + $tag = $possible; + break; + } + + // Item codes are complicated buggers... they are implicit [li]s and can make [list]s! + if ($smileys !== false && $tag === null && isset($itemcodes[substr($message, $pos + 1, 1)]) && substr($message, $pos + 2, 1) == ']' && !isset($disabled['list']) && !isset($disabled['li'])) + { + if (substr($message, $pos + 1, 1) == '0' && !in_array(substr($message, $pos - 1, 1), array(';', ' ', "\t", '>'))) + continue; + $tag = $itemcodes[substr($message, $pos + 1, 1)]; + + // First let's set up the tree: it needs to be in a list, or after an li. + if ($inside === null || ($inside['tag'] != 'list' && $inside['tag'] != 'li')) + { + $open_tags[] = array( + 'tag' => 'list', + 'after' => '', + 'block_level' => true, + 'require_children' => array('li'), + 'disallow_children' => isset($inside['disallow_children']) ? $inside['disallow_children'] : null, + ); + $code = '
      '; + } + // We're in a list item already: another itemcode? Close it first. + elseif ($inside['tag'] == 'li') + { + array_pop($open_tags); + $code = ''; + } + else + $code = ''; + + // Now we open a new tag. + $open_tags[] = array( + 'tag' => 'li', + 'after' => '', + 'trim' => 'outside', + 'block_level' => true, + 'disallow_children' => isset($inside['disallow_children']) ? $inside['disallow_children'] : null, + ); + + // First, open the tag... + $code .= ''; + $message = substr($message, 0, $pos) . $code . substr($message, $pos + 3); + $pos += strlen($code) - 1; + + // Next, find the next break (if any.) If there's more itemcode after it, keep it going - otherwise close! + $pos2 = strpos($message, '
      ', $pos); + $pos3 = strpos($message, '[/', $pos); + if ($pos2 !== false && ($pos2 <= $pos3 || $pos3 === false)) + { + preg_match('~^(
      | |\s|\[)+~', substr($message, $pos2 + 6), $matches); + $message = substr($message, 0, $pos2) . (!empty($matches[0]) && substr($matches[0], -1) == '[' ? '[/li]' : '[/li][/list]') . substr($message, $pos2); + + $open_tags[count($open_tags) - 2]['after'] = '
    '; + } + // Tell the [list] that it needs to close specially. + else + { + if(count($open_tags)<2) { + return 'INVALID BBCODE: messed-up itemcodes'; + } + // Move the li over, because we're not sure what we'll hit. + $open_tags[count($open_tags) - 1]['after'] = ''; + $open_tags[count($open_tags) - 2]['after'] = ''; + } + + continue; + } + + // Implicitly close lists and tables if something other than what's required is in them. This is needed for itemcode. + if ($tag === null && $inside !== null && !empty($inside['require_children'])) + { + array_pop($open_tags); + + $message = substr($message, 0, $pos) . $inside['after'] . substr($message, $pos); + $pos += strlen($inside['after']) - 1; + } + + // No tag? Keep looking, then. Silly people using brackets without actual tags. + if ($tag === null) + continue; + + // Propagate the list to the child (so wrapping the disallowed tag won't work either.) + if (isset($inside['disallow_children'])) + $tag['disallow_children'] = isset($tag['disallow_children']) ? array_unique(array_merge($tag['disallow_children'], $inside['disallow_children'])) : $inside['disallow_children']; + + // Is this tag disabled? + if (isset($disabled[$tag['tag']])) + { + if (!isset($tag['disabled_before']) && !isset($tag['disabled_after']) && !isset($tag['disabled_content'])) + { + $tag['before'] = !empty($tag['block_level']) ? '
    ' : ''; + $tag['after'] = !empty($tag['block_level']) ? '
    ' : ''; + $tag['content'] = isset($tag['type']) && $tag['type'] == 'closed' ? '' : (!empty($tag['block_level']) ? '
    $1
    ' : '$1'); + } + elseif (isset($tag['disabled_before']) || isset($tag['disabled_after'])) + { + $tag['before'] = isset($tag['disabled_before']) ? $tag['disabled_before'] : (!empty($tag['block_level']) ? '
    ' : ''); + $tag['after'] = isset($tag['disabled_after']) ? $tag['disabled_after'] : (!empty($tag['block_level']) ? '
    ' : ''); + } + else + $tag['content'] = $tag['disabled_content']; + } + + // The only special case is 'html', which doesn't need to close things. + if (!empty($tag['block_level']) && $tag['tag'] != 'html' && empty($inside['block_level'])) + { + $n = count($open_tags) - 1; + while (empty($open_tags[$n]['block_level']) && $n >= 0) + $n--; + + // Close all the non block level tags so this tag isn't surrounded by them. + for ($i = count($open_tags) - 1; $i > $n; $i--) + { + $message = substr($message, 0, $pos) . $open_tags[$i]['after'] . substr($message, $pos); + $pos += strlen($open_tags[$i]['after']); + $pos1 += strlen($open_tags[$i]['after']); + + // Trim or eat trailing stuff... see comment at the end of the big loop. + if (!empty($open_tags[$i]['block_level']) && substr($message, $pos, 6) == '
    ') + $message = substr($message, 0, $pos) . substr($message, $pos + 6); + if (!empty($open_tags[$i]['trim']) && $tag['trim'] != 'inside' && preg_match('~(
    | |\s)*~', substr($message, $pos), $matches) != 0) + $message = substr($message, 0, $pos) . substr($message, $pos + strlen($matches[0])); + + array_pop($open_tags); + } + } + + // No type means 'parsed_content'. + if (!isset($tag['type'])) + { + // !!! Check for end tag first, so people can say "I like that [i] tag"? + $open_tags[] = $tag; + $message = substr($message, 0, $pos) . $tag['before'] . substr($message, $pos1); + $pos += strlen($tag['before']) - 1; + } + // Don't parse the content, just skip it. + elseif ($tag['type'] == 'unparsed_content') + { + $pos2 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos1); + if ($pos2 === false) + continue; + + $data = substr($message, $pos1, $pos2 - $pos1); + + if (!empty($tag['block_level']) && substr($data, 0, 6) == '
    ') + $data = substr($data, 6); + + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + $code = strtr($tag['content'], array('$1' => $data)); + $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + 3 + strlen($tag['tag'])); + $pos += strlen($code) - 1; + } + // Don't parse the content, just skip it. + elseif ($tag['type'] == 'unparsed_equals_content') + { + // The value may be quoted for some tags - check. + if (isset($tag['quoted'])) + { + $quoted = substr($message, $pos1, 6) == '"'; + if ($tag['quoted'] != 'optional' && !$quoted) + continue; + + if ($quoted) + $pos1 += 6; + } + else + $quoted = false; + + $pos2 = strpos($message, $quoted == false ? ']' : '"]', $pos1); + if ($pos2 === false) + continue; + $pos3 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos2); + if ($pos3 === false) + continue; + + $data = array( + substr($message, $pos2 + ($quoted == false ? 1 : 7), $pos3 - ($pos2 + ($quoted == false ? 1 : 7))), + substr($message, $pos1, $pos2 - $pos1) + ); + + if (!empty($tag['block_level']) && substr($data[0], 0, 6) == '
    ') + $data[0] = substr($data[0], 6); + + // Validation for my parking, please! + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + $code = strtr($tag['content'], array('$1' => $data[0], '$2' => $data[1])); + $message = substr($message, 0, $pos) . $code . substr($message, $pos3 + 3 + strlen($tag['tag'])); + $pos += strlen($code) - 1; + } + // A closed tag, with no content or value. + elseif ($tag['type'] == 'closed') + { + $pos2 = strpos($message, ']', $pos); + $message = substr($message, 0, $pos) . $tag['content'] . substr($message, $pos2 + 1); + $pos += strlen($tag['content']) - 1; + } + // This one is sorta ugly... :/. Unforunately, it's needed for flash. + elseif ($tag['type'] == 'unparsed_commas_content') + { + $pos2 = strpos($message, ']', $pos1); + if ($pos2 === false) + continue; + $pos3 = stripos($message, '[/' . substr($message, $pos + 1, strlen($tag['tag'])) . ']', $pos2); + if ($pos3 === false) + continue; + + // We want $1 to be the content, and the rest to be csv. + $data = explode(',', ',' . substr($message, $pos1, $pos2 - $pos1)); + $data[0] = substr($message, $pos2 + 1, $pos3 - $pos2 - 1); + + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + $code = $tag['content']; + foreach ($data as $k => $d) + $code = strtr($code, array('$' . ($k + 1) => trim($d))); + $message = substr($message, 0, $pos) . $code . substr($message, $pos3 + 3 + strlen($tag['tag'])); + $pos += strlen($code) - 1; + } + // This has parsed content, and a csv value which is unparsed. + elseif ($tag['type'] == 'unparsed_commas') + { + $pos2 = strpos($message, ']', $pos1); + if ($pos2 === false) + continue; + + $data = explode(',', substr($message, $pos1, $pos2 - $pos1)); + + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + // Fix after, for disabled code mainly. + foreach ($data as $k => $d) + $tag['after'] = strtr($tag['after'], array('$' . ($k + 1) => trim($d))); + + $open_tags[] = $tag; + + // Replace them out, $1, $2, $3, $4, etc. + $code = $tag['before']; + foreach ($data as $k => $d) + $code = strtr($code, array('$' . ($k + 1) => trim($d))); + $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + 1); + $pos += strlen($code) - 1; + } + // A tag set to a value, parsed or not. + elseif ($tag['type'] == 'unparsed_equals' || $tag['type'] == 'parsed_equals') + { + // The value may be quoted for some tags - check. + if (isset($tag['quoted'])) + { + $quoted = substr($message, $pos1, 6) == '"'; + if ($tag['quoted'] != 'optional' && !$quoted) + continue; + + if ($quoted) + $pos1 += 6; + } + else + $quoted = false; + + $pos2 = strpos($message, $quoted == false ? ']' : '"]', $pos1); + if ($pos2 === false) + continue; + + $data = substr($message, $pos1, $pos2 - $pos1); + + // Validation for my parking, please! + if (isset($tag['validate'])) + $tag['validate']($tag, $data, $disabled); + + // For parsed content, we must recurse to avoid security problems. + if ($tag['type'] != 'unparsed_equals') + $data = parse_bbc($data); + + $tag['after'] = strtr($tag['after'], array('$1' => $data)); + + $open_tags[] = $tag; + + $code = strtr($tag['before'], array('$1' => $data)); + $message = substr($message, 0, $pos) . $code . substr($message, $pos2 + ($quoted == false ? 1 : 7)); + $pos += strlen($code) - 1; + } + + // If this is block level, eat any breaks after it. + if (!empty($tag['block_level']) && substr($message, $pos + 1, 6) == '
    ') + $message = substr($message, 0, $pos + 1) . substr($message, $pos + 7); + + // Are we trimming outside this tag? + if (!empty($tag['trim']) && $tag['trim'] != 'outside' && preg_match('~(
    | |\s)*~', substr($message, $pos + 1), $matches) != 0) + $message = substr($message, 0, $pos + 1) . substr($message, $pos + 1 + strlen($matches[0])); + } + + // Close any remaining tags. + while ($tag = array_pop($open_tags)) { + if(in_array($tag['tag'], array('table','td','tr','th'))) + return 'INVALID BBCODE: close of unopened tag in table (1)'; + $message .= $tag['after']; + } + + if (substr($message, 0, 1) == ' ') + $message = ' ' . substr($message, 1); + + // Cleanup whitespace. + $message = strtr($message, array(' ' => '  ', "\r" => '', "\n" => '
    ', '
    ' => '
     ', ' ' => "\n")); + + if(in_array('ugc', $local_disable)) + $message = str_replace(' 0.05) + cache_put_data($cache_key, $message, 600); + + return $message; } // Parse smileys in the passed message. function parsesmileys(&$message) { - global $modSettings, $db_prefix, $txt, $user_info, $context; - static $smileyfromcache = array(), $smileytocache = array(); - - // No smiley set at all?! - if ($user_info['smiley_set'] == 'none') - return; - - // If the smiley array hasn't been set, do it now. - if (empty($smileyfromcache)) - { - // Use the default smileys if it is disabled. (better for "portability" of smileys.) - if (empty($modSettings['smiley_enable'])) - { - $smileysfrom = array('>:D', ':D', '::)', '>:(', ':)', ';)', ';D', ':(', ':o', '8)', ':P', '???', ':-[', ':-X', ':-*', ':\'(', ':-\\', '^-^', 'O0', 'C:-)', '0:)'); - $smileysto = array('evil.gif', 'cheesy.gif', 'rolleyes.gif', 'angry.gif', 'smiley.gif', 'wink.gif', 'grin.gif', 'sad.gif', 'shocked.gif', 'cool.gif', 'tongue.gif', 'huh.gif', 'embarrassed.gif', 'lipsrsealed.gif', 'kiss.gif', 'cry.gif', 'undecided.gif', 'azn.gif', 'afro.gif', 'police.gif', 'angel.gif'); - $smileysdescs = array('', $txt[289], $txt[450], $txt[288], $txt[287], $txt[292], $txt[293], $txt[291], $txt[294], $txt[295], $txt[451], $txt[296], $txt[526], $txt[527], $txt[529], $txt[530], $txt[528], '', '', '', ''); - } - else - { - // Load the smileys in reverse order by length so they don't get parsed wrong. - if (($temp = cache_get_data('parsing_smileys', 480)) == null) - { - $result = db_query(" - SELECT code, filename, description - FROM {$db_prefix}smileys", __FILE__, __LINE__); - $smileysfrom = array(); - $smileysto = array(); - $smileysdescs = array(); - while ($row = mysql_fetch_assoc($result)) - { - $smileysfrom[] = $row['code']; - $smileysto[] = $row['filename']; - $smileysdescs[] = $row['description']; - } - mysql_free_result($result); - - cache_put_data('parsing_smileys', array($smileysfrom, $smileysto, $smileysdescs), 480); - } - else - list ($smileysfrom, $smileysto, $smileysdescs) = $temp; - } - - // The non-breaking-space is a complex thing... - $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{A0}' : pack('C*', 0xC2, 0xA0)) : '\xA0'; - - // This smiley regex makes sure it doesn't parse smileys within code tags (so [url=mailto:David@bla.com] doesn't parse the :D smiley) - for ($i = 0, $n = count($smileysfrom); $i < $n; $i++) - { - $smileyfromcache[] = '/(?<=[>:\?\.\s' . $non_breaking_space . '[\]()*\\\;]|^)(' . preg_quote($smileysfrom[$i], '/') . '|' . preg_quote(htmlspecialchars($smileysfrom[$i], ENT_QUOTES), '/') . ')(?=[^[:alpha:]0-9]|$)/' . ($context['utf8'] ? 'u' : ''); - // Escape a bunch of smiley-related characters in the description so it doesn't get a double dose :P. - $smileytocache[] = '' . strtr(htmlspecialchars($smileysdescs[$i]), array(':' => ':', '(' => '(', ')' => ')', '$' => '$', '[' => '[')) . ''; - } - } - - // Replace away! - // !!! There must be a way to speed this up. - $message = preg_replace($smileyfromcache, $smileytocache, $message); + global $modSettings, $db_prefix, $txt, $user_info, $context; + static $smileyfromcache = array(), $smileytocache = array(); + + // No smiley set at all?! + if ($user_info['smiley_set'] == 'none') + return; + + // If the smiley array hasn't been set, do it now. + if (empty($smileyfromcache)) + { + // Use the default smileys if it is disabled. (better for "portability" of smileys.) + if (empty($modSettings['smiley_enable'])) + { + $smileysfrom = array('>:D', ':D', '::)', '>:(', ':)', ';)', ';D', ':(', ':o', '8)', ':P', '???', ':-[', ':-X', ':-*', ':\'(', ':-\\', '^-^', 'O0', 'C:-)', '0:)'); + $smileysto = array('evil.gif', 'cheesy.gif', 'rolleyes.gif', 'angry.gif', 'smiley.gif', 'wink.gif', 'grin.gif', 'sad.gif', 'shocked.gif', 'cool.gif', 'tongue.gif', 'huh.gif', 'embarrassed.gif', 'lipsrsealed.gif', 'kiss.gif', 'cry.gif', 'undecided.gif', 'azn.gif', 'afro.gif', 'police.gif', 'angel.gif'); + $smileysdescs = array('', $txt[289], $txt[450], $txt[288], $txt[287], $txt[292], $txt[293], $txt[291], $txt[294], $txt[295], $txt[451], $txt[296], $txt[526], $txt[527], $txt[529], $txt[530], $txt[528], '', '', '', ''); + } + else + { + // Load the smileys in reverse order by length so they don't get parsed wrong. + if (($temp = cache_get_data('parsing_smileys', 480)) == null) + { + $result = db_query(" + SELECT code, filename, description + FROM {$db_prefix}smileys", __FILE__, __LINE__); + $smileysfrom = array(); + $smileysto = array(); + $smileysdescs = array(); + while ($row = mysql_fetch_assoc($result)) + { + $smileysfrom[] = $row['code']; + $smileysto[] = $row['filename']; + $smileysdescs[] = $row['description']; + } + mysql_free_result($result); + + cache_put_data('parsing_smileys', array($smileysfrom, $smileysto, $smileysdescs), 480); + } + else + list ($smileysfrom, $smileysto, $smileysdescs) = $temp; + } + + // The non-breaking-space is a complex thing... + $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{A0}' : pack('C*', 0xC2, 0xA0)) : '\xA0'; + + // This smiley regex makes sure it doesn't parse smileys within code tags (so [url=mailto:David@bla.com] doesn't parse the :D smiley) + for ($i = 0, $n = count($smileysfrom); $i < $n; $i++) + { + $smileyfromcache[] = '/(?<=[>:\?\.\s' . $non_breaking_space . '[\]()*\\\;]|^)(' . preg_quote($smileysfrom[$i], '/') . '|' . preg_quote(htmlspecialchars($smileysfrom[$i], ENT_QUOTES), '/') . ')(?=[^[:alpha:]0-9]|$)/' . ($context['utf8'] ? 'u' : ''); + // Escape a bunch of smiley-related characters in the description so it doesn't get a double dose :P. + $smileytocache[] = '' . strtr(htmlspecialchars($smileysdescs[$i]), array(':' => ':', '(' => '(', ')' => ')', '$' => '$', '[' => '[')) . ''; + } + } + + // Replace away! + // !!! There must be a way to speed this up. + $message = preg_replace($smileyfromcache, $smileytocache, $message); } // Parses some bbc before sending into the database... function preparsecode(&$message, $previewing = false) { - global $user_info, $modSettings, $context; - - - // Clean up after nobbc ;). - $message = preg_replace_callback('~\[nobbc\](.+?)\[/nobbc\]~is', 'nobbc__preg_callback', $message); - - //$message = preg_replace('~\[([^\]=\s]+)[^\]]*\](?' . '>\s|(?R))*?\[/\1\]\s?~i', '', $message); - - // Remove \r's... they're evil! - $message = strtr($message, array("\r" => '')); - - // You won't believe this - but too many periods upsets apache it seems! - $message = preg_replace('~\.{100,}~', '...', $message); - - // Trim off trailing quotes - these often happen by accident. - while (substr($message, -7) == '[quote]') - $message = substr($message, 0, -7); - while (substr($message, 0, 8) == '[/quote]') - $message = substr($message, 8); - - // Check if all code tags are closed. - $codeopen = preg_match_all('~(\[code(?:=[^\]]+)?\])~is', $message, $dummy); - $codeclose = preg_match_all('~(\[/code\])~is', $message, $dummy); - - // Close/open all code tags... - if ($codeopen > $codeclose) - $message .= str_repeat('[/code]', $codeopen - $codeclose); - elseif ($codeclose > $codeopen) - $message = str_repeat('[code]', $codeclose - $codeopen) . $message; - - // Now that we've fixed all the code tags, let's fix the img and url tags... - $parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $message, -1, PREG_SPLIT_DELIM_CAPTURE); - - // The regular expression non breaking space has many versions. - $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{A0}' : pack('C*', 0xC2, 0xA0)) : '\xA0'; - - // Only mess with stuff outside [code] tags. - for ($i = 0, $n = count($parts); $i < $n; $i++) - { - // It goes 0 = outside, 1 = begin tag, 2 = inside, 3 = close tag, repeat. - if ($i % 4 == 0) - { - fixTags($parts[$i]); - - // Replace /me.+?\n with [me=name]dsf[/me]\n. - if (strpos($user_info['name'], '[') !== false || strpos($user_info['name'], ']') !== false || strpos($user_info['name'], '\'') !== false || strpos($user_info['name'], '"') !== false) - $parts[$i] = preg_replace('~(?:\A|\n)/me(?: | )([^\n]*)(?:\z)?~i', '[me="' . $user_info['name'] . '"]$1[/me]', $parts[$i]); - else - $parts[$i] = preg_replace('~(?:\A|\n)/me(?: | )([^\n]*)(?:\z)?~i', '[me=' . $user_info['name'] . ']$1[/me]', $parts[$i]); - - if (!$previewing && strpos($parts[$i], '[html]') !== false) - { - //if (false && allowedTo('admin_forum')) - //$parts[$i] = preg_replace('~\[html\](.+?)\[/html\]~ise', '\'[html]\' . strtr(un_htmlspecialchars(\'$1\'), array("\n" => \' \', \' \' => \' \')) . \'[/html]\'', $parts[$i]); - // We should edit them out, or else if an admin edits the message they will get shown... - //else - //{ - while (strpos($parts[$i], '[html]') !== false) - $parts[$i] = preg_replace('~\[[/]?html\]~i', '', $parts[$i]); - //} - } - - // Let's look at the time tags... - $parts[$i] = preg_replace_callback('~\[time(?:=(absolute))*\](.+?)\[/time\]~i', 'time_fix__preg_callback', $parts[$i]); - - $list_open = substr_count($parts[$i], '[list]') + substr_count($parts[$i], '[list '); - $list_close = substr_count($parts[$i], '[/list]'); - if ($list_close - $list_open > 0) - $parts[$i] = str_repeat('[list]', $list_close - $list_open) . $parts[$i]; - if ($list_open - $list_close > 0) - $parts[$i] = $parts[$i] . str_repeat('[/list]', $list_open - $list_close); - - // Make sure all tags are lowercase. - $parts[$i] = preg_replace_callback('~\[([/]?)(list|li|table|tr|td)((\s[^\]]+)*)\]~i', 'lowercase_tags__preg_callback', $parts[$i]); - - $mistake_fixes = array( - // Find [table]s not followed by [tr]. - '~\[table\](?![\s' . $non_breaking_space . ']*\[tr\])~s' . ($context['utf8'] ? 'u' : '') => '[table][tr]', - // Find [tr]s not followed by [td]. - '~\[tr\](?![\s' . $non_breaking_space . ']*\[td\])~s' . ($context['utf8'] ? 'u' : '') => '[tr][td]', - // Find [/td]s not followed by something valid. - '~\[/td\](?![\s' . $non_breaking_space . ']*(?:\[td\]|\[/tr\]|\[/table\]))~s' . ($context['utf8'] ? 'u' : '') => '[/td][/tr]', - // Find [/tr]s not followed by something valid. - '~\[/tr\](?![\s' . $non_breaking_space . ']*(?:\[tr\]|\[/table\]))~s' . ($context['utf8'] ? 'u' : '') => '[/tr][/table]', - // Find [/td]s incorrectly followed by [/table]. - '~\[/td\][\s' . $non_breaking_space . ']*\[/table\]~s' . ($context['utf8'] ? 'u' : '') => '[/td][/tr][/table]', - // Find [table]s, [tr]s, and [/td]s (possibly correctly) followed by [td]. - '~\[(table|tr|/td)\]([\s' . $non_breaking_space . ']*)\[td\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_td_]', - // Now, any [td]s left should have a [tr] before them. - '~\[td\]~s' => '[tr][td]', - // Look for [tr]s which are correctly placed. - '~\[(table|/tr)\]([\s' . $non_breaking_space . ']*)\[tr\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_tr_]', - // Any remaining [tr]s should have a [table] before them. - '~\[tr\]~s' => '[table][tr]', - // Look for [/td]s followed by [/tr]. - '~\[/td\]([\s' . $non_breaking_space . ']*)\[/tr\]~s' . ($context['utf8'] ? 'u' : '') => '[/td]$1[_/tr_]', - // Any remaining [/tr]s should have a [/td]. - '~\[/tr\]~s' => '[/td][/tr]', - // Look for properly opened [li]s which aren't closed. - '~\[li\]([^\[\]]+?)\[li\]~s' => '[li]$1[_/li_][_li_]', - '~\[li\]([^\[\]]+?)$~s' => '[li]$1[/li]', - // Lists - find correctly closed items/lists. - '~\[/li\]([\s' . $non_breaking_space . ']*)\[/list\]~s' . ($context['utf8'] ? 'u' : '') => '[_/li_]$1[/list]', - // Find list items closed and then opened. - '~\[/li\]([\s' . $non_breaking_space . ']*)\[li\]~s' . ($context['utf8'] ? 'u' : '') => '[_/li_]$1[_li_]', - // Now, find any [list]s or [/li]s followed by [li]. - '~\[(list(?: [^\]]*?)?|/li)\]([\s' . $non_breaking_space . ']*)\[li\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_li_]', - // Any remaining [li]s weren't inside a [list]. - '~\[li\]~' => '[list][li]', - // Any remaining [/li]s weren't before a [/list]. - '~\[/li\]~' => '[/li][/list]', - // Put the correct ones back how we found them. - '~\[_(li|/li|td|tr|/tr)_\]~' => '[$1]', - ); - - // Fix up some use of tables without [tr]s, etc. (it has to be done more than once to catch it all.) - for ($j = 0; $j < 3; $j++) - $parts[$i] = preg_replace(array_keys($mistake_fixes), $mistake_fixes, $parts[$i]); - } - } - - // Put it back together! - if (!$previewing) - $message = strtr(implode('', $parts), array(' ' => '  ', "\n" => '
    ', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); - else - $message = strtr(implode('', $parts), array(' ' => '  ', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); - - // Now let's quickly clean up things that will slow our parser (which are common in posted code.) - $message = strtr($message, array('[]' => '[]', '['' => '['')); + global $user_info, $modSettings, $context; + + + // Clean up after nobbc ;). + $message = preg_replace_callback('~\[nobbc\](.+?)\[/nobbc\]~is', 'nobbc__preg_callback', $message); + + //$message = preg_replace('~\[([^\]=\s]+)[^\]]*\](?' . '>\s|(?R))*?\[/\1\]\s?~i', '', $message); + + // Remove \r's... they're evil! + $message = strtr($message, array("\r" => '')); + + // You won't believe this - but too many periods upsets apache it seems! + $message = preg_replace('~\.{100,}~', '...', $message); + + // Trim off trailing quotes - these often happen by accident. + while (substr($message, -7) == '[quote]') + $message = substr($message, 0, -7); + while (substr($message, 0, 8) == '[/quote]') + $message = substr($message, 8); + + // Check if all code tags are closed. + $codeopen = preg_match_all('~(\[code(?:=[^\]]+)?\])~is', $message, $dummy); + $codeclose = preg_match_all('~(\[/code\])~is', $message, $dummy); + + // Close/open all code tags... + if ($codeopen > $codeclose) + $message .= str_repeat('[/code]', $codeopen - $codeclose); + elseif ($codeclose > $codeopen) + $message = str_repeat('[code]', $codeclose - $codeopen) . $message; + + // Now that we've fixed all the code tags, let's fix the img and url tags... + $parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $message, -1, PREG_SPLIT_DELIM_CAPTURE); + + // The regular expression non breaking space has many versions. + $non_breaking_space = $context['utf8'] ? ($context['server']['complex_preg_chars'] ? '\x{A0}' : pack('C*', 0xC2, 0xA0)) : '\xA0'; + + // Only mess with stuff outside [code] tags. + for ($i = 0, $n = count($parts); $i < $n; $i++) + { + // It goes 0 = outside, 1 = begin tag, 2 = inside, 3 = close tag, repeat. + if ($i % 4 == 0) + { + fixTags($parts[$i]); + + // Replace /me.+?\n with [me=name]dsf[/me]\n. + if (strpos($user_info['name'], '[') !== false || strpos($user_info['name'], ']') !== false || strpos($user_info['name'], '\'') !== false || strpos($user_info['name'], '"') !== false) + $parts[$i] = preg_replace('~(?:\A|\n)/me(?: | )([^\n]*)(?:\z)?~i', '[me="' . $user_info['name'] . '"]$1[/me]', $parts[$i]); + else + $parts[$i] = preg_replace('~(?:\A|\n)/me(?: | )([^\n]*)(?:\z)?~i', '[me=' . $user_info['name'] . ']$1[/me]', $parts[$i]); + + if (!$previewing && strpos($parts[$i], '[html]') !== false) + { + //if (false && allowedTo('admin_forum')) + //$parts[$i] = preg_replace('~\[html\](.+?)\[/html\]~ise', '\'[html]\' . strtr(un_htmlspecialchars(\'$1\'), array("\n" => \' \', \' \' => \' \')) . \'[/html]\'', $parts[$i]); + // We should edit them out, or else if an admin edits the message they will get shown... + //else + //{ + while (strpos($parts[$i], '[html]') !== false) + $parts[$i] = preg_replace('~\[[/]?html\]~i', '', $parts[$i]); + //} + } + + // Let's look at the time tags... + $parts[$i] = preg_replace_callback('~\[time(?:=(absolute))*\](.+?)\[/time\]~i', 'time_fix__preg_callback', $parts[$i]); + + $list_open = substr_count($parts[$i], '[list]') + substr_count($parts[$i], '[list '); + $list_close = substr_count($parts[$i], '[/list]'); + if ($list_close - $list_open > 0) + $parts[$i] = str_repeat('[list]', $list_close - $list_open) . $parts[$i]; + if ($list_open - $list_close > 0) + $parts[$i] = $parts[$i] . str_repeat('[/list]', $list_open - $list_close); + + // Make sure all tags are lowercase. + $parts[$i] = preg_replace_callback('~\[([/]?)(list|li|table|tr|td)((\s[^\]]+)*)\]~i', 'lowercase_tags__preg_callback', $parts[$i]); + + $mistake_fixes = array( + // Find [table]s not followed by [tr]. + '~\[table\](?![\s' . $non_breaking_space . ']*\[tr\])~s' . ($context['utf8'] ? 'u' : '') => '[table][tr]', + // Find [tr]s not followed by [td]. + '~\[tr\](?![\s' . $non_breaking_space . ']*\[td\])~s' . ($context['utf8'] ? 'u' : '') => '[tr][td]', + // Find [/td]s not followed by something valid. + '~\[/td\](?![\s' . $non_breaking_space . ']*(?:\[td\]|\[/tr\]|\[/table\]))~s' . ($context['utf8'] ? 'u' : '') => '[/td][/tr]', + // Find [/tr]s not followed by something valid. + '~\[/tr\](?![\s' . $non_breaking_space . ']*(?:\[tr\]|\[/table\]))~s' . ($context['utf8'] ? 'u' : '') => '[/tr][/table]', + // Find [/td]s incorrectly followed by [/table]. + '~\[/td\][\s' . $non_breaking_space . ']*\[/table\]~s' . ($context['utf8'] ? 'u' : '') => '[/td][/tr][/table]', + // Find [table]s, [tr]s, and [/td]s (possibly correctly) followed by [td]. + '~\[(table|tr|/td)\]([\s' . $non_breaking_space . ']*)\[td\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_td_]', + // Now, any [td]s left should have a [tr] before them. + '~\[td\]~s' => '[tr][td]', + // Look for [tr]s which are correctly placed. + '~\[(table|/tr)\]([\s' . $non_breaking_space . ']*)\[tr\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_tr_]', + // Any remaining [tr]s should have a [table] before them. + '~\[tr\]~s' => '[table][tr]', + // Look for [/td]s followed by [/tr]. + '~\[/td\]([\s' . $non_breaking_space . ']*)\[/tr\]~s' . ($context['utf8'] ? 'u' : '') => '[/td]$1[_/tr_]', + // Any remaining [/tr]s should have a [/td]. + '~\[/tr\]~s' => '[/td][/tr]', + // Look for properly opened [li]s which aren't closed. + '~\[li\]([^\[\]]+?)\[li\]~s' => '[li]$1[_/li_][_li_]', + '~\[li\]([^\[\]]+?)$~s' => '[li]$1[/li]', + // Lists - find correctly closed items/lists. + '~\[/li\]([\s' . $non_breaking_space . ']*)\[/list\]~s' . ($context['utf8'] ? 'u' : '') => '[_/li_]$1[/list]', + // Find list items closed and then opened. + '~\[/li\]([\s' . $non_breaking_space . ']*)\[li\]~s' . ($context['utf8'] ? 'u' : '') => '[_/li_]$1[_li_]', + // Now, find any [list]s or [/li]s followed by [li]. + '~\[(list(?: [^\]]*?)?|/li)\]([\s' . $non_breaking_space . ']*)\[li\]~s' . ($context['utf8'] ? 'u' : '') => '[$1]$2[_li_]', + // Any remaining [li]s weren't inside a [list]. + '~\[li\]~' => '[list][li]', + // Any remaining [/li]s weren't before a [/list]. + '~\[/li\]~' => '[/li][/list]', + // Put the correct ones back how we found them. + '~\[_(li|/li|td|tr|/tr)_\]~' => '[$1]', + ); + + // Fix up some use of tables without [tr]s, etc. (it has to be done more than once to catch it all.) + for ($j = 0; $j < 3; $j++) + $parts[$i] = preg_replace(array_keys($mistake_fixes), $mistake_fixes, $parts[$i]); + } + } + + // Put it back together! + if (!$previewing) + $message = strtr(implode('', $parts), array(' ' => '  ', "\n" => '
    ', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); + else + $message = strtr(implode('', $parts), array(' ' => '  ', $context['utf8'] ? "\xC2\xA0" : "\xA0" => ' ')); + + // Now let's quickly clean up things that will slow our parser (which are common in posted code.) + $message = strtr($message, array('[]' => '[]', '['' => '['')); } // This is very simple, and just removes things done by preparsecode. function un_preparsecode($message) { - $parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $message, -1, PREG_SPLIT_DELIM_CAPTURE); - - // We're going to unparse only the stuff outside [code]... - for ($i = 0, $n = count($parts); $i < $n; $i++) - { - // If $i is a multiple of four (0, 4, 8, ...) then it's not a code section... - if ($i % 4 == 0) - { - $parts[$i] = preg_replace_callback('~\[html\](.+?)\[/html\]~i', function($m){return '[html]' . strtr(htmlspecialchars(stripslashes($m[1]), ENT_QUOTES), array('&#13;' => '
    ', '&#32;' => ' ')) . '[/html]';}, $parts[$i]); - - // Attempt to un-parse the time to something less awful. - $parts[$i] = preg_replace_callback('~\[time\](\d{0,10})\[/time\]~i', 'time_format__preg_callback', $parts[$i]); - } - } - - // Change breaks back to \n's and &nsbp; back to spaces. - return preg_replace('~~', "\n", str_replace(' ', ' ', implode('', $parts))); -} - -function action_fix__preg_callback($matches) -{ - return $matches[1] . preg_replace('~action(=|%3d)(?!dlattach)~i', 'action-', $matches[2]) . '[/img]'; + $parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $message, -1, PREG_SPLIT_DELIM_CAPTURE); + + // We're going to unparse only the stuff outside [code]... + for ($i = 0, $n = count($parts); $i < $n; $i++) + { + // If $i is a multiple of four (0, 4, 8, ...) then it's not a code section... + if ($i % 4 == 0) + { + $parts[$i] = preg_replace_callback('~\[html\](.+?)\[/html\]~i', function($m){return '[html]' . strtr(htmlspecialchars(stripslashes($m[1]), ENT_QUOTES), array('&#13;' => '
    ', '&#32;' => ' ')) . '[/html]';}, $parts[$i]); + + // Attempt to un-parse the time to something less awful. + $parts[$i] = preg_replace_callback('~\[time\](\d{0,10})\[/time\]~i', 'time_format__preg_callback', $parts[$i]); + } + } + + // Change breaks back to \n's and &nsbp; back to spaces. + return preg_replace('~~', "\n", str_replace(' ', ' ', implode('', $parts))); } function mime_convert__preg_callback($matches) { - $c = $matches[1]; - if (strlen($c) === 1 && ord($c[0]) <= 0x7F) - return $c; - elseif (strlen($c) === 2 && ord($c[0]) >= 0xC0 && ord($c[0]) <= 0xDF) - return '&#' . (((ord($c[0]) ^ 0xC0) << 6) + (ord($c[1]) ^ 0x80)) . ';'; - elseif (strlen($c) === 3 && ord($c[0]) >= 0xE0 && ord($c[0]) <= 0xEF) - return '&#' . (((ord($c[0]) ^ 0xE0) << 12) + ((ord($c[1]) ^ 0x80) << 6) + (ord($c[2]) ^ 0x80)) . ';'; - elseif (strlen($c) === 4 && ord($c[0]) >= 0xF0 && ord($c[0]) <= 0xF7) - return '&#' . (((ord($c[0]) ^ 0xF0) << 18) + ((ord($c[1]) ^ 0x80) << 12) + ((ord($c[2]) ^ 0x80) << 6) + (ord($c[3]) ^ 0x80)) . ';'; - else - return ''; + $c = $matches[1]; + if (strlen($c) === 1 && ord($c[0]) <= 0x7F) + return $c; + elseif (strlen($c) === 2 && ord($c[0]) >= 0xC0 && ord($c[0]) <= 0xDF) + return '&#' . (((ord($c[0]) ^ 0xC0) << 6) + (ord($c[1]) ^ 0x80)) . ';'; + elseif (strlen($c) === 3 && ord($c[0]) >= 0xE0 && ord($c[0]) <= 0xEF) + return '&#' . (((ord($c[0]) ^ 0xE0) << 12) + ((ord($c[1]) ^ 0x80) << 6) + (ord($c[2]) ^ 0x80)) . ';'; + elseif (strlen($c) === 4 && ord($c[0]) >= 0xF0 && ord($c[0]) <= 0xF7) + return '&#' . (((ord($c[0]) ^ 0xF0) << 18) + ((ord($c[1]) ^ 0x80) << 12) + ((ord($c[2]) ^ 0x80) << 6) + (ord($c[3]) ^ 0x80)) . ';'; + else + return ''; } function time_fix__preg_callback($matches) { - global $modSettings, $user_info; - return '[time]' . (is_numeric($matches[2]) || @strtotime($matches[2]) == 0 ? $matches[2] : strtotime($matches[2]) - ($matches[1] == 'absolute' ? 0 : (($modSettings['time_offset'] + $user_info['time_offset']) * 3600))) . '[/time]'; + global $modSettings, $user_info; + return '[time]' . (is_numeric($matches[2]) || @strtotime($matches[2]) == 0 ? $matches[2] : strtotime($matches[2]) - ($matches[1] == 'absolute' ? 0 : (($modSettings['time_offset'] + $user_info['time_offset']) * 3600))) . '[/time]'; } function nobbc__preg_callback($matches) { - return '[nobbc]' . strtr($matches[1], array('[' => '[', ']' => ']', ':' => ':', '@' => '@')) . '[/nobbc]'; + return '[nobbc]' . strtr($matches[1], array('[' => '[', ']' => ']', ':' => ':', '@' => '@')) . '[/nobbc]'; } function lowercase_tags__preg_callback($matches) { - return '[' . $matches[1] . strtolower($matches[2]) . $matches[3] . ']'; + return '[' . $matches[1] . strtolower($matches[2]) . $matches[3] . ']'; } function htmlspecial_html__preg_callback($matches) { - global $modSettings, $txt; - static $charset = null; - if ($charset === null) - $charset = empty($modSettings['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']; + global $modSettings, $txt; + static $charset = null; + if ($charset === null) + $charset = empty($modSettings['global_character_set']) ? $txt['lang_character_set'] : $modSettings['global_character_set']; - return '[html]' . strtr(htmlspecialchars($matches[1], ENT_QUOTES, $charset), array('\\"' => '"', '&#13;' => '
    ', '&#32;' => ' ', '&#91;' => '[', '&#93;' => ']')) . '[/html]'; + return '[html]' . strtr(htmlspecialchars($matches[1], ENT_QUOTES, $charset), array('\\"' => '"', '&#13;' => '
    ', '&#32;' => ' ', '&#91;' => '[', '&#93;' => ']')) . '[/html]'; } function time_format__preg_callback($matches) { - return '[time]' . timeformat($matches[1], false) . '[/time]'; + return '[time]' . timeformat($matches[1], false) . '[/time]'; } function word_break__preg_callback($matches) { - global $modSettings, $context; - return preg_replace('~(.{' . ($modSettings['fixLongWords'] - 1) . '})~' . ($context['utf8'] ? 'u' : ''), '$1< >', $matches[1]); + global $modSettings, $context; + return preg_replace('~(.{' . ($modSettings['fixLongWords'] - 1) . '})~' . ($context['utf8'] ? 'u' : ''), '$1< >', $matches[1]); } //this would normally transform the URL into a proxied URL, but here it does nothing function proxyurl($url) { - return $url; + return $url; } ?> diff --git a/parsing_extra.php b/parsing_extra.php new file mode 100644 index 00000000..5f9a5e3a --- /dev/null +++ b/parsing_extra.php @@ -0,0 +1,649 @@ +array()); + $context['browser']['is_gecko'] = true; + $context['browser']['is_konqueror'] = false; + $context['browser']['is_opera'] = false; + $context['browser']['is_ie'] = false; + + $txt['lang_character_set'] = 'ISO-8859-1'; + $txt['smf238'] = 'Code'; + $txt['smf240'] = 'Quote'; + $txt['smf239'] = 'Quote from'; + $txt[176] = 'on'; + $txt['lang_locale'] = 'en_US'; + $txt[289] = 'Cheesy'; + $txt[450] = 'Roll Eyes'; + $txt[288] = 'Angry'; + $txt[287] = 'Smiley'; + $txt[292] = 'Wink'; + $txt[293] = 'Grin'; + $txt[291] = 'Sad'; + $txt[294] = 'Shocked'; + $txt[295] = 'Cool'; + $txt[451] = 'Tongue'; + $txt[296] = 'Huh'; + $txt[526] = 'Embarrassed'; + $txt[527] = 'Lips sealed'; + $txt[529] = 'Kiss'; + $txt[530] = 'Cry'; + $txt[528] = 'Undecided'; + $txt['smf10'] = 'Today at '; + $txt['smf10b'] = 'Yesterday at '; + $txt['days_short'] = array('Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'); + $txt['days'] = array('Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'); + $txt['months_short'] = array(1 => 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'); + $txt['months'] = array(1 => 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'); + + $modSettings['enableBBC'] = 1; + $modSettings['cache_enable'] = 0; + $modSettings['enablePostHTML'] = 0; + $modSettings['max_image_width'] = 0; + $modSettings['max_image_height'] = 0; + $modSettings['autoLinkUrls'] = 1; + $modSettings['fixLongWords'] = 80; + $modSettings['smileys_url'] = 'https://bitcointalk.org/Smileys'; + $modSettings['time_offset'] = 0; + $modSettings['todayMod'] = 1; + + // Need these from theymos + $user_info['smiley_set'] = 'default'; + $user_info['time_offset'] = 0; + $user_info['time_format'] = '%I:%M:%S %p'; +} + +//the cache functions are highly implementation-dependant, so here they are just no-ops +function cache_put_data($key, $value, $ttl = 120) +{ + return; +} + +function cache_get_data($key, $ttl = 120) +{ + return; +} + +function highlight_php_code($code) +{ + global $context; + + // Remove special characters. + $code = un_htmlspecialchars(strtr($code, array('
    ' => "\n", "\t" => 'SMF_TAB();', '[' => '['))); + + $oldlevel = error_reporting(0); + + // It's easier in 4.2.x+. + if (@version_compare(PHP_VERSION, '4.2.0') == -1) + { + ob_start(); + @highlight_string($code); + $buffer = str_replace(array("\n", "\r"), '', ob_get_contents()); + ob_end_clean(); + } + else + $buffer = str_replace(array("\n", "\r"), '', @highlight_string($code, true)); + + error_reporting($oldlevel); + + // Yes, I know this is kludging it, but this is the best way to preserve tabs from PHP :P. + $buffer = preg_replace('~SMF_TAB(<(font color|span style)="[^"]*?">)?\(\);~', "
    \t
    ", $buffer); + + return strtr($buffer, array('\'' => ''', '' => '', '' => '')); +} +function un_htmlspecialchars($string) +{ + return strtr($string, array_flip(get_html_translation_table(HTML_SPECIALCHARS, ENT_QUOTES)) + array(''' => '\'', ' ' => ' ')); +} + +// Format a time to make it look purdy. +function timeformat($logTime, $show_today = true) +{ + global $user_info, $txt, $db_prefix, $modSettings, $func; + + // Offset the time. + $time = $logTime + ($user_info['time_offset'] + $modSettings['time_offset']) * 3600; + + // We can't have a negative date (on Windows, at least.) + if ($time < 0) + $time = 0; + + // Today and Yesterday? + if ($modSettings['todayMod'] >= 1 && $show_today === true) + { + // Get the current time. + $nowtime = forum_time(); + + $then = @getdate($time); + $now = @getdate($nowtime); + if(!$then) + $then = @getdate(0); + if(!$now) + $now = @getdate(0); + + // Try to make something of a time format string... + $s = strpos($user_info['time_format'], '%S') === false ? '' : ':%S'; + if (strpos($user_info['time_format'], '%H') === false && strpos($user_info['time_format'], '%T') === false) + $today_fmt = '%I:%M' . $s . ' %p'; + else + $today_fmt = '%H:%M' . $s; + + // Same day of the year, same year.... Today! + if ($then['yday'] == $now['yday'] && $then['year'] == $now['year']) + return $txt['smf10'] . timeformat($logTime, $today_fmt); + + // Day-of-year is one less and same year, or it's the first of the year and that's the last of the year... + if ($modSettings['todayMod'] == '2' && (($then['yday'] == $now['yday'] - 1 && $then['year'] == $now['year']) || ($now['yday'] == 0 && $then['year'] == $now['year'] - 1) && $then['mon'] == 12 && $then['mday'] == 31)) + return $txt['smf10b'] . timeformat($logTime, $today_fmt); + } + + $str = !is_bool($show_today) ? $show_today : $user_info['time_format']; + + if (setlocale(LC_TIME, $txt['lang_locale'])) + { + foreach (array('%a', '%A', '%b', '%B') as $token) + if (strpos($str, $token) !== false) + $str = str_replace($token, $func['ucwords'](strftime_updated($token, (int)$time)), $str); + } + else + { + // Do-it-yourself time localization. Fun. + foreach (array('%a' => 'days_short', '%A' => 'days', '%b' => 'months_short', '%B' => 'months') as $token => $text_label) + if (strpos($str, $token) !== false) + $str = str_replace($token, $txt[$text_label][(int) strftime_updated($token === '%a' || $token === '%A' ? '%w' : '%m', $time)], $str); + if (strpos($str, '%p')) + $str = str_replace('%p', (strftime_updated('%H', $time) < 12 ? 'am' : 'pm'), $str); + } + + // Format any other characters.. + return strftime_updated($str, (int)$time); +} + +function forum_time($use_user_offset = true, $timestamp = null) +{ + global $user_info, $modSettings; + + if ($timestamp === null) + $timestamp = time(); + elseif ($timestamp == 0) + return 0; + + return $timestamp + ($modSettings['time_offset'] + ($use_user_offset ? $user_info['time_offset'] : 0)) * 3600; +} + +function safe_serialize($value) +{ + // Make sure we use the byte count for strings even when strlen() is overloaded by mb_strlen() + if (function_exists('mb_internal_encoding') && + (((int) ini_get('mbstring.func_overload')) & 2)) + { + $mbIntEnc = mb_internal_encoding(); + mb_internal_encoding('ASCII'); + } + + $out = _safe_serialize($value); + + if (isset($mbIntEnc)) + mb_internal_encoding($mbIntEnc); + + return $out; +} +function _safe_serialize($value) +{ + if(is_null($value)) + return 'N;'; + + if(is_bool($value)) + return 'b:'. (int) $value .';'; + + if(is_int($value)) + return 'i:'. $value .';'; + + if(is_float($value)) + return 'd:'. str_replace(',', '.', $value) .';'; + + if(is_string($value)) + return 's:'. strlen($value) .':"'. $value .'";'; + + if(is_array($value)) + { + $out = ''; + foreach($value as $k => $v) + $out .= _safe_serialize($k) . _safe_serialize($v); + + return 'a:'. count($value) .':{'. $out .'}'; + } + + // safe_serialize cannot serialize resources or objects. + return false; +} + +// This gets all possible permutations of an array. +function permute($array) +{ + $orders = array($array); + + $n = count($array); + $p = range(0, $n); + for ($i = 1; $i < $n; null) + { + $p[$i]--; + $j = $i % 2 != 0 ? $p[$i] : 0; + + $temp = $array[$i]; + $array[$i] = $array[$j]; + $array[$j] = $temp; + + for ($i = 1; $p[$i] == 0; $i++) + $p[$i] = 1; + + $orders[] = $array; + } + + return $orders; +} +// Fix any URLs posted - ie. remove 'javascript:'. +function fixTags(&$message) +{ + global $modSettings; + + // WARNING: Editing the below can cause large security holes in your forum. + // Edit only if you are sure you know what you are doing. + + $fixArray = array( + // [img]http://...[/img] or [img width=1]http://...[/img] + array( + 'tag' => 'img', + 'protocols' => array('http', 'https'), + 'embeddedUrl' => false, + 'hasEqualSign' => false, + 'hasExtra' => true, + ), + // [url]http://...[/url] + array( + 'tag' => 'url', + 'protocols' => array('http', 'https', 'bitcoin:', 'magnet:'), + 'embeddedUrl' => true, + 'hasEqualSign' => false, + ), + // [url=http://...]name[/url] + array( + 'tag' => 'url', + 'protocols' => array('http', 'https', 'bitcoin:', 'magnet:'), + 'embeddedUrl' => true, + 'hasEqualSign' => true, + ), + // [iurl]http://...[/iurl] + array( + 'tag' => 'iurl', + 'protocols' => array('http', 'https', 'bitcoin:', 'magnet:'), + 'embeddedUrl' => true, + 'hasEqualSign' => false, + ), + // [iurl=http://...]name[/iurl] + array( + 'tag' => 'iurl', + 'protocols' => array('http', 'https', 'bitcoin:', 'magnet:'), + 'embeddedUrl' => true, + 'hasEqualSign' => true, + ), + // [ftp]ftp://...[/ftp] + array( + 'tag' => 'ftp', + 'protocols' => array('ftp', 'ftps'), + 'embeddedUrl' => true, + 'hasEqualSign' => false, + ), + // [ftp=ftp://...]name[/ftp] + array( + 'tag' => 'ftp', + 'protocols' => array('ftp', 'ftps'), + 'embeddedUrl' => true, + 'hasEqualSign' => true, + ), + // [flash]http://...[/flash] + array( + 'tag' => 'flash', + 'protocols' => array('http', 'https'), + 'embeddedUrl' => false, + 'hasEqualSign' => false, + 'hasExtra' => true, + ), + ); + + // Fix each type of tag. + foreach ($fixArray as $param) + fixTag($message, $param['tag'], $param['protocols'], $param['embeddedUrl'], $param['hasEqualSign'], !empty($param['hasExtra'])); + + // Now fix possible security problems with images loading links automatically... + $message = preg_replace_callback('~(\[img.*?\])(.+?)\[/img\]~is', 'action_fix__preg_callback', $message); + + // Limit the size of images posted? + if (!empty($modSettings['max_image_width']) || !empty($modSettings['max_image_height'])) + { + // Find all the img tags - with or without width and height. + preg_match_all('~\[img(\s+width=\d+)?(\s+height=\d+)?(\s+width=\d+)?\](.+?)\[/img\]~is', $message, $matches, PREG_PATTERN_ORDER); + + $replaces = array(); + foreach ($matches[0] as $match => $dummy) + { + // If the width was after the height, handle it. + $matches[1][$match] = !empty($matches[3][$match]) ? $matches[3][$match] : $matches[1][$match]; + + // Now figure out if they had a desired height or width... + $desired_width = !empty($matches[1][$match]) ? (int) substr(trim($matches[1][$match]), 6) : 0; + $desired_height = !empty($matches[2][$match]) ? (int) substr(trim($matches[2][$match]), 7) : 0; + + // One was omitted, or both. We'll have to find its real size... + if (empty($desired_width) || empty($desired_height)) + { + list ($width, $height) = url_image_size(un_htmlspecialchars($matches[4][$match])); //this is dead code, don't worry about url_image_size + + // They don't have any desired width or height! + if (empty($desired_width) && empty($desired_height)) + { + $desired_width = $width; + $desired_height = $height; + } + // Scale it to the width... + elseif (empty($desired_width) && !empty($height)) + $desired_width = (int) (($desired_height * $width) / $height); + // Scale if to the height. + elseif (!empty($width)) + $desired_height = (int) (($desired_width * $height) / $width); + } + + // If the width and height are fine, just continue along... + if ($desired_width <= $modSettings['max_image_width'] && $desired_height <= $modSettings['max_image_height']) + continue; + + // Too bad, it's too wide. Make it as wide as the maximum. + if ($desired_width > $modSettings['max_image_width'] && !empty($modSettings['max_image_width'])) + { + $desired_height = (int) (($modSettings['max_image_width'] * $desired_height) / $desired_width); + $desired_width = $modSettings['max_image_width']; + } + + // Now check the height, as well. Might have to scale twice, even... + if ($desired_height > $modSettings['max_image_height'] && !empty($modSettings['max_image_height'])) + { + $desired_width = (int) (($modSettings['max_image_height'] * $desired_width) / $desired_height); + $desired_height = $modSettings['max_image_height']; + } + + $replaces[$matches[0][$match]] = '[img' . (!empty($desired_width) ? ' width=' . $desired_width : '') . (!empty($desired_height) ? ' height=' . $desired_height : '') . ']' . $matches[4][$match] . '[/img]'; + } + + // If any img tags were actually changed... + if (!empty($replaces)) + $message = strtr($message, $replaces); + } +} + +function action_fix__preg_callback($matches) +{ + return $matches[1] . preg_replace('~action(=|%3d)(?!dlattach)~i', 'action-', $matches[2]) . '[/img]'; +} + +// Fix a specific class of tag - ie. url with =. +function fixTag(&$message, $myTag, $protocols, $embeddedUrl = false, $hasEqualSign = false, $hasExtra = false) +{ + global $boardurl, $scripturl; + + if (preg_match('~^([^:]+://[^/]+)~', $boardurl, $match) != 0) + $domain_url = $match[1]; + else + $domain_url = $boardurl . '/'; + + $replaces = array(); + + if ($hasEqualSign) + preg_match_all('~\[(' . $myTag . ')=([^\]]*?)\](?:(.+?)\[/(' . $myTag . ')\])?~is', $message, $matches); + else + preg_match_all('~\[(' . $myTag . ($hasExtra ? '(?:[^\]]*?)' : '') . ')\](.+?)\[/(' . $myTag . ')\]~is', $message, $matches); + + foreach ($matches[0] as $k => $dummy) + { + // Remove all leading and trailing whitespace. + $replace = trim($matches[2][$k]); + $this_tag = $matches[1][$k]; + $this_close = $hasEqualSign ? (empty($matches[4][$k]) ? '' : $matches[4][$k]) : $matches[3][$k]; + + $found = false; + foreach ($protocols as $protocol) + { + if (strpos($protocol, ':') === false) + $found = strncasecmp($replace, $protocol . '://', strlen($protocol) + 3) === 0; + else + $found = strncasecmp($replace, $protocol, strlen($protocol)) === 0; + if ($found) + break; + } + + if (!$found && $protocols[0] == 'http') + { + if (substr($replace, 0, 1) == '/') + $replace = $domain_url . $replace; + elseif (substr($replace, 0, 1) == '?') + $replace = $scripturl . $replace; + elseif (substr($replace, 0, 1) == '#' && $embeddedUrl) + { + $replace = '#' . preg_replace('~[^A-Za-z0-9_\-#]~', '', substr($replace, 1)); + $this_tag = 'iurl'; + $this_close = 'iurl'; + } + else + $replace = $protocols[0] . '://' . $replace; + } + elseif (!$found) + $replace = $protocols[0] . '://' . $replace; + + if ($hasEqualSign && $embeddedUrl) + $replaces[$matches[0][$k]] = '[' . $this_tag . '=' . $replace . ']' . (empty($matches[4][$k]) ? '' : $matches[3][$k] . '[/' . $this_close . ']'); + elseif ($hasEqualSign) + $replaces['[' . $matches[1][$k] . '=' . $matches[2][$k] . ']'] = '[' . $this_tag . '=' . $replace . ']'; + elseif ($embeddedUrl) + $replaces['[' . $matches[1][$k] . ']' . $matches[2][$k] . '[/' . $matches[3][$k] . ']'] = '[' . $this_tag . '=' . $replace . ']' . $matches[2][$k] . '[/' . $this_close . ']'; + else + $replaces['[' . $matches[1][$k] . ']' . $matches[2][$k] . '[/' . $matches[3][$k] . ']'] = '[' . $this_tag . ']' . $replace . '[/' . $this_close . ']'; + + } + + foreach ($replaces as $k => $v) + { + if ($k == $v) + unset($replaces[$k]); + } + + if (!empty($replaces)) + $message = strtr($message, $replaces); +} + +/** + * Locale-formatted strftime_updated using \IntlDateFormatter (PHP 8.1 compatible) + * This provides a cross-platform alternative to strftime_updated() for when it will be removed from PHP. + * Note that output can be slightly different between libc sprintf and this function as it is using ICU. + * + * Usage: + * use function \PHP81_BC\strftime_updated; + * echo strftime_updated('%A %e %B %Y %X', new \DateTime('2021-09-28 00:00:00'), 'fr_FR'); + * + * Original use: + * \setlocale('fr_FR.UTF-8', LC_TIME); + * echo \strftime_updated('%A %e %B %Y %X', strtotime('2021-09-28 00:00:00')); + * + * @param string $format Date format + * @param integer|string|DateTime $timestamp Timestamp + * @return string + * @author BohwaZ + */ +function strftime_updated(string $format, $timestamp = null, ?string $locale = null): string +{ + if (null === $timestamp) { + $timestamp = new \DateTime; + } + elseif (is_numeric($timestamp)) { + $timestamp = date_create('@' . $timestamp); + + if ($timestamp) { + $timestamp->setTimezone(new \DateTimezone(date_default_timezone_get())); + } + } + elseif (is_string($timestamp)) { + $timestamp = date_create($timestamp); + } + + if (!($timestamp instanceof \DateTimeInterface)) { + throw new \InvalidArgumentException('$timestamp argument is neither a valid UNIX timestamp, a valid date-time string or a DateTime object.'); + } + + $locale = substr((string) $locale, 0, 5); + + $intl_formats = [ + '%a' => 'EEE', // An abbreviated textual representation of the day Sun through Sat + '%A' => 'EEEE', // A full textual representation of the day Sunday through Saturday + '%b' => 'MMM', // Abbreviated month name, based on the locale Jan through Dec + '%B' => 'MMMM', // Full month name, based on the locale January through December + '%h' => 'MMM', // Abbreviated month name, based on the locale (an alias of %b) Jan through Dec + ]; + + $intl_formatter = function (\DateTimeInterface $timestamp, string $format) use ($intl_formats, $locale) { + $tz = $timestamp->getTimezone(); + $date_type = \IntlDateFormatter::FULL; + $time_type = \IntlDateFormatter::FULL; + $pattern = ''; + + // %c = Preferred date and time stamp based on locale + // Example: Tue Feb 5 00:45:10 2009 for February 5, 2009 at 12:45:10 AM + if ($format == '%c') { + $date_type = \IntlDateFormatter::LONG; + $time_type = \IntlDateFormatter::SHORT; + } + // %x = Preferred date representation based on locale, without the time + // Example: 02/05/09 for February 5, 2009 + elseif ($format == '%x') { + $date_type = \IntlDateFormatter::SHORT; + $time_type = \IntlDateFormatter::NONE; + } + // Localized time format + elseif ($format == '%X') { + $date_type = \IntlDateFormatter::NONE; + $time_type = \IntlDateFormatter::MEDIUM; + } + else { + $pattern = $intl_formats[$format]; + } + + return (new \IntlDateFormatter($locale, $date_type, $time_type, $tz, null, $pattern))->format($timestamp); + }; + + // Same order as https://www.php.net/manual/en/function.strftime_updated.php + $translation_table = [ + // Day + '%a' => $intl_formatter, + '%A' => $intl_formatter, + '%d' => 'd', + '%e' => function ($timestamp) { + return sprintf('% 2u', $timestamp->format('j')); + }, + '%j' => function ($timestamp) { + // Day number in year, 001 to 366 + return sprintf('%03d', $timestamp->format('z')+1); + }, + '%u' => 'N', + '%w' => 'w', + + // Week + '%U' => function ($timestamp) { + // Number of weeks between date and first Sunday of year + $day = new \DateTime(sprintf('%d-01 Sunday', $timestamp->format('Y'))); + return sprintf('%02u', 1 + ($timestamp->format('z') - $day->format('z')) / 7); + }, + '%V' => 'W', + '%W' => function ($timestamp) { + // Number of weeks between date and first Monday of year + $day = new \DateTime(sprintf('%d-01 Monday', $timestamp->format('Y'))); + return sprintf('%02u', 1 + ($timestamp->format('z') - $day->format('z')) / 7); + }, + + // Month + '%b' => $intl_formatter, + '%B' => $intl_formatter, + '%h' => $intl_formatter, + '%m' => 'm', + + // Year + '%C' => function ($timestamp) { + // Century (-1): 19 for 20th century + return floor($timestamp->format('Y') / 100); + }, + '%g' => function ($timestamp) { + return substr($timestamp->format('o'), -2); + }, + '%G' => 'o', + '%y' => 'y', + '%Y' => 'Y', + + // Time + '%H' => 'H', + '%k' => function ($timestamp) { + return sprintf('% 2u', $timestamp->format('G')); + }, + '%I' => 'h', + '%l' => function ($timestamp) { + return sprintf('% 2u', $timestamp->format('g')); + }, + '%M' => 'i', + '%p' => 'A', // AM PM (this is reversed on purpose!) + '%P' => 'a', // am pm + '%r' => 'h:i:s A', // %I:%M:%S %p + '%R' => 'H:i', // %H:%M + '%S' => 's', + '%T' => 'H:i:s', // %H:%M:%S + '%X' => $intl_formatter, // Preferred time representation based on locale, without the date + + // Timezone + '%z' => 'O', + '%Z' => 'T', + + // Time and Date Stamps + '%c' => $intl_formatter, + '%D' => 'm/d/Y', + '%F' => 'Y-m-d', + '%s' => 'U', + '%x' => $intl_formatter, + ]; + + $out = preg_replace_callback('/(?format($replace); + } + else { + return $replace($timestamp, $match[1]); + } + }, $format); + + $out = str_replace('%%', '%', $out); + return $out; +} + + +?> From 2c381e7c75360f59b602bef551ea3f0194b8d0c0 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 11 Jun 2024 11:14:11 -1000 Subject: [PATCH 050/231] ci(dockerfile): install php for bbcode parser --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index ab78109a..70ce0af6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,9 @@ FROM elixir:1.14.0 +# install php +RUN curl -sSL https://packages.sury.org/php/README.txt | bash -x +RUN apt update +RUN apt install php8.3 + # work in /app instead of / RUN mkdir -p /app WORKDIR /app From 9018fd267a7b84682fefb92e49f798f37f98627a Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 11 Jun 2024 11:18:53 -1000 Subject: [PATCH 051/231] ci(dockerfile): -y tho --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 70ce0af6..9490f2e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM elixir:1.14.0 # install php RUN curl -sSL https://packages.sury.org/php/README.txt | bash -x RUN apt update -RUN apt install php8.3 +RUN apt install -y php8.3 # work in /app instead of / RUN mkdir -p /app From 9f6f4d8b03b1a2a54714a95f8bd57bc6476ac280 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 11 Jun 2024 15:10:56 -1000 Subject: [PATCH 052/231] fix(json/post:by_thread_proxy): convert singular post to list --- lib/epochtalk_server_web/json/post_json.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 83673dc0..66e3592b 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -133,6 +133,9 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do board |> Map.put(:moderators, []) + # convert singular post to list + posts = if !is_map(posts), do: [posts] + # format post data posts = posts From c89b329daefcfa46e122143b46d5bef3c4f640fa Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 11 Jun 2024 15:13:26 -1000 Subject: [PATCH 053/231] fix(json/post:by_thread_proxy): remove ! --- lib/epochtalk_server_web/json/post_json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 66e3592b..ebf83844 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -134,7 +134,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do |> Map.put(:moderators, []) # convert singular post to list - posts = if !is_map(posts), do: [posts] + posts = if is_map(posts), do: [posts] # format post data posts = From b5dbccb39c90bcbd4a792f4598c4985493e20e59 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 13 Jun 2024 10:42:03 -1000 Subject: [PATCH 054/231] fix(bbcode-parser): use smileys and increase max input length --- lib/epochtalk_server_web/controllers/post.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 698d0634..4abb02d5 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -383,9 +383,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do Map.get(Application.get_env(:epochtalk_server, :frontend_config), :post_max_length) || Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"], body <- - Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 2, min: 1) do - body = String.replace(body, "'", "\'") - %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "', false);\"") + Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 4, min: 1) do + + body = String.replace(body, "'", "\'") + + %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") IO.inspect "'#{body}'" IO.inspect output IO.inspect status From 0b40270f59a506dba0106aed79ef4b790ce8d998 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 13 Jun 2024 11:25:56 -1000 Subject: [PATCH 055/231] fix(json/post:by_thread_proxy): add else case for post list conversion was breaking all other posts --- lib/epochtalk_server_web/json/post_json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index ebf83844..dded022e 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -134,7 +134,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do |> Map.put(:moderators, []) # convert singular post to list - posts = if is_map(posts), do: [posts] + posts = if is_map(posts), do: [posts], else: posts # format post data posts = From 33d37ee1fab775bb63cd803e2a65a3275b9f696a Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 17 Jun 2024 11:02:55 -1000 Subject: [PATCH 056/231] refactor(parser): move parsing code out of controller/route and directly parse when proxy data is returned --- lib/epochtalk_server_web/controllers/post.ex | 23 -------------------- lib/epochtalk_server_web/json/post_json.ex | 10 +++++++-- lib/epochtalk_server_web/router.ex | 1 - 3 files changed, 8 insertions(+), 26 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 2ddcfd38..27c46467 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -378,29 +378,6 @@ defmodule EpochtalkServerWeb.Controllers.Post do end end - @doc """ - Parse legacy `Post` containing bbcode using Porcelain - """ - def parse_legacy(conn, attrs) do - with post_max_length <- - Map.get(Application.get_env(:epochtalk_server, :frontend_config), :post_max_length) || - Application.get_env(:epochtalk_server, :frontend_config)["post_max_length"], - body <- - Validate.cast(attrs, "body", :string, required: true, max: post_max_length * 4, min: 1) do - - body = String.replace(body, "'", "\'") - - %Porcelain.Result{out: output, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - IO.inspect "'#{body}'" - IO.inspect output - IO.inspect status - render(conn, :parse_legacy, %{parsed_body: output}) - else - _ -> - ErrorHelpers.render_json_error(conn, 400, "Error, cannot parse") - end - end - ## === Private Authorization Helper Functions === defp can_authed_user_view_deleted_posts(nil, _thread_id), do: false diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index dded022e..1b7597ff 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -372,9 +372,15 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do end defp format_proxy_post_data_for_by_thread(post) do + body = String.replace(post.body || post.body_html, "'", "\'") + + %Porcelain.Result{out: parsed_body, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + IO.inspect "'#{body}'" + IO.inspect parsed_body + IO.inspect status + post - # if body_html does not exist, default to post.body - |> Map.put(:body_html, post.body) + |> Map.put(:body_html, parsed_body) |> Map.put(:user, %{ id: post.user_id, username: post.poster_name diff --git a/lib/epochtalk_server_web/router.ex b/lib/epochtalk_server_web/router.ex index 12292b0e..ad5e0c1e 100644 --- a/lib/epochtalk_server_web/router.ex +++ b/lib/epochtalk_server_web/router.ex @@ -77,7 +77,6 @@ defmodule EpochtalkServerWeb.Router do post "/register", User, :register post "/login", User, :login post "/confirm", User, :confirm - post "/bbcode", Post, :parse_legacy delete "/logout", User, :logout end From b489577cf4a996e0a0feffbcba4b1b0b2127888a Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 17 Jul 2024 17:38:38 -0700 Subject: [PATCH 057/231] fix(config/runtime): cast system env values to bool --- config/runtime.exs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 23dec5f9..8a4e17c9 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -164,7 +164,7 @@ if config_env() == :prod do # virtual_host: # true -> https://.s3..amazonaws.com # false -> https://s3..amazonaws.com/ - virtual_host: System.get_env("S3_VIRTUAL_HOST") || true, + virtual_host: System.get_env("S3_VIRTUAL_HOST") == "TRUE" || true, bucket: s3_bucket, path: System.get_env("S3_PATH") || "images/" end @@ -200,9 +200,9 @@ if config_env() == :prod do hostname: smf_repo_hostname, database: smf_repo_database, port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), - stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, + stacktrace: System.get_env("SMF_REPO_STACKTRACE") == "TRUE" || true, show_sensitive_data_on_connection_error: - System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || false, + System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") == "TRUE" || false, pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") # Configure Guardian for Runtime @@ -247,10 +247,10 @@ if config_env() == :prod do frontend_config: %{ frontend_url: System.get_env("FRONTEND_URL") || "http://localhost:8000", backend_url: System.get_env("BACKEND_URL") || "http://localhost:4000", - newbie_enabled: System.get_env("NEWBIE_ENABLED") || false, - login_required: System.get_env("LOGIN_REQUIRED") || false, - invite_only: System.get_env("INVITE_ONLY") || false, - verify_registration: System.get_env("VERIFY_REGISTRATION") || true, + newbie_enabled: System.get_env("NEWBIE_ENABLED") == "TRUE" || false, + login_required: System.get_env("LOGIN_REQUIRED") == "TRUE" || false, + invite_only: System.get_env("INVITE_ONLY") == "TRUE" || false, + verify_registration: System.get_env("VERIFY_REGISTRATION") == "TRUE" || true, post_max_length: System.get_env("POST_MAX_LENGTH") || 10_000, max_image_size: System.get_env("MAX_IMAGE_SIZE") || 10_485_760, max_avatar_size: System.get_env("MAX_AVATAR_SIZE") || 102_400, @@ -268,17 +268,17 @@ if config_env() == :prod do default_avatar_shape: System.get_env("WEBSITE_DEFAULT_AVATAR_SHAPE") || "circle" }, portal: %{ - enabled: System.get_env("PORTAL_ENABLED") || false, + enabled: System.get_env("PORTAL_ENABLED") == "TRUE" || false, board_id: System.get_env("PORTAL_BOARD_ID") || nil }, emailer: %{ - ses_mode: System.get_env("EMAILER_SES_MODE") || false, + ses_mode: System.get_env("EMAILER_SES_MODE") == "TRUE" || false, options: %{ from_address: System.get_env("EMAILER_OPTIONS_FROM_ADDRESS") || "info@epochtalk.com" } }, images: %{ - s3_mode: System.get_env("IMAGES_S3_MODE") || false, + s3_mode: System.get_env("IMAGES_S3_MODE") == "TRUE" || false, options: %{ local_host: System.get_env("IMAGES_OPTIONS_LOCAL_HOST") || "http://localhost:4000" } From 1da71dd90ffcd76c2e6593e9d2595ea5939d878e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 16 Aug 2024 02:56:42 -0700 Subject: [PATCH 058/231] yolo(config/runtime): add issuer to prod config --- config/runtime.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/config/runtime.exs b/config/runtime.exs index 5e7c87e7..665dc1c6 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -117,6 +117,7 @@ guardian_config = case config_env() do :prod -> [ + issuer: "EpochtalkServer", secret_key: get_env_or_raise_with_message.( "GUARDIAN_SECRET_KEY", From 2240cb19225d2a894b20bab4c8e71b5492398530 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 18 Sep 2024 10:44:31 -0700 Subject: [PATCH 059/231] feat(smf_loader): add smf_loader loads data from smf query output tsv file --- lib/epochtalk_server/smf_loader.ex | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 lib/epochtalk_server/smf_loader.ex diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex new file mode 100644 index 00000000..3c5f1b5c --- /dev/null +++ b/lib/epochtalk_server/smf_loader.ex @@ -0,0 +1,23 @@ +defmodule EpochtalkServer.SMFLoader do + # loads smf data from a tsv file + def load_from_tsv_file(path) do + with true <- if(File.exists?(path), do: true, else: "ファイルがない"), + {:ok, file} <- File.open(path), + headers <- + IO.read(file, :line) + |> String.split("\t") + |> Enum.map(&(&1 |> String.trim)), + file_stream <- + IO.stream(file, :line) + |> Stream.map(&(&1 |> String.split("\t"))) + |> Stream.map(fn line -> + # map each line to headers + Enum.zip(headers, line) + |> Enum.into(%{}) + end) do + file_stream + else + problem -> IO.puts("問題がある： #{inspect(problem)}") + end + end +end From 029e7cd6d3b6d35dc94c4923a7f161682056ee66 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 18 Sep 2024 10:51:55 -0700 Subject: [PATCH 060/231] refactor(smf_loader): trim lines before splitting them removes "\n" without having to iterate over each item in the new list again --- lib/epochtalk_server/smf_loader.ex | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 3c5f1b5c..297589c7 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -5,11 +5,18 @@ defmodule EpochtalkServer.SMFLoader do {:ok, file} <- File.open(path), headers <- IO.read(file, :line) - |> String.split("\t") - |> Enum.map(&(&1 |> String.trim)), + # clean line + |> String.trim() + |> String.split("\t"), + # |> Enum.map(&(&1 |> String.trim)), file_stream <- IO.stream(file, :line) - |> Stream.map(&(&1 |> String.split("\t"))) + |> Stream.map(&( + &1 + # clean line + |> String.trim() + |> String.split("\t") + )) |> Stream.map(fn line -> # map each line to headers Enum.zip(headers, line) From f590b8ff8d655d919c21685c367b7230c04bfb5a Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 24 Sep 2024 10:36:38 -0700 Subject: [PATCH 061/231] feat(smf_loader): map boards from smf to epochtalk format --- lib/epochtalk_server/smf_loader.ex | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 297589c7..42ad0885 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -27,4 +27,45 @@ defmodule EpochtalkServer.SMFLoader do problem -> IO.puts("問題がある： #{inspect(problem)}") end end + def map_boards_stream(boards_stream) do + now = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + {boards_stream, _slug_duplicate_index} = + boards_stream + |> Enum.map_reduce(%{}, fn smf_board, slugs -> + slug = + smf_board["name"] + |> String.replace(~r{ }, "-") + |> String.slice(0..99) + # handle duplicate slugs + slug_duplicate_index = Map.get(slugs, slug) + {slugs, slug} = + if slug_duplicate_index == nil do + # keep track of used slugs, return original slug + {Map.put(slugs, slug, 0), slug} + else + # replace last characters with index + new_slug = slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) + new_slug = new_slug <> to_string(slug_duplicate_index) + # increment used slugs, return new slug + {Map.put(slugs, slug, slug_duplicate_index + 1), new_slug} + end + board = + %{} + |> Map.put(:id, smf_board["ID_BOARD"]) + |> Map.put(:name, smf_board["name"]) + |> Map.put(:description, smf_board["description"]) + |> Map.put(:post_count, smf_board["numPosts"]) + |> Map.put(:thread_count, smf_board["numTopics"]) + |> Map.put(:viewable_by, "") + |> Map.put(:postable_by, "") + |> Map.put(:created_at, now) + |> Map.put(:imported_at, now) + |> Map.put(:updated_at, now) + |> Map.put(:meta, "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"") + |> Map.put(:right_to_left, "f") + |> Map.put(:slug, slug) + {board, slugs} + end) + boards_stream + end end From ad1c071bc9fcb9c66abce5d75e4a83495457ed77 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 24 Sep 2024 10:37:26 -0700 Subject: [PATCH 062/231] feat(smf_loader): implement tabulate_boards_map generate a string from epochtalk formatted data with header for insertion into postgres --- lib/epochtalk_server/smf_loader.ex | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 42ad0885..c710fcf4 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -68,4 +68,43 @@ defmodule EpochtalkServer.SMFLoader do end) boards_stream end + def tabulate_boards_map(boards_map) do + data = + boards_map + |> Enum.map(fn board -> + [ + board[:id], + board[:name], + board[:description], + board[:post_count], + board[:thread_count], + board[:viewable_by], + board[:postable_by], + board[:created_at], + board[:imported_at], + board[:updated_at], + board[:meta], + board[:right_to_left], + board[:slug] + ] + |> Enum.join("\t") + end) + + header = [ + "id", + "name", + "description", + "post_count", + "thread_count", + "viewable_by", + "postable_by", + "created_at", + "imported_at", + "updated_at", + "meta", + "right_to_left", + "slug" + ] |> Enum.join("\t") + [ header | data ] + end end From 4c917be72246c046bb28bd8b10aa31db73a1f730 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 24 Sep 2024 10:39:47 -0700 Subject: [PATCH 063/231] feat(smf_loader): write data to tsv file --- lib/epochtalk_server/smf_loader.ex | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index c710fcf4..ab9c3f31 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -107,4 +107,12 @@ defmodule EpochtalkServer.SMFLoader do ] |> Enum.join("\t") [ header | data ] end + def write_to_tsv_file(data, path) do + with false <- if(File.exists?(path), do: "ファイルがもうある", else: false), + file <- File.stream!(path) do + data |> Enum.into(file, fn line -> line <> "\n" end) + else + problem -> IO.puts("問題がある： #{inspect(problem)}") + end + end end From b719af730a83d69246fd65b15fef4b26db37d75a Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 24 Sep 2024 10:40:19 -0700 Subject: [PATCH 064/231] feat(smf_loader): use tsv file parser/converter/writer in convert_smf_boards_tsv_file --- lib/epochtalk_server/smf_loader.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index ab9c3f31..d316c555 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -1,4 +1,11 @@ defmodule EpochtalkServer.SMFLoader do + # converts smf_boards tsv file to epochtalk boards tsv file + def convert_smf_boards_tsv_file(path) do + load_from_tsv_file(path) + |> map_boards_stream() + |> tabulate_boards_map() + |> write_to_tsv_file("boards.tsv") + end # loads smf data from a tsv file def load_from_tsv_file(path) do with true <- if(File.exists?(path), do: true, else: "ファイルがない"), From c615bdd77e84bad3593ebc98a50af9798a55e780 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 24 Sep 2024 10:40:58 -0700 Subject: [PATCH 065/231] feat(mix): add recase to deps --- mix.exs | 1 + mix.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/mix.exs b/mix.exs index cd2da86a..2bf86022 100644 --- a/mix.exs +++ b/mix.exs @@ -65,6 +65,7 @@ defmodule EpochtalkServer.MixProject do {:plug_cowboy, "~> 2.5"}, {:poison, "~> 3.0"}, {:postgrex, "~> 0.17.1"}, + {:recase, "~> 0.8.1", only: [:dev]}, {:redix, "~> 1.2.2"}, {:remote_ip, "~> 1.1.0"}, {:sweet_xml, "~> 0.7"}, diff --git a/mix.lock b/mix.lock index 26e0df97..a932ef50 100644 --- a/mix.lock +++ b/mix.lock @@ -67,6 +67,7 @@ "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "recase": {:hex, :recase, "0.8.1", "ab98cd35857a86fa5ca99036f575241d71d77d9c2ab0c39aacf1c9b61f6f7d1d", [:mix], [], "hexpm", "9fd8d63e7e43bd9ea385b12364e305778b2bbd92537e95c4b2e26fc507d5e4c2"}, "redix": {:hex, :redix, "1.2.4", "8d980da0800262d5e8dd718af09d549bd788b1b08651d03cbc0854f5fb35f0e6", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fb6d7327b0d4190e88e35cf68551130b2c24f6662ccc08007bee64cbd312e1c5"}, "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, From 180fd2b8760f02b4d7616a6f8075aa0101982516 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 24 Sep 2024 11:42:39 -0700 Subject: [PATCH 066/231] feat(smf_loader): process board mappings with boards and return --- lib/epochtalk_server/smf_loader.ex | 44 +++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index d316c555..577663d0 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -1,10 +1,23 @@ defmodule EpochtalkServer.SMFLoader do + alias EpochtalkServer.Models.BoardMapping + # converts smf_boards tsv file to epochtalk boards tsv file def convert_smf_boards_tsv_file(path) do - load_from_tsv_file(path) + {boards, board_mappings} = load_from_tsv_file(path) |> map_boards_stream() + + # write boards to file for import + boards |> tabulate_boards_map() |> write_to_tsv_file("boards.tsv") + + # return board mappings for later insertion + board_mappings + end + def insert_board_mappings(board_mappings) do + # board mappings + board_mappings + |> BoardMapping.update() end # loads smf data from a tsv file def load_from_tsv_file(path) do @@ -36,7 +49,7 @@ defmodule EpochtalkServer.SMFLoader do end def map_boards_stream(boards_stream) do now = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - {boards_stream, _slug_duplicate_index} = + {board_mapping_pairs, _slug_duplicate_index} = boards_stream |> Enum.map_reduce(%{}, fn smf_board, slugs -> slug = @@ -56,6 +69,7 @@ defmodule EpochtalkServer.SMFLoader do # increment used slugs, return new slug {Map.put(slugs, slug, slug_duplicate_index + 1), new_slug} end + # build board board = %{} |> Map.put(:id, smf_board["ID_BOARD"]) @@ -71,9 +85,31 @@ defmodule EpochtalkServer.SMFLoader do |> Map.put(:meta, "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"") |> Map.put(:right_to_left, "f") |> Map.put(:slug, slug) - {board, slugs} + + # build board mapping + board_mapping = + case {smf_board["childLevel"], smf_board["ID_PARENT"]} do + # top level board, map under category + {"0", "0"} -> + %{ + type: "board", + id: smf_board["ID_BOARD"] |> String.to_integer(), + name: smf_board["name"], + category_id: smf_board["ID_CAT"] |> String.to_integer(), + view_order: smf_board["boardOrder"] |> String.to_integer() + } + _ -> + %{ + type: "board", + id: smf_board["ID_BOARD"] |> String.to_integer(), + name: smf_board["name"], + parent_id: smf_board["ID_PARENT"] |> String.to_integer(), + view_order: smf_board["boardOrder"] |> String.to_integer() + } + end + {{board, board_mapping}, slugs} end) - boards_stream + Enum.unzip(board_mapping_pairs) end def tabulate_boards_map(boards_map) do data = From 6de12cf0db3165bffee190e6b70a8fe80c138121 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 25 Sep 2024 13:21:35 -0700 Subject: [PATCH 067/231] fix(smf_loader): switch file import mode for boards from Stream to File read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes issue with ç converting to other characters --- lib/epochtalk_server/smf_loader.ex | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 577663d0..608ad3d9 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -22,27 +22,28 @@ defmodule EpochtalkServer.SMFLoader do # loads smf data from a tsv file def load_from_tsv_file(path) do with true <- if(File.exists?(path), do: true, else: "ファイルがない"), - {:ok, file} <- File.open(path), + {:ok, file} <- File.read(path), + file_lines <- file |> String.trim() |> String.split("\n"), + [header_line | file_lines] <- file_lines, headers <- - IO.read(file, :line) # clean line + header_line |> String.trim() |> String.split("\t"), - # |> Enum.map(&(&1 |> String.trim)), - file_stream <- - IO.stream(file, :line) - |> Stream.map(&( + smf_boards <- + file_lines + |> Enum.map(&( &1 # clean line |> String.trim() |> String.split("\t") )) - |> Stream.map(fn line -> + |> Enum.map(fn line -> # map each line to headers Enum.zip(headers, line) |> Enum.into(%{}) end) do - file_stream + smf_boards else problem -> IO.puts("問題がある： #{inspect(problem)}") end From 7b24ef88310668f4c79bda088119862d7af6d563 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 25 Sep 2024 13:23:10 -0700 Subject: [PATCH 068/231] fix(smf_loader): decode html entities in board name and description fixes issue with html entities displaying in frontend also resolves issue with slug names being too long --- lib/epochtalk_server/smf_loader.ex | 9 +++++---- mix.exs | 1 + mix.lock | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 608ad3d9..13c72dab 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -55,6 +55,7 @@ defmodule EpochtalkServer.SMFLoader do |> Enum.map_reduce(%{}, fn smf_board, slugs -> slug = smf_board["name"] + |> HtmlEntities.decode() |> String.replace(~r{ }, "-") |> String.slice(0..99) # handle duplicate slugs @@ -74,8 +75,8 @@ defmodule EpochtalkServer.SMFLoader do board = %{} |> Map.put(:id, smf_board["ID_BOARD"]) - |> Map.put(:name, smf_board["name"]) - |> Map.put(:description, smf_board["description"]) + |> Map.put(:name, smf_board["name"] |> HtmlEntities.decode()) + |> Map.put(:description, smf_board["description"] |> HtmlEntities.decode()) |> Map.put(:post_count, smf_board["numPosts"]) |> Map.put(:thread_count, smf_board["numTopics"]) |> Map.put(:viewable_by, "") @@ -95,7 +96,7 @@ defmodule EpochtalkServer.SMFLoader do %{ type: "board", id: smf_board["ID_BOARD"] |> String.to_integer(), - name: smf_board["name"], + name: smf_board["name"] |> HtmlEntities.decode(), category_id: smf_board["ID_CAT"] |> String.to_integer(), view_order: smf_board["boardOrder"] |> String.to_integer() } @@ -103,7 +104,7 @@ defmodule EpochtalkServer.SMFLoader do %{ type: "board", id: smf_board["ID_BOARD"] |> String.to_integer(), - name: smf_board["name"], + name: smf_board["name"] |> HtmlEntities.decode(), parent_id: smf_board["ID_PARENT"] |> String.to_integer(), view_order: smf_board["boardOrder"] |> String.to_integer() } diff --git a/mix.exs b/mix.exs index 2bf86022..bfad03be 100644 --- a/mix.exs +++ b/mix.exs @@ -55,6 +55,7 @@ defmodule EpochtalkServer.MixProject do {:hackney, "~> 1.9"}, {:hammer, "~> 6.2"}, {:hammer_backend_redis, "~> 6.1"}, + {:html_entities, "~> 0.5.2", only: [:dev]}, {:html_sanitize_ex, "~> 1.4"}, {:iteraptor, git: "https://github.com/epochtalk/elixir-iteraptor.git", tag: "1.13.1"}, {:jason, "~> 1.4.0"}, diff --git a/mix.lock b/mix.lock index a932ef50..faacf046 100644 --- a/mix.lock +++ b/mix.lock @@ -37,6 +37,7 @@ "hammer": {:hex, :hammer, "6.2.1", "5ae9c33e3dceaeb42de0db46bf505bd9c35f259c8defb03390cd7556fea67ee2", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "b9476d0c13883d2dc0cc72e786bac6ac28911fba7cc2e04b70ce6a6d9c4b2bdc"}, "hammer_backend_redis": {:hex, :hammer_backend_redis, "6.1.2", "eb296bb4924928e24135308b2afc189201fd09411c870c6bbadea444a49b2f2c", [:mix], [{:hammer, "~> 6.0", [hex: :hammer, repo: "hexpm", optional: false]}, {:redix, "~> 1.1", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "217ea066278910543a5e9b577d5bf2425419446b94fe76bdd9f255f39feec9fa"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, + "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.3", "67b3d9fa8691b727317e0cc96b9b3093be00ee45419ffb221cdeee88e75d1360", [:mix], [{:mochiweb, "~> 2.15 or ~> 3.1", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "87748d3c4afe949c7c6eb7150c958c2bcba43fc5b2a02686af30e636b74bccb7"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "iteraptor": {:git, "https://github.com/epochtalk/elixir-iteraptor.git", "d8d1c386c38e06bdfcf60c9ce1abf8e49161cab4", [tag: "1.13.1"]}, From 431bc7a7663a950b93d3d942e52317a837658211 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 25 Sep 2024 13:25:13 -0700 Subject: [PATCH 069/231] feat(smf_loader): implement categories mappings load categories from postgtres tsv file and prepare for insertion in board mappings --- lib/epochtalk_server/smf_loader.ex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 13c72dab..3c65a1ab 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -14,6 +14,20 @@ defmodule EpochtalkServer.SMFLoader do # return board mappings for later insertion board_mappings end + def load_categories_mappings_from_tsv_file(path) do + # load categories (in postgres format) + # and return category mappings for later insertion + load_from_tsv_file(path) + |> Enum.map(fn category -> + # generate category mapping + %{ + type: "category", + id: category["id"] |> String.to_integer(), + name: category["name"], + view_order: category["view_order"] |> String.to_integer() + } + end) + end def insert_board_mappings(board_mappings) do # board mappings board_mappings From 7e94c0b767e3785d1698df2e594c1aff9df91b45 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 25 Sep 2024 13:28:06 -0700 Subject: [PATCH 070/231] refactor(smf_loader): remove recase as dependency no longer used --- mix.exs | 1 - mix.lock | 1 - 2 files changed, 2 deletions(-) diff --git a/mix.exs b/mix.exs index bfad03be..d8cdc1cf 100644 --- a/mix.exs +++ b/mix.exs @@ -66,7 +66,6 @@ defmodule EpochtalkServer.MixProject do {:plug_cowboy, "~> 2.5"}, {:poison, "~> 3.0"}, {:postgrex, "~> 0.17.1"}, - {:recase, "~> 0.8.1", only: [:dev]}, {:redix, "~> 1.2.2"}, {:remote_ip, "~> 1.1.0"}, {:sweet_xml, "~> 0.7"}, diff --git a/mix.lock b/mix.lock index faacf046..766fe1c8 100644 --- a/mix.lock +++ b/mix.lock @@ -68,7 +68,6 @@ "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "recase": {:hex, :recase, "0.8.1", "ab98cd35857a86fa5ca99036f575241d71d77d9c2ab0c39aacf1c9b61f6f7d1d", [:mix], [], "hexpm", "9fd8d63e7e43bd9ea385b12364e305778b2bbd92537e95c4b2e26fc507d5e4c2"}, "redix": {:hex, :redix, "1.2.4", "8d980da0800262d5e8dd718af09d549bd788b1b08651d03cbc0854f5fb35f0e6", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fb6d7327b0d4190e88e35cf68551130b2c24f6662ccc08007bee64cbd312e1c5"}, "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, From 80d48a889106f87f25e64442726f5ee25a5e55cd Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 26 Sep 2024 15:03:46 -1000 Subject: [PATCH 071/231] feat(recent-thread-proxy): implement proxy for recent threads --- .../controllers/thread.ex | 17 +++++- .../helpers/proxy_conversion.ex | 54 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 4f5a4b61..78451ee4 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -28,7 +28,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do alias EpochtalkServer.Models.Mention alias EpochtalkServerWeb.Helpers.ProxyConversion - plug :check_proxy when action in [:by_board, :slug_to_id, :viewed] + plug :check_proxy when action in [:by_board, :slug_to_id, :viewed, :recent] @doc """ Used to retrieve recent threads @@ -744,6 +744,11 @@ defmodule EpochtalkServerWeb.Controllers.Thread do |> send_resp(200, []) |> halt() + :recent -> + conn + |> proxy_recent(conn.params) + |> halt() + _ -> conn end @@ -772,6 +777,16 @@ defmodule EpochtalkServerWeb.Controllers.Thread do end end + defp proxy_recent(conn, _attrs) do + with user <- Guardian.Plug.current_resource(conn), + user_priority <- ACL.get_user_priority(conn), + threads <- ProxyConversion.build_model("threads.recent") do + render(conn, :recent, %{threads: threads}) + else + _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch recent threads") + end + end + # === Private Authorization Helpers Functions === defp handle_check_user_can_moderate(user, moderated) do diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index be1a411b..a2a4bcec 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -47,6 +47,60 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_model(model_type) do + case model_type do + "threads.recent" -> + build_recent_threads() + + _ -> + build_model(nil, nil, nil, nil) + end + end + + def build_recent_threads() do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + from(t in "smf_topics", + limit: 5, + where: t.id_board not in ^id_board_blacklist, + order_by: [desc: t.id_last_msg] + ) + |> join(:left, [t], m in "smf_messages", on: t.id_first_msg == m.id_msg) + |> join(:left, [t], m in "smf_messages", on: t.id_last_msg == m.id_msg) + |> join(:left, [t], b in "smf_boards", on: t.id_board == b.id_board) + |> select([t, f, l, b], %{ + id: t.id_topic, + slug: t.id_topic, + board_id: t.id_board, + board_name: b.name, + board_slug: t.id_board, + sticky: t.isSticky, + locked: t.locked, + poll: t.id_poll > 0, + moderated: t.selfModerated, + first_post_id: t.id_first_msg, + last_post_id: t.id_last_msg, + title: f.subject, + updated_at: l.posterTime * 1000, + last_post_created_at: l.posterTime * 1000, + last_post_user_id: l.id_member, + last_post_username: l.posterName, + view_count: t.numViews, + last_post_position: nil, + last_post_deleted: false, + last_post_user_deleted: false, + new_post_id: nil, + new_post_position: nil, + is_proxy: true + }) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Recent threads not found"} + + threads -> threads + end + end + def build_categories(ids) do from(c in "smf_categories", limit: ^length(ids), From 120605d8c02a030a77feebac36fe401909490a84 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 27 Sep 2024 09:31:52 -0700 Subject: [PATCH 072/231] fix(config): Fix error with missing id_board_blacklist --- config/config.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 7d4ecedf..5e196796 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,7 +14,8 @@ config :epochtalk_server, ecto_repos: [EpochtalkServer.Repo] config :epochtalk_server, proxy_config: %{ threads_seq: System.get_env("THREADS_SEQ") || "6000000", - boards_seq: System.get_env("BOARDS_SEQ") || "500" + boards_seq: System.get_env("BOARDS_SEQ") || "500", + id_board_blacklist: [] } # Configure Porcelain From ef3062994cc7f2eac8012d1477adc63c3440efd9 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 27 Sep 2024 09:32:45 -0700 Subject: [PATCH 073/231] refactor(dev): Refactored port and pool_size function --- config/dev.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 6f86300e..0fc4128b 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -5,11 +5,11 @@ config :epochtalk_server, EpochtalkServer.SmfRepo, password: System.get_env("SMF_REPO_PASSWORD"), hostname: System.get_env("SMF_REPO_HOSTNAME"), database: System.get_env("SMF_REPO_DATABASE"), - port: System.get_env("SMF_REPO_PORT") || "3306" |> String.to_integer(), + port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, show_sensitive_data_on_connection_error: System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || true, - pool_size: System.get_env("SMF_REPO_POOL_SIZE") || "10" |> String.to_integer() + pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") # For Development `config/dev.exs` loads the "Local" adapter which allows preview # of sent emails at the url `/dev/mailbox`. To test SMTP in Development mode, From 564b2908deef0117a2d0339676bd126e2770a6ff Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 27 Sep 2024 09:34:05 -0700 Subject: [PATCH 074/231] feat(user-avatars): Added functionality to pull user-avatars from proxy db to show on posts --- .../helpers/proxy_conversion.ex | 85 +++++++++---------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index be1a411b..5eb4a12c 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -110,9 +110,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, order_by: [desc: t.id_last_msg] ) - |> join(:left, [t], m in "smf_messages", on: t.id_first_msg == m.id_msg) - |> join(:left, [t], m in "smf_messages", on: t.id_last_msg == m.id_msg) - |> select([t, f, l], %{ + |> join(:left, [t], f in "smf_messages", on: t.id_first_msg == f.id_msg) + |> join(:left, [t], l in "smf_messages", on: t.id_last_msg == l.id_msg) + |> join(:left, [t, f, l], m in "smf_members", on: l.id_member == m.id_member) + |> join(:left, [t, f, l, m], a in "smf_attachments", + on: m.id_member == a.id_member and a.attachmentType == 1 + ) + |> select([t, f, l, m, a], %{ id: t.id_topic, slug: t.id_topic, board_id: t.id_board, @@ -135,6 +139,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_post_user_id: l.id_member, last_post_username: l.posterName, last_post_user_deleted: false, + last_post_avatar: + fragment( + "if(? <>'',concat('/avatars/',?),ifnull(concat('/useravatars/',?),''))", + m.avatar, + m.avatar, + a.filename + ), last_viewed: nil, is_proxy: true }) @@ -220,23 +231,34 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do from m in "smf_messages", where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, select: %{count: count(m.id_topic)} from(m in "smf_messages", - limit: 15, + limit: ^per_page, where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, - order_by: [asc: m.posterTime], - select: %{ - id: m.id_msg, - thread_id: m.id_topic, - board_id: m.id_board, - user_id: m.id_member, - title: m.subject, - body: m.body, - updated_at: m.modifiedTime, - username: m.posterName, - poster_time: m.posterTime, - poster_name: m.posterName, - modified_time: m.modifiedTime - } + order_by: [asc: m.posterTime] ) + |> join(:left, [m], u in "smf_members", on: m.id_member == u.id_member) + |> join(:left, [m, u], a in "smf_attachments", + on: u.id_member == a.id_member and a.attachmentType == 1 + ) + |> select([m, u, a], %{ + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + user_id: m.id_member, + title: m.subject, + body: m.body, + updated_at: m.modifiedTime, + username: m.posterName, + poster_time: m.posterTime, + poster_name: m.posterName, + modified_time: m.modifiedTime, + avatar: + fragment( + "if(? <>'',concat('/avatars/',?),ifnull(concat('/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ) + }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do {:ok, [], _} -> @@ -272,31 +294,4 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do {:ok, List.first(object), data} end end - - defp get_post(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) - from(m in "smf_messages", - limit: 1, - where: m.id_msg == ^id and m.id_board not in ^id_board_blacklist, - select: %{ - id: m.id_msg, - thread_id: m.id_topic, - board_id: m.id_board, - user_id: m.id_member, - title: m.subject, - body: m.body, - username: m.posterName, - created_at: m.posterTime, - modified_time: m.modifiedTime - } - ) - |> SmfRepo.one() - |> case do - nil -> - {:error, "Post not found for id: #{id}"} - - post -> - post - end - end end From 69d28f77b60065cac69dc0b7b6e3074e093b779a Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 27 Sep 2024 11:25:02 -1000 Subject: [PATCH 075/231] fix(recent-threads): add json for recent threads, remove unneeded params --- .../controllers/thread.ex | 4 +- lib/epochtalk_server_web/json/thread_json.ex | 42 ++++++++++++++++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 78451ee4..b0fb79fb 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -778,9 +778,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do end defp proxy_recent(conn, _attrs) do - with user <- Guardian.Plug.current_resource(conn), - user_priority <- ACL.get_user_priority(conn), - threads <- ProxyConversion.build_model("threads.recent") do + with threads <- ProxyConversion.build_model("threads.recent") do render(conn, :recent, %{threads: threads}) else _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch recent threads") diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 2417a089..3111d0e7 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -13,8 +13,46 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do threads: threads }) do threads - |> Enum.map(fn thread -> - %{id: thread.id, slug: thread.slug, updated_at: thread.updated_at} + |> Enum.map(fn t -> + %{ + id: t.id, + slug: t.slug, + locked: t.locked, + sticky: t.sticky, + moderated: t.moderated, + poll: t.poll, + title: t.title, + updated_at: t.updated_at, + view_count: t.view_count, + board: %{ + id: t.board_id, + name: t.board_name, + slug: t.board_slug + }, + post: %{ + id: t.last_post_id, + position: t.last_post_position, + created_at: t.last_post_created_at, + deleted: t.last_post_deleted + }, + user: + if(t.last_post_user_deleted || t.last_post_deleted, + do: %{ + id: "", + username: "", + deleted: true + }, + else: %{ + id: t.last_post_user_id, + username: t.last_post_username, + deleted: t.last_post_user_deleted + } + ), + lastest: + if(t.new_post_id || t.new_post_position, + do: %{id: t.new_post_id, position: t.new_post_position || 1} + ) + } end) end From 73dac84d96dd3ae20ab11818859d2b2477ee3b3e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 2 Oct 2024 12:03:02 -0700 Subject: [PATCH 076/231] feat(smf_loader): replace "/" in board slug with "-" allows board to be loaded properly --- lib/epochtalk_server/smf_loader.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 3c65a1ab..9c0655e3 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -70,7 +70,7 @@ defmodule EpochtalkServer.SMFLoader do slug = smf_board["name"] |> HtmlEntities.decode() - |> String.replace(~r{ }, "-") + |> String.replace(~r{[ /]}, "-") |> String.slice(0..99) # handle duplicate slugs slug_duplicate_index = Map.get(slugs, slug) From 85e1e6ddd7f7cf2973cd8187cb70c74ed3d2aa55 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 3 Oct 2024 10:31:09 -1000 Subject: [PATCH 077/231] feat(smf-polls): query poll data from smf with threads --- lib/epochtalk_server_web/controllers/post.ex | 4 +- .../controllers/thread.ex | 77 +++++++++---------- .../helpers/proxy_conversion.ex | 52 +++++++++++++ lib/epochtalk_server_web/json/post_json.ex | 3 + 4 files changed, 95 insertions(+), 41 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 99c44e57..60923609 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -484,9 +484,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "posts.byThread"), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_thread", thread_id, page, limit) do + ProxyConversion.build_model("posts.by_thread", thread_id, page, limit), + poll <- ProxyConversion.build_model("poll.by_thread", thread_id) do render(conn, :by_thread_proxy, %{ posts: posts, + poll: poll, user: user, page: page, limit: limit, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index b0fb79fb..b64ced04 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -705,55 +705,52 @@ defmodule EpochtalkServerWeb.Controllers.Thread do do: UserThreadView.upsert(user.id, thread_id) defp check_proxy(conn, _) do - conn = - case conn.private.phoenix_action do - :by_board -> - %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) - boards_seq = boards_seq |> String.to_integer() + case conn.private.phoenix_action do + :by_board -> + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() - if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do - conn - |> proxy_by_board(conn.params) - |> halt() - else - conn - end - - :slug_to_id -> - case Integer.parse(conn.params["slug"]) do - {_, ""} -> - slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) + if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do + conn + |> proxy_by_board(conn.params) + |> halt() + else + conn + end - %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) - threads_seq = threads_seq |> String.to_integer() + :slug_to_id -> + case Integer.parse(conn.params["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - if slug_as_id < threads_seq do - conn - |> render(:slug_to_id, id: slug_as_id) - |> halt() - else - conn - end + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() - _ -> + if slug_as_id < threads_seq do conn - end + |> render(:slug_to_id, id: slug_as_id) + |> halt() + else + conn + end - :viewed -> - conn - |> send_resp(200, []) - |> halt() + _ -> + conn + end - :recent -> - conn - |> proxy_recent(conn.params) - |> halt() + :viewed -> + conn + |> send_resp(200, []) + |> halt() - _ -> - conn - end + :recent -> + conn + |> proxy_recent(conn.params) + |> halt() - conn + _ -> + conn + end end defp proxy_by_board(conn, attrs) do diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index a2a4bcec..7c46b8ca 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -47,6 +47,16 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_model(model_type, id) do + case model_type do + "poll.by_thread" -> + build_poll(id) + + _ -> + build_model(nil, nil, nil, nil) + end + end + def build_model(model_type) do case model_type do "threads.recent" -> @@ -57,6 +67,48 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_poll(thread_id) do + from(t in "smf_topics", + where: t.id_topic == ^thread_id + ) + |> join(:left, [t], p in "smf_polls", on: t.id_poll == p.id_poll) + |> select([t, p], %{ + id: t.id_poll, + change_vote: p.changeVote, + display_mode: "always", + expiration: p.expireTime * 1000, + has_voted: false, + locked: (p.votingLocked == 1), + max_answers: p.maxVotes, + question: p.question + }) + |> SmfRepo.one() + |> case do + [] -> + {:error, "Poll for thread not found"} + + thread -> + from(t in "smf_topics", + where: t.id_topic == ^thread_id + ) + |> join(:left, [t], pc in "smf_poll_choices", on: t.id_poll == pc.id_poll) + |> select([t, pc], %{ + id: pc.id_choice, + selected: false, + votes: pc.votes, + answer: pc.label + }) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Poll for thread not found"} + + answers -> + if thread.id > 0, do: Map.put(thread, :answers, answers), else: nil + end + end + end + def build_recent_threads() do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(t in "smf_topics", diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 1b7597ff..eb49f42e 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -117,6 +117,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do def by_thread_proxy(%{ posts: posts, page: page, + poll: poll, limit: limit }) do {:ok, thread} = @@ -126,6 +127,8 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do ProxyConversion.build_model("thread", [List.first(posts).thread_id], page, limit) end + thread = Map.put(thread, :poll, poll) + # format board data {:ok, board} = ProxyConversion.build_model("board", [thread.board_id], 1, 1) From b134fd72d673dff10c8dcc7cebf0a1dc50658510 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 3 Oct 2024 10:47:27 -1000 Subject: [PATCH 078/231] fix(long-post-query): order by id_msg instead of poster time to speed up query for posts --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 62c3255e..c23f5a28 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -287,7 +287,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do from(m in "smf_messages", limit: ^per_page, where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, - order_by: [asc: m.posterTime] + order_by: [asc: m.id_msg] ) |> join(:left, [m], u in "smf_members", on: m.id_member == u.id_member) |> join(:left, [m, u], a in "smf_attachments", From 14fa2e2b33134a394259be6fa9b5cc31137f6086 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 3 Oct 2024 10:48:15 -1000 Subject: [PATCH 079/231] fix(parser-variables): add missing brower check variables for bbcode parser --- parsing_extra.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parsing_extra.php b/parsing_extra.php index 5f9a5e3a..aa17088a 100644 --- a/parsing_extra.php +++ b/parsing_extra.php @@ -11,6 +11,9 @@ function setReasonableValues() $context['browser']['is_konqueror'] = false; $context['browser']['is_opera'] = false; $context['browser']['is_ie'] = false; + $context['browser']['is_ie4'] = false; + $context['browser']['is_ie5'] = false; + $context['browser']['is_ie5.5'] = false; $txt['lang_character_set'] = 'ISO-8859-1'; $txt['smf238'] = 'Code'; From 974bcec44e7575270d0cfb4c6d9fc6d0c87c10ff Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 3 Oct 2024 10:53:48 -1000 Subject: [PATCH 080/231] refactor(polls): update variable name --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 7c46b8ca..c257ce66 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -87,7 +87,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Poll for thread not found"} - thread -> + poll -> from(t in "smf_topics", where: t.id_topic == ^thread_id ) @@ -104,7 +104,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do {:error, "Poll for thread not found"} answers -> - if thread.id > 0, do: Map.put(thread, :answers, answers), else: nil + if poll.id > 0, do: Map.put(poll, :answers, answers), else: nil end end end From 2c98405b2687eb8af74cac3b76d1090cee14457c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 11:31:19 -0700 Subject: [PATCH 081/231] feat(helpers/proxy_conversion): build board counts from smf data select thread and post counts from smf_boards --- .../helpers/proxy_conversion.ex | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index c23f5a28..2d097067 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -49,6 +49,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do def build_model(model_type) do case model_type do + "boards.counts" -> + build_board_counts() + "threads.recent" -> build_recent_threads() @@ -121,6 +124,24 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_board_counts() do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", + where: b.id_board not in ^id_board_blacklist, + select: %{ + id: b.id_board, + thread_count: b.numTopics, + post_count: b.numPosts + } + ) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Boards not found"} + + boards -> return_tuple(boards) + end + end def build_boards(ids) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", From 7fcbce7ea14d01b0244e011a81eb3a1181d10605 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 11:32:31 -0700 Subject: [PATCH 082/231] feat(controllers/board): implement proxy_by_category and use board_counts --- lib/epochtalk_server_web/controllers/board.ex | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 2aff6cf9..5a8490cd 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -11,8 +11,9 @@ defmodule EpochtalkServerWeb.Controllers.Board do alias EpochtalkServerWeb.ErrorHelpers alias EpochtalkServerWeb.Helpers.Validate alias EpochtalkServerWeb.Helpers.ACL + alias EpochtalkServerWeb.Helpers.ProxyConversion - plug :check_proxy when action in [:slug_to_id] + plug :check_proxy when action in [:slug_to_id, :by_category] @doc """ Used to retrieve categorized boards @@ -130,10 +131,35 @@ defmodule EpochtalkServerWeb.Controllers.Board do conn end + :by_category -> + conn + |> proxy_by_category(conn.params) + |> halt() + _ -> conn end conn end + + defp proxy_by_category(conn, attrs) do + with :ok <- ACL.allow!(conn, "boards.allCategories"), + stripped <- Validate.cast(attrs, "stripped", :boolean, default: false), + user_priority <- ACL.get_user_priority(conn), + board_mapping <- BoardMapping.all(stripped: stripped), + board_moderators <- BoardModerator.all(), + {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), + categories <- Category.all() do + render(conn, :proxy_by_category, %{ + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: board_counts + }) + else + _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch boards") + end + end end From d19ac53eba594adb31cc62661bf2e34654c897fd Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 11:34:12 -0700 Subject: [PATCH 083/231] feat(json/board): stitch in board thread/post counts on render in proxy_by_category --- lib/epochtalk_server_web/json/board_json.ex | 51 +++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 9f626a1f..abef6600 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -8,6 +8,33 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do """ def slug_to_id(%{id: id}), do: %{id: id} + @doc """ + Renders proxy version of `Board` data by `Category`. + """ + def proxy_by_category(%{ + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: board_counts + }) do + board_counts = + board_counts + |> Enum.reduce(%{}, fn b, acc -> + acc + |> Map.put(b.id, %{post_count: b.post_count, thread_count: b.thread_count}) + end) + data = by_category(%{ + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: board_counts + }) + + data + end + @doc """ Renders `Board` data by `Category`. """ @@ -15,7 +42,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, - user_priority: user_priority + user_priority: user_priority, + # board counts for proxy version + board_counts: board_counts }) do # append board moderators to each board in board mapping board_mapping = @@ -45,7 +74,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_mapping, :boards, category, - user_priority + user_priority, + # board counts for proxy version + board_counts ) acc ++ [category] @@ -134,7 +165,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_mapping, child_key, parent, - user_priority + user_priority, + # board counts for proxy version + board_counts \\ nil ) do # get id of parent object could be board or category parent_id = if is_integer(Map.get(parent, :board_id)), do: parent.board_id, else: parent.id @@ -161,6 +194,15 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do |> Map.merge(remove_nil(board.stats)) |> Map.merge(board.thread) + # add board counts for proxy version + board = if board_counts != nil do + board + |> Map.put(:post_count, board_counts[board.id][:post_count]) + |> Map.put(:thread_count, board_counts[board.id][:thread_count]) + else + board + end + # delete unneeded properties board = board @@ -190,7 +232,8 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_mapping, :children, board, - user_priority + user_priority, + board_counts ) acc ++ [board] From 6c8e768a227a62d8536850ab26947bb699945cc3 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 13:04:31 -0700 Subject: [PATCH 084/231] style(json/thread): lastest -> latest --- lib/epochtalk_server_web/json/thread_json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 3111d0e7..98bd76a8 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -48,7 +48,7 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do deleted: t.last_post_user_deleted } ), - lastest: + latest: if(t.new_post_id || t.new_post_position, do: %{id: t.new_post_id, position: t.new_post_position || 1} ) From 6698019a44869372c73a38238e6a08b84ead1344 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 13:54:59 -0700 Subject: [PATCH 085/231] feat(helpers/proxy_conversion): build board_last_post_info from smf query --- .../helpers/proxy_conversion.ex | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 2d097067..0c99d8d6 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -52,6 +52,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "boards.counts" -> build_board_counts() + "boards.last_post_info" -> + build_board_last_post_info() + "threads.recent" -> build_recent_threads() @@ -142,6 +145,34 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do boards -> return_tuple(boards) end end + + def build_board_last_post_info() do + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", + where: b.id_board not in ^id_board_blacklist + ) + |> join(:left, [b], m in "smf_messages", on: b.id_last_msg == m.id_msg) + |> join(:left, [b, m], t in "smf_topics", on: m.id_topic == t.id_topic) + |> select([b, m, t], %{ + id: b.id_board, + last_post_created_at: m.posterTime * 1000, + last_post_position: t.numReplies, + last_post_username: m.posterName, + last_thread_created_at: t.id_member_started, + last_thread_id: t.id_topic, + last_thread_post_count: t.numReplies, + last_thread_slug: t.id_topic, + last_thread_title: m.subject, + last_thread_updated_at: m.posterTime * 1000 + }) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Boards not found"} + + boards -> return_tuple(boards) + end + end def build_boards(ids) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", From d59639c4f4c24d612fa3a43079a891ce45634be5 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 14:06:14 -0700 Subject: [PATCH 086/231] feat(controllers/board): load board_last_post_info --- lib/epochtalk_server_web/controllers/board.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 5a8490cd..7aa3c4ef 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -150,13 +150,15 @@ defmodule EpochtalkServerWeb.Controllers.Board do board_mapping <- BoardMapping.all(stripped: stripped), board_moderators <- BoardModerator.all(), {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), + {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), categories <- Category.all() do render(conn, :proxy_by_category, %{ categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, user_priority: user_priority, - board_counts: board_counts + board_counts: board_counts, + board_last_post_info: board_last_post_info }) else _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch boards") From 8cb3eda3f016c6d772a67faf418e5097c259202d Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 14:07:39 -0700 Subject: [PATCH 087/231] refactor(json/board): extract functionality to map query return to model id --- lib/epochtalk_server_web/json/board_json.ex | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index abef6600..bbf37791 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -18,12 +18,7 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do user_priority: user_priority, board_counts: board_counts }) do - board_counts = - board_counts - |> Enum.reduce(%{}, fn b, acc -> - acc - |> Map.put(b.id, %{post_count: b.post_count, thread_count: b.thread_count}) - end) + board_counts = map_to_id(board_counts) data = by_category(%{ categories: categories, board_moderators: board_moderators, @@ -34,6 +29,7 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do data end + defp map_to_id(data), do: Enum.reduce(data, %{}, &(&2 |> Map.put(&1.id, Map.delete(&1, :id)))) @doc """ Renders `Board` data by `Category`. From 2cf1d78d88e988cd7d4df2edfe17e36531ddb885 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 14:14:09 -0700 Subject: [PATCH 088/231] refactor(json/board): use map merge instead of loading by key --- lib/epochtalk_server_web/json/board_json.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index bbf37791..c6582228 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -191,13 +191,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do |> Map.merge(board.thread) # add board counts for proxy version - board = if board_counts != nil do - board - |> Map.put(:post_count, board_counts[board.id][:post_count]) - |> Map.put(:thread_count, board_counts[board.id][:thread_count]) - else - board - end + board = if board_counts != nil, + do: Map.merge(board, board_counts[board.id]), + else: board # delete unneeded properties board = From 7c2feb16cd16ec17a350d00603d8508529bced1a Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 4 Oct 2024 14:23:23 -0700 Subject: [PATCH 089/231] feat(json/board): append smf proxy last post info to boards --- lib/epochtalk_server_web/json/board_json.ex | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index c6582228..6ffb2e75 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -16,15 +16,18 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_moderators: board_moderators, board_mapping: board_mapping, user_priority: user_priority, - board_counts: board_counts + board_counts: board_counts, + board_last_post_info: board_last_post_info }) do board_counts = map_to_id(board_counts) + board_last_post_info = map_to_id(board_last_post_info) data = by_category(%{ categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, user_priority: user_priority, - board_counts: board_counts + board_counts: board_counts, + board_last_post_info: board_last_post_info }) data @@ -39,8 +42,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_moderators: board_moderators, board_mapping: board_mapping, user_priority: user_priority, - # board counts for proxy version - board_counts: board_counts + # board counts and last post info for proxy version + board_counts: board_counts, + board_last_post_info: board_last_post_info }) do # append board moderators to each board in board mapping board_mapping = @@ -71,8 +75,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do :boards, category, user_priority, - # board counts for proxy version - board_counts + # board counts and last post info for proxy version + board_counts, + board_last_post_info ) acc ++ [category] @@ -162,8 +167,9 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do child_key, parent, user_priority, - # board counts for proxy version - board_counts \\ nil + # board counts and last post info for proxy version + board_counts \\ nil, + board_last_post_info \\ nil ) do # get id of parent object could be board or category parent_id = if is_integer(Map.get(parent, :board_id)), do: parent.board_id, else: parent.id @@ -194,6 +200,10 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board = if board_counts != nil, do: Map.merge(board, board_counts[board.id]), else: board + # add board last post info for proxy version + board = if board_last_post_info != nil, + do: Map.merge(board, board_last_post_info[board.id]), + else: board # delete unneeded properties board = @@ -225,7 +235,8 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do :children, board, user_priority, - board_counts + board_counts, + board_last_post_info ) acc ++ [board] From 93e8fb703f95a696fa6b7881db859521f7e998a4 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 4 Oct 2024 11:34:56 -1000 Subject: [PATCH 090/231] feat(signatures): implement signatures and merit and activity --- .../helpers/proxy_conversion.ex | 57 +++++++++---------- lib/epochtalk_server_web/json/post_json.ex | 38 +++++-------- 2 files changed, 41 insertions(+), 54 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ff73a25c..0cfcf42c 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -346,42 +346,39 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do on: u.id_member == a.id_member and a.attachmentType == 1 ) |> select([m, u, a], %{ - id: m.id_msg, - thread_id: m.id_topic, - board_id: m.id_board, - user_id: m.id_member, - title: m.subject, - body: m.body, - updated_at: m.modifiedTime, - username: m.posterName, - poster_time: m.posterTime, - poster_name: m.posterName, - modified_time: m.modifiedTime, - avatar: - fragment( - "if(? <>'',concat('/avatars/',?),ifnull(concat('/useravatars/',?),''))", - u.avatar, - u.avatar, - a.filename - ) - }) + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + title: m.subject, + body: m.body, + updated_at: m.modifiedTime, + username: m.posterName, + created_at: m.posterTime * 1000, + modified_time: m.modifiedTime, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ), + user: %{ + id: m.id_member, + username: m.posterName, + signature: u.signature, + activity: u.activity, + merit: u.merit, + title: u.usertitle + } + } + ) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do {:ok, [], _} -> {:error, "Posts not found for thread_id: #{id}"} {:ok, posts, data} -> - posts = - Enum.reduce(posts, [], fn post, acc -> - post = - post - |> Map.put(:created_at, post.poster_time * 1000) - |> Map.delete(:poster_time) - - [post | acc] - end) - - return_tuple(Enum.reverse(posts), data) + return_tuple(posts, data) end end diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index eb49f42e..34eecf03 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -377,30 +377,20 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(post.body || post.body_html, "'", "\'") - %Porcelain.Result{out: parsed_body, status: status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - IO.inspect "'#{body}'" - IO.inspect parsed_body - IO.inspect status + %Porcelain.Result{out: parsed_body, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - post - |> Map.put(:body_html, parsed_body) - |> Map.put(:user, %{ - id: post.user_id, - username: post.poster_name - # original_poster: post.original_poster, - # username: post.username, - # priority: if(is_nil(post.priority), do: post.default_priority, else: post.priority) - # deleted: post.user_deleted, - # signature: post.signature, - # post_count: post.post_count, - # highlight_color: post.highlight_color, - # role_name: post.role_name, - # stats: Map.get(post, :user_trust_stats), - # ignored: Map.get(post, :user_ignored), - # _ignored: Map.get(post, :user_ignored), - # activity: Map.get(post, :user_activity) - }) - |> Map.delete(:user_id) - |> Map.delete(:poster_name) + signature = if post.user.signature, + do: String.replace(post.user.signature, "'", "\'"), + else: nil + + parsed_signature = if signature do + %Porcelain.Result{out: parsed_sig, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"") + parsed_sig + else + nil + end + + user = post.user |> Map.put(:signature, parsed_signature) + post |> Map.put(:body_html, parsed_body) |> Map.put(:user, user) end end From d41750c849c77abacb203053edb824b51cd6a270 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 10 Oct 2024 14:14:25 -0700 Subject: [PATCH 091/231] fix(post-avatars): Fixed avatars not loading for last post on thread list view --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 0c80871f..1eaecc41 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -299,7 +299,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_post_user_deleted: false, last_post_avatar: fragment( - "if(? <>'',concat('/avatars/',?),ifnull(concat('/useravatars/',?),''))", + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", m.avatar, m.avatar, a.filename From f18e58d1a1cbfd01653f8d3e5a2e7d4e3b11ffcb Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 10 Oct 2024 16:42:48 -0700 Subject: [PATCH 092/231] refactor(breadcrumbs): use proxy to build breadcrumbs for threads --- lib/epochtalk_server/models/thread.ex | 23 --------------- .../controllers/breadcrumb.ex | 14 ++++++---- .../helpers/breadcrumbs.ex | 28 ++++++++++++++----- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/lib/epochtalk_server/models/thread.ex b/lib/epochtalk_server/models/thread.ex index 4c2d7ba0..fa3ce785 100644 --- a/lib/epochtalk_server/models/thread.ex +++ b/lib/epochtalk_server/models/thread.ex @@ -674,29 +674,6 @@ defmodule EpochtalkServer.Models.Thread do first_post != nil and first_post.user_id == user_id and first_post.moderated end - @doc """ - Used to obtain breadcrumb data for a specific `Thread` given it's `slug` - """ - @spec breadcrumb(slug :: String.t()) :: - {:ok, thread :: t()} | {:error, :thread_does_not_exist} - def breadcrumb(slug) when is_binary(slug) do - post_query = - from p in Post, - order_by: p.created_at, - limit: 1 - - thread_query = - from t in Thread, - where: t.slug == ^slug, - preload: [:board, posts: ^post_query] - - thread = Repo.one(thread_query) - - if thread, - do: {:ok, thread}, - else: {:error, :thread_does_not_exist} - end - @doc """ Used to get first `Post` data given a `Thread` id """ diff --git a/lib/epochtalk_server_web/controllers/breadcrumb.ex b/lib/epochtalk_server_web/controllers/breadcrumb.ex index 8ca0f077..f77a448b 100644 --- a/lib/epochtalk_server_web/controllers/breadcrumb.ex +++ b/lib/epochtalk_server_web/controllers/breadcrumb.ex @@ -12,7 +12,7 @@ defmodule EpochtalkServerWeb.Controllers.Breadcrumb do Used to return breadcrumbs to the frontend` """ def breadcrumbs(conn, attrs) do - with id <- parse_id(attrs, attrs["id"]), + with id <- parse_id(attrs), type <- Validate.cast(attrs, "type", :string, required: true), {:ok, breadcrumbs} <- Breadcrumbs.build_crumbs(id, type, []) do render(conn, :breadcrumbs, breadcrumbs: breadcrumbs) @@ -25,9 +25,13 @@ defmodule EpochtalkServerWeb.Controllers.Breadcrumb do end end - defp parse_id(attrs, id) when is_integer(id), - do: Validate.cast(attrs, "id", :integer, min: 1, required: true) + defp parse_id(attrs) do + case Integer.parse(attrs["id"]) do + {_, ""} -> + Validate.cast(attrs, "id", :integer, min: 1, required: true) - defp parse_id(attrs, id) when is_binary(id), - do: Validate.cast(attrs, "id", :string, required: true) + _ -> + Validate.cast(attrs, "id", :string, required: true) + end + end end diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index fa3b2f2c..6822e599 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -2,6 +2,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do @moduledoc """ Helper for creating breadcrumbs """ + alias EpochtalkServerWeb.Helpers.ProxyConversion alias EpochtalkServer.Models.Category alias EpochtalkServer.Models.Board alias EpochtalkServer.Models.Thread @@ -69,15 +70,28 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do end defp build_thread_crumbs(id, crumbs) do - case Thread.breadcrumb(id) do - {:ok, thread} -> - post = Enum.at(thread.posts, 0) + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() + + thread = + cond do + is_integer(id) && id < threads_seq -> + ProxyConversion.build_model("thread", [id], 1, 1) + + is_binary(id) -> + Thread.find(id) + true -> + {:error, :thread_does_not_exist} + end + + case thread do + {:ok, thread} -> crumbs = [ %{ - label: post.content["title"], + label: thread.title, routeName: "Posts", - opts: %{slug: id, locked: post.locked} + opts: %{slug: id, locked: thread.locked} } | crumbs ] @@ -106,8 +120,8 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do end "thread" -> - %{id: object.board.slug, type: "board"} - + {:ok, board} = Board.find_by_id(object.board_id) + %{id: board.slug, type: "board"} _ -> nil end From 23f0cc867894a6c4723184747614a97ce2db6ccc Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 10 Oct 2024 17:29:43 -0700 Subject: [PATCH 093/231] refactor(proxy-conversion): refactored some proxy_conversion functions to take a single id parameter instead of a list of ids --- .../helpers/proxy_conversion.ex | 92 +++++++------------ 1 file changed, 34 insertions(+), 58 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 0c80871f..768d03de 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -24,31 +24,24 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do build_posts_by_thread(id, page, per_page) _ -> - build_model(model_type, [id], nil, nil) + build_model(nil, nil, nil, nil) end end - def build_model(model_type, ids, _, _) do + def build_model(model_type, id) do case model_type do "category" -> - build_categories(ids) + build_category(id) "board" -> - build_boards(ids) + build_board(id) "thread" -> - build_threads(ids) + build_thread(id) "post" -> - build_posts(ids) - - _ -> - build_model(nil, nil, nil, nil) - end - end + build_post(id) - def build_model(model_type, id) do - case model_type do "poll.by_thread" -> build_poll(id) @@ -159,23 +152,22 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end - def build_categories(ids) do + def build_category(id) do from(c in "smf_categories", - limit: ^length(ids), - where: c.id_cat in ^ids, + where: c.id_cat == ^id, select: %{ id: c.id_cat, name: c.name, view_order: c.catOrder } ) - |> SmfRepo.all() + |> SmfRepo.one() |> case do [] -> - {:error, "Categories not found for ids: #{ids}"} + {:error, "Category not found for id: #{id}"} - categories -> - return_tuple(categories) + category -> + category end end @@ -225,11 +217,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do boards -> return_tuple(boards) end end - def build_boards(ids) do + def build_board(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", - limit: ^length(ids), - where: b.id_board in ^ids and b.id_board not in ^id_board_blacklist, + where: b.id_board == ^id and b.id_board not in ^id_board_blacklist, select: %{ id: b.id_board, slug: b.id_board, @@ -240,22 +231,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do post_count: b.numPosts } ) - |> SmfRepo.all() + |> SmfRepo.one() |> case do [] -> - {:error, "Boards not found for ids: #{ids}"} + {:error, "Board not found for id: #{id}"} - boards -> - boards = - Enum.map(boards, fn board -> - board = - board - |> Map.put(:children, []) - - board - end) - - return_tuple(boards) + board -> + board end end @@ -310,11 +292,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> ProxyPagination.page_simple(count_query, page, per_page: per_page) end - def build_threads(ids) do + def build_thread(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(t in "smf_topics", - limit: ^length(ids), - where: t.id_topic in ^ids and t.id_board not in ^id_board_blacklist + where: t.id_topic == ^id and t.id_board not in ^id_board_blacklist ) # get first and last message for thread |> join(:left, [t], m in "smf_messages", on: t.id_first_msg == m.id_msg) @@ -344,15 +325,20 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_post_user_deleted: false, last_viewed: nil }) - |> SmfRepo.all() - |> return_tuple() + |> SmfRepo.one() + |> case do + [] -> + {:error, "Thread not found for id: #{id}"} + + thread -> + thread + end end - def build_posts(ids) do + def build_post(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(m in "smf_messages", - limit: ^length(ids), - where: m.id_msg in ^ids and m.id_board not in ^id_board_blacklist, + where: m.id_msg == ^id and m.id_board not in ^id_board_blacklist, select: %{ id: m.id_msg, thread_id: m.id_topic, @@ -363,23 +349,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do updated_at: m.modifiedTime } ) - |> SmfRepo.all() + |> SmfRepo.one() |> case do [] -> - {:error, "Posts not found for ids: #{ids}"} - - posts -> - posts = - Enum.reduce(posts, [], fn post, acc -> - post = - post - |> Map.put(:created_at, post.poster_time * 1000) - |> Map.delete(:poster_time) - - [post | acc] - end) + {:error, "Post not found for id: #{id}"} - return_tuple(posts) + post -> + post end end From 13ee89900aaa663fb9635a7954d3b51f5f4e47ca Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 10 Oct 2024 17:36:10 -0700 Subject: [PATCH 094/231] refactor(child-boards): refactored proxy_by_board() and proxy_by_thread() to no longer pull boards and thread data from proxy --- lib/epochtalk_server_web/controllers/post.ex | 12 +++++- .../controllers/thread.ex | 7 ++++ lib/epochtalk_server_web/json/post_json.ex | 40 +++++++++---------- lib/epochtalk_server_web/json/thread_json.ex | 18 +++++---- 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 60923609..e52c16c0 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -482,14 +482,22 @@ defmodule EpochtalkServerWeb.Controllers.Post do page <- Validate.cast(attrs, "page", :integer, default: 1), limit <- Validate.cast(attrs, "limit", :integer, default: 5), user <- Guardian.Plug.current_resource(conn), + user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "posts.byThread"), + board_mapping <- BoardMapping.all(), + board_moderators <- BoardModerator.all(), + thread <- ProxyConversion.build_model("thread", thread_id), + poll <- ProxyConversion.build_model("poll.by_thread", thread_id), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_thread", thread_id, page, limit), - poll <- ProxyConversion.build_model("poll.by_thread", thread_id) do + ProxyConversion.build_model("posts.by_thread", thread_id, page, limit) do render(conn, :by_thread_proxy, %{ posts: posts, poll: poll, + thread: thread, user: user, + user_priority: user_priority, + board_mapping: board_mapping, + board_moderators: board_moderators, page: page, limit: limit, pagination_data: data diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 30879ccf..34d975e5 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -756,12 +756,19 @@ defmodule EpochtalkServerWeb.Controllers.Thread do page <- Validate.cast(attrs, "page", :integer, default: 1), limit <- Validate.cast(attrs, "limit", :integer, default: 5), user <- Guardian.Plug.current_resource(conn), + user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "threads.byBoard"), + board_mapping <- BoardMapping.all(), + board_moderators <- BoardModerator.all(), {:ok, threads, data} <- ProxyConversion.build_model("threads.by_board", board_id, page, limit) do render(conn, :by_board_proxy, %{ threads: threads, user: user, + user_priority: user_priority, + board_id: board_id, + board_mapping: board_mapping, + board_moderators: board_moderators, page: page, limit: limit, pagination_data: data diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 34eecf03..7532f9c1 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -2,7 +2,6 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do alias EpochtalkServerWeb.Controllers.BoardJSON alias EpochtalkServerWeb.Controllers.ThreadJSON alias EpochtalkServerWeb.Helpers.ACL - alias EpochtalkServerWeb.Helpers.ProxyConversion @moduledoc """ Renders and formats `Post` data, in JSON format for frontend @@ -116,25 +115,24 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do def by_thread_proxy(%{ posts: posts, - page: page, poll: poll, + thread: thread, + user_priority: user_priority, + board_mapping: board_mapping, + board_moderators: board_moderators, + page: page, limit: limit }) do - {:ok, thread} = - if is_map(posts) do - ProxyConversion.build_model("thread", [posts.thread_id], page, limit) - else - ProxyConversion.build_model("thread", [List.first(posts).thread_id], page, limit) - end - - thread = Map.put(thread, :poll, poll) - # format board data - {:ok, board} = ProxyConversion.build_model("board", [thread.board_id], 1, 1) - board = - board - |> Map.put(:moderators, []) + BoardJSON.format_board_data_for_find( + board_moderators, + board_mapping, + thread.board_id, + user_priority + ) + + thread = Map.put(thread, :poll, poll) # convert singular post to list posts = if is_map(posts), do: [posts], else: posts @@ -380,15 +378,15 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do %Porcelain.Result{out: parsed_body, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") signature = if post.user.signature, - do: String.replace(post.user.signature, "'", "\'"), - else: nil + do: String.replace(post.user.signature, "'", "\'"), + else: nil parsed_signature = if signature do %Porcelain.Result{out: parsed_sig, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"") - parsed_sig - else - nil - end + parsed_sig + else + nil + end user = post.user |> Map.put(:signature, parsed_signature) post |> Map.put(:body_html, parsed_body) |> Map.put(:user, user) diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 98bd76a8..139e3dd3 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -1,6 +1,5 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do alias EpochtalkServerWeb.Controllers.BoardJSON - alias EpochtalkServerWeb.Helpers.ProxyConversion @moduledoc """ Renders and formats `Thread` data, in JSON format for frontend @@ -131,16 +130,21 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do def by_board_proxy(%{ threads: threads, user: user, + user_priority: user_priority, + board_id: board_id, + board_mapping: board_mapping, + board_moderators: board_moderators, page: page, limit: limit }) do # format board data - {:ok, board} = - if is_map(threads) do - ProxyConversion.build_model("board", [threads.board_id], 1, 1) - else - ProxyConversion.build_model("board", [List.first(threads).board_id], 1, 1) - end + board = + BoardJSON.format_board_data_for_find( + board_moderators, + board_mapping, + board_id, + user_priority + ) # format thread data user_id = if is_nil(user), do: nil, else: user.id From ebad48b84b21a2e31c6b55641e88e7ce3aa5dfaf Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 11 Oct 2024 14:54:06 -0700 Subject: [PATCH 095/231] fix(breadcrumbs): fixing errors with breadcrumbs after child-boards merge --- lib/epochtalk_server_web/helpers/breadcrumbs.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 6822e599..1a6af71d 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -76,7 +76,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do thread = cond do is_integer(id) && id < threads_seq -> - ProxyConversion.build_model("thread", [id], 1, 1) + ProxyConversion.build_model("thread", id) is_binary(id) -> Thread.find(id) @@ -86,7 +86,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do end case thread do - {:ok, thread} -> + thread -> crumbs = [ %{ label: thread.title, From a7524a033615bec0bf555e249f37aa084ed8891d Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 11 Oct 2024 15:00:15 -0700 Subject: [PATCH 096/231] fix(breadcrumbs): refactor thread breadcrumbs --- .../helpers/breadcrumbs.ex | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 1a6af71d..737f1324 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -82,26 +82,20 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do Thread.find(id) true -> - {:error, :thread_does_not_exist} + build_crumbs(nil, nil, crumbs) end - case thread do - thread -> - crumbs = [ - %{ - label: thread.title, - routeName: "Posts", - opts: %{slug: id, locked: thread.locked} - } - | crumbs - ] - - next_data = get_next_data(thread, "thread") - build_crumbs(next_data[:id], next_data[:type], crumbs) - - {:error, :thread_does_not_exist} -> - build_crumbs(nil, nil, crumbs) - end + crumbs = [ + %{ + label: thread.title, + routeName: "Posts", + opts: %{slug: id, locked: thread.locked} + } + | crumbs + ] + + next_data = get_next_data(thread, "thread") + build_crumbs(next_data[:id], next_data[:type], crumbs) end defp get_next_data(object, type) do From 7241ee41a003caa3c119e6a29b6c07a93e7e5d75 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 16 Oct 2024 17:22:58 -0700 Subject: [PATCH 097/231] ci(github/workflows/main): remove test and release tasks these are not needed for this proxy version of the project.. yet --- .github/workflows/main.yml | 202 ++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 906b1167..74abbb57 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,104 +102,104 @@ jobs: priv/plts - name: Run dialyzer run: mix dialyzer --format github - test: - needs: dependencies - name: Run tests - runs-on: ubuntu-latest - services: - postgres: - image: postgres:14.2 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: epochtalk_server_test - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - redis: - ports: - - 6379:6379 - image: redis:7.0.4 - # Set health checks to wait until redis has started - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - name: Checkout - uses: actions/checkout@v3.6.0 - with: - ref: ${{ github.event.client_payload.branch }} - - name: Parse .tool-versions - id: tool-versions - uses: paulo-ferraz-oliveira/parse-tool-versions@v1 - - name: Sets up an Erlang/OTP environment - uses: erlef/setup-beam@v1 - with: - version-file: .tool-versions - version-type: strict - - name: Retrieve cached dependencies - uses: actions/cache@v3.3.1 - id: mix-cache - with: - path: | - deps - _build - key: ${{ runner.os }}-${{ steps.tool-versions.outputs.erlang }}-${{ steps.tool-versions.outputs.elixir }}-${{ hashFiles('mix.lock') }} - - name: Run tests - run: mix test - env: - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - release: - needs: [test, static_code_analysis] - name: Release - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3.6.0 - - name: Semantic Release - uses: cycjimmy/semantic-release-action@v4.0.0 - with: - branches: | - [ - 'main', - { - name: 'prerelease', - prerelease: true - }, - ] - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - elixir_docs: - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/elixir-docs' }} - name: Generate project documentation - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3.6.0 - - name: Sets up an Erlang/OTP environment - uses: erlef/setup-beam@v1 - with: - version-file: .tool-versions - version-type: strict - - name: Build docs - uses: lee-dohm/generate-elixir-docs@v1.0.1 - - name: Publish to Pages - uses: peaceiris/actions-gh-pages@v3.9.3 - with: - deploy_key: ${{ secrets.DOCS_DEPLOY_KEY }} - external_repository: epochtalk/server.epochtalk.github.io - publish_dir: ./doc - publish_branch: gh-pages + # test: + # needs: dependencies + # name: Run tests + # runs-on: ubuntu-latest + # services: + # postgres: + # image: postgres:14.2 + # env: + # POSTGRES_USER: postgres + # POSTGRES_PASSWORD: postgres + # POSTGRES_DB: epochtalk_server_test + # ports: + # - 5432:5432 + # # Set health checks to wait until postgres has started + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # redis: + # ports: + # - 6379:6379 + # image: redis:7.0.4 + # # Set health checks to wait until redis has started + # options: >- + # --health-cmd "redis-cli ping" + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Checkout + # uses: actions/checkout@v3.6.0 + # with: + # ref: ${{ github.event.client_payload.branch }} + # - name: Parse .tool-versions + # id: tool-versions + # uses: paulo-ferraz-oliveira/parse-tool-versions@v1 + # - name: Sets up an Erlang/OTP environment + # uses: erlef/setup-beam@v1 + # with: + # version-file: .tool-versions + # version-type: strict + # - name: Retrieve cached dependencies + # uses: actions/cache@v3.3.1 + # id: mix-cache + # with: + # path: | + # deps + # _build + # key: ${{ runner.os }}-${{ steps.tool-versions.outputs.erlang }}-${{ steps.tool-versions.outputs.elixir }}-${{ hashFiles('mix.lock') }} + # - name: Run tests + # run: mix test + # env: + # AWS_REGION: ${{ secrets.AWS_REGION }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # release: + # needs: [test, static_code_analysis] + # name: Release + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v3.6.0 + # - name: Semantic Release + # uses: cycjimmy/semantic-release-action@v4.0.0 + # with: + # branches: | + # [ + # 'main', + # { + # name: 'prerelease', + # prerelease: true + # }, + # ] + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # elixir_docs: + # if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/elixir-docs' }} + # name: Generate project documentation + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v3.6.0 + # - name: Sets up an Erlang/OTP environment + # uses: erlef/setup-beam@v1 + # with: + # version-file: .tool-versions + # version-type: strict + # - name: Build docs + # uses: lee-dohm/generate-elixir-docs@v1.0.1 + # - name: Publish to Pages + # uses: peaceiris/actions-gh-pages@v3.9.3 + # with: + # deploy_key: ${{ secrets.DOCS_DEPLOY_KEY }} + # external_repository: epochtalk/server.epochtalk.github.io + # publish_dir: ./doc + # publish_branch: gh-pages From 7b73724ff9e336733800e9261f3e93f8759b5197 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 16 Oct 2024 20:05:17 -0700 Subject: [PATCH 098/231] style(): mix format --- config/runtime.exs | 2 +- lib/epochtalk_server/smf_loader.ex | 110 +++++++++++------- .../helpers/breadcrumbs.ex | 1 + .../helpers/proxy_conversion.ex | 99 ++++++++++------ lib/epochtalk_server_web/json/board_json.ex | 34 +++--- lib/epochtalk_server_web/json/post_json.ex | 15 ++- 6 files changed, 160 insertions(+), 101 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 665dc1c6..3ae66c39 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -460,7 +460,7 @@ if config_env() == :prod do id_board_blacklist = id_board_blacklist |> String.split() - |> Enum.map(&(String.to_integer(&1))) + |> Enum.map(&String.to_integer(&1)) # Configure proxy config :epochtalk_server, diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 9c0655e3..280e3986 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -3,8 +3,9 @@ defmodule EpochtalkServer.SMFLoader do # converts smf_boards tsv file to epochtalk boards tsv file def convert_smf_boards_tsv_file(path) do - {boards, board_mappings} = load_from_tsv_file(path) - |> map_boards_stream() + {boards, board_mappings} = + load_from_tsv_file(path) + |> map_boards_stream() # write boards to file for import boards @@ -14,6 +15,7 @@ defmodule EpochtalkServer.SMFLoader do # return board mappings for later insertion board_mappings end + def load_categories_mappings_from_tsv_file(path) do # load categories (in postgres format) # and return category mappings for later insertion @@ -28,42 +30,46 @@ defmodule EpochtalkServer.SMFLoader do } end) end + def insert_board_mappings(board_mappings) do # board mappings board_mappings |> BoardMapping.update() end + # loads smf data from a tsv file def load_from_tsv_file(path) do with true <- if(File.exists?(path), do: true, else: "ファイルがない"), - {:ok, file} <- File.read(path), - file_lines <- file |> String.trim() |> String.split("\n"), - [header_line | file_lines] <- file_lines, - headers <- - # clean line - header_line - |> String.trim() - |> String.split("\t"), - smf_boards <- - file_lines - |> Enum.map(&( - &1 - # clean line - |> String.trim() - |> String.split("\t") - )) - |> Enum.map(fn line -> - # map each line to headers - Enum.zip(headers, line) - |> Enum.into(%{}) - end) do - smf_boards + {:ok, file} <- File.read(path), + file_lines <- file |> String.trim() |> String.split("\n"), + [header_line | file_lines] <- file_lines, + # clean line + headers <- + header_line + |> String.trim() + |> String.split("\t"), + smf_boards <- + file_lines + |> Enum.map( + &(&1 + # clean line + |> String.trim() + |> String.split("\t")) + ) + |> Enum.map(fn line -> + # map each line to headers + Enum.zip(headers, line) + |> Enum.into(%{}) + end) do + smf_boards else problem -> IO.puts("問題がある： #{inspect(problem)}") end end + def map_boards_stream(boards_stream) do now = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + {board_mapping_pairs, _slug_duplicate_index} = boards_stream |> Enum.map_reduce(%{}, fn smf_board, slugs -> @@ -72,19 +78,24 @@ defmodule EpochtalkServer.SMFLoader do |> HtmlEntities.decode() |> String.replace(~r{[ /]}, "-") |> String.slice(0..99) + # handle duplicate slugs slug_duplicate_index = Map.get(slugs, slug) + {slugs, slug} = if slug_duplicate_index == nil do # keep track of used slugs, return original slug {Map.put(slugs, slug, 0), slug} else # replace last characters with index - new_slug = slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) + new_slug = + slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) + new_slug = new_slug <> to_string(slug_duplicate_index) # increment used slugs, return new slug {Map.put(slugs, slug, slug_duplicate_index + 1), new_slug} end + # build board board = %{} @@ -98,7 +109,10 @@ defmodule EpochtalkServer.SMFLoader do |> Map.put(:created_at, now) |> Map.put(:imported_at, now) |> Map.put(:updated_at, now) - |> Map.put(:meta, "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"") + |> Map.put( + :meta, + "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"" + ) |> Map.put(:right_to_left, "f") |> Map.put(:slug, slug) @@ -114,6 +128,7 @@ defmodule EpochtalkServer.SMFLoader do category_id: smf_board["ID_CAT"] |> String.to_integer(), view_order: smf_board["boardOrder"] |> String.to_integer() } + _ -> %{ type: "board", @@ -123,10 +138,13 @@ defmodule EpochtalkServer.SMFLoader do view_order: smf_board["boardOrder"] |> String.to_integer() } end + {{board, board_mapping}, slugs} end) + Enum.unzip(board_mapping_pairs) end + def tabulate_boards_map(boards_map) do data = boards_map @@ -149,27 +167,31 @@ defmodule EpochtalkServer.SMFLoader do |> Enum.join("\t") end) - header = [ - "id", - "name", - "description", - "post_count", - "thread_count", - "viewable_by", - "postable_by", - "created_at", - "imported_at", - "updated_at", - "meta", - "right_to_left", - "slug" - ] |> Enum.join("\t") - [ header | data ] + header = + [ + "id", + "name", + "description", + "post_count", + "thread_count", + "viewable_by", + "postable_by", + "created_at", + "imported_at", + "updated_at", + "meta", + "right_to_left", + "slug" + ] + |> Enum.join("\t") + + [header | data] end + def write_to_tsv_file(data, path) do with false <- if(File.exists?(path), do: "ファイルがもうある", else: false), - file <- File.stream!(path) do - data |> Enum.into(file, fn line -> line <> "\n" end) + file <- File.stream!(path) do + data |> Enum.into(file, fn line -> line <> "\n" end) else problem -> IO.puts("問題がある： #{inspect(problem)}") end diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 737f1324..4e9e84ad 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -116,6 +116,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do "thread" -> {:ok, board} = Board.find_by_id(object.board_id) %{id: board.slug, type: "board"} + _ -> nil end diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index e55d4d23..ae4e291b 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -77,7 +77,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do display_mode: "always", expiration: p.expireTime * 1000, has_voted: false, - locked: (p.votingLocked == 1), + locked: p.votingLocked == 1, max_answers: p.maxVotes, question: p.question }) @@ -109,7 +109,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_recent_threads() do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(t in "smf_topics", limit: 5, where: t.id_board not in ^id_board_blacklist, @@ -148,7 +150,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Recent threads not found"} - threads -> threads + threads -> + threads end end @@ -172,7 +175,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_board_counts() do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist, select: %{ @@ -186,12 +191,15 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Boards not found"} - boards -> return_tuple(boards) + boards -> + return_tuple(boards) end end def build_board_last_post_info() do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist ) @@ -214,11 +222,15 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Boards not found"} - boards -> return_tuple(boards) + boards -> + return_tuple(boards) end end + def build_board(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", where: b.id_board == ^id and b.id_board not in ^id_board_blacklist, select: %{ @@ -242,9 +254,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_threads_by_board(id, page, per_page) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + count_query = - from t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, select: %{count: count(t.id_topic)} + from t in "smf_topics", + where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, + select: %{count: count(t.id_topic)} from(t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, @@ -293,7 +309,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_thread(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(t in "smf_topics", where: t.id_topic == ^id and t.id_board not in ^id_board_blacklist ) @@ -336,7 +354,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_post(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(m in "smf_messages", where: m.id_msg == ^id and m.id_board not in ^id_board_blacklist, select: %{ @@ -360,9 +380,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_posts_by_thread(id, page, per_page) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + count_query = - from m in "smf_messages", where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, select: %{count: count(m.id_topic)} + from m in "smf_messages", + where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, + select: %{count: count(m.id_topic)} from(m in "smf_messages", limit: ^per_page, @@ -374,32 +398,31 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do on: u.id_member == a.id_member and a.attachmentType == 1 ) |> select([m, u, a], %{ - id: m.id_msg, - thread_id: m.id_topic, - board_id: m.id_board, - title: m.subject, - body: m.body, - updated_at: m.modifiedTime, + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + title: m.subject, + body: m.body, + updated_at: m.modifiedTime, + username: m.posterName, + created_at: m.posterTime * 1000, + modified_time: m.modifiedTime, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ), + user: %{ + id: m.id_member, username: m.posterName, - created_at: m.posterTime * 1000, - modified_time: m.modifiedTime, - avatar: - fragment( - "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", - u.avatar, - u.avatar, - a.filename - ), - user: %{ - id: m.id_member, - username: m.posterName, - signature: u.signature, - activity: u.activity, - merit: u.merit, - title: u.usertitle - } + signature: u.signature, + activity: u.activity, + merit: u.merit, + title: u.usertitle } - ) + }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do {:ok, [], _} -> diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 6ffb2e75..2d1ae252 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -12,16 +12,18 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do Renders proxy version of `Board` data by `Category`. """ def proxy_by_category(%{ - categories: categories, - board_moderators: board_moderators, - board_mapping: board_mapping, - user_priority: user_priority, - board_counts: board_counts, - board_last_post_info: board_last_post_info - }) do + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: board_counts, + board_last_post_info: board_last_post_info + }) do board_counts = map_to_id(board_counts) board_last_post_info = map_to_id(board_last_post_info) - data = by_category(%{ + + data = + by_category(%{ categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, @@ -32,6 +34,7 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do data end + defp map_to_id(data), do: Enum.reduce(data, %{}, &(&2 |> Map.put(&1.id, Map.delete(&1, :id)))) @doc """ @@ -197,13 +200,16 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do |> Map.merge(board.thread) # add board counts for proxy version - board = if board_counts != nil, - do: Map.merge(board, board_counts[board.id]), - else: board + board = + if board_counts != nil, + do: Map.merge(board, board_counts[board.id]), + else: board + # add board last post info for proxy version - board = if board_last_post_info != nil, - do: Map.merge(board, board_last_post_info[board.id]), - else: board + board = + if board_last_post_info != nil, + do: Map.merge(board, board_last_post_info[board.id]), + else: board # delete unneeded properties board = diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 7532f9c1..80216c4d 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -375,14 +375,21 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(post.body || post.body_html, "'", "\'") - %Porcelain.Result{out: parsed_body, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + %Porcelain.Result{out: parsed_body, status: _status} = + Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - signature = if post.user.signature, + signature = + if post.user.signature, do: String.replace(post.user.signature, "'", "\'"), else: nil - parsed_signature = if signature do - %Porcelain.Result{out: parsed_sig, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"") + parsed_signature = + if signature do + %Porcelain.Result{out: parsed_sig, status: _status} = + Porcelain.shell( + "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"" + ) + parsed_sig else nil From 1b23033a0bfaaf5a7a200707b0b9014473e15443 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 16 Oct 2024 20:09:37 -0700 Subject: [PATCH 099/231] refactor(smf_loader): add module doc --- lib/epochtalk_server/smf_loader.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 280e3986..d99bcbb1 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -1,4 +1,8 @@ defmodule EpochtalkServer.SMFLoader do + @moduledoc """ + Load data from smf mysql output, format for import + and create/insert mappings for categories and boards + """ alias EpochtalkServer.Models.BoardMapping # converts smf_boards tsv file to epochtalk boards tsv file From 6f6c3c2987b4db6b39ffbccc42a215a5442925a7 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 16 Oct 2024 20:32:08 -0700 Subject: [PATCH 100/231] refactor(controllers/thread): split check_proxy, match on action previously didn't pass credo checks for cyclomatic complexity separating functionality lowers complexity below acceptable threshold --- .../controllers/thread.ex | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 34d975e5..319faff0 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -702,54 +702,56 @@ defmodule EpochtalkServerWeb.Controllers.Thread do defp update_user_thread_view_count(user, thread_id), do: UserThreadView.upsert(user.id, thread_id) - defp check_proxy(conn, _) do - case conn.private.phoenix_action do - :by_board -> - %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) - boards_seq = boards_seq |> String.to_integer() + # check proxy for :by_board action + defp check_proxy(%{private: %{phoenix_action: :by_board}} = conn, _) do + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() - if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do + if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do + conn + |> proxy_by_board(conn.params) + |> halt() + else + conn + end + end + # check proxy for :slug_to_id action + defp check_proxy(%{private: %{phoenix_action: :slug_to_id}} = conn, _) do + case Integer.parse(conn.params["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) + + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() + + if slug_as_id < threads_seq do conn - |> proxy_by_board(conn.params) + |> render(:slug_to_id, id: slug_as_id) |> halt() else conn end - :slug_to_id -> - case Integer.parse(conn.params["slug"]) do - {_, ""} -> - slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - - %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) - threads_seq = threads_seq |> String.to_integer() - - if slug_as_id < threads_seq do - conn - |> render(:slug_to_id, id: slug_as_id) - |> halt() - else - conn - end - - _ -> - conn - end - - :viewed -> - conn - |> send_resp(200, []) - |> halt() - - :recent -> - conn - |> proxy_recent(conn.params) - |> halt() - _ -> conn end end + # check proxy for :viewed action + defp check_proxy(%{private: %{phoenix_action: :viewed}} = conn, _) do + conn + |> send_resp(200, []) + |> halt() + end + # check proxy for :recent action + defp check_proxy(%{private: %{phoenix_action: :recent}} = conn, _) do + conn + |> proxy_recent(conn.params) + |> halt() + end + # check proxy default + defp check_proxy(%{private: %{phoenix_action: _}} = conn, _) do + conn + end defp proxy_by_board(conn, attrs) do with board_id <- Validate.cast(attrs, "board_id", :integer, required: true), From d2e7ce732a3278195f27a6dea7aa8f06b9342732 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Wed, 16 Oct 2024 20:37:39 -0700 Subject: [PATCH 101/231] style(controllers/thread): mix format --- lib/epochtalk_server_web/controllers/thread.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 319faff0..9ae9a20b 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -715,6 +715,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do conn end end + # check proxy for :slug_to_id action defp check_proxy(%{private: %{phoenix_action: :slug_to_id}} = conn, _) do case Integer.parse(conn.params["slug"]) do @@ -736,18 +737,21 @@ defmodule EpochtalkServerWeb.Controllers.Thread do conn end end + # check proxy for :viewed action defp check_proxy(%{private: %{phoenix_action: :viewed}} = conn, _) do conn |> send_resp(200, []) |> halt() end + # check proxy for :recent action defp check_proxy(%{private: %{phoenix_action: :recent}} = conn, _) do conn |> proxy_recent(conn.params) |> halt() end + # check proxy default defp check_proxy(%{private: %{phoenix_action: _}} = conn, _) do conn From 1bce199db19aa0da55103d713e988de4cefa9213 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 17 Oct 2024 08:48:49 -0700 Subject: [PATCH 102/231] refactor(config): re-consolidate proxy configs conditionally show debug logs in production move proxy configs out of emailer config block move functionality for proxy_config from config/config.exs to config/runtime.exs section off proxy_config for prod, dev, default(test) provide more solid default configs for dev/test and fail conditions for unconfigured values in prod don't configure proxy repo in :test env --- config/config.exs | 8 ---- config/runtime.exs | 100 +++++++++++++++++++++++++++++++-------------- 2 files changed, 70 insertions(+), 38 deletions(-) diff --git a/config/config.exs b/config/config.exs index 5e196796..244623a5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,14 +10,6 @@ import Config # Set ecto repos config :epochtalk_server, ecto_repos: [EpochtalkServer.Repo] -# Configure proxy -config :epochtalk_server, - proxy_config: %{ - threads_seq: System.get_env("THREADS_SEQ") || "6000000", - boards_seq: System.get_env("BOARDS_SEQ") || "500", - id_board_blacklist: [] - } - # Configure Porcelain config :porcelain, driver: Porcelain.Driver.Basic diff --git a/config/runtime.exs b/config/runtime.exs index 3ae66c39..2f6d319c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -403,9 +403,12 @@ if config_env() == :prod do password: System.get_env("EMAILER_SMTP_PASSWORD", "password"), port: get_env_cast_integer_with_default.("EMAILER_SMTP_PORT", "465") end +end - ##### ALL CONFIGURATIONS BELOW ARE PROXY REPO CONFIGURATIONS ##### +##### PROXY REPO CONFIGURATIONS ##### +# conditionally show debug logs in prod +if config_env() == :prod do logger_level = System.get_env("LOGGER_LEVEL") |> case do @@ -414,8 +417,47 @@ if config_env() == :prod do end config :logger, level: logger_level +end - # Configure SmfRepo for proxy +# Configure proxy sequences and boards blacklist +proxy_config = case config_env() do + :prod -> + id_board_blacklist = + System.get_env("ID_BOARD_BLACKLIST") || + raise """ + environment variable ID_BOARD_BLACKLIST is missing. + """ + + id_board_blacklist = + id_board_blacklist + |> String.split() + |> Enum.map(&String.to_integer(&1)) + + %{ + threads_seq: System.get_env("THREADS_SEQ") || "6000000", + boards_seq: System.get_env("BOARDS_SEQ") || "500", + id_board_blacklist: id_board_blacklist + } + :dev -> + %{ + threads_seq: "6000000", + boards_seq: "500", + id_board_blacklist: [] + } + # default thread/board seq and empty blacklist + _ -> + %{ + threads_seq: "0", + boards_seq: "0", + id_board_blacklist: [] + } +end + +config :epochtalk_server, proxy_config: proxy_config + +# Configure SmfRepo for proxy +# - do not configure for testing +if config_env() != :test do smf_repo_username = System.get_env("SMF_REPO_USERNAME") || raise """ @@ -440,33 +482,31 @@ if config_env() == :prod do environment variable SMF_REPO_DATABASE is missing. """ - config :epochtalk_server, EpochtalkServer.SmfRepo, - username: smf_repo_username, - password: smf_repo_password, - hostname: smf_repo_hostname, - database: smf_repo_database, - port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), - stacktrace: System.get_env("SMF_REPO_STACKTRACE") == "TRUE" || true, - show_sensitive_data_on_connection_error: - System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") == "TRUE" || false, - pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") - - id_board_blacklist = - System.get_env("ID_BOARD_BLACKLIST") || - raise """ - environment variable ID_BOARD_BLACKLIST is missing. - """ - - id_board_blacklist = - id_board_blacklist - |> String.split() - |> Enum.map(&String.to_integer(&1)) + proxy_repo_config = case config_env() do + :prod -> + [ + username: smf_repo_username, + password: smf_repo_password, + hostname: smf_repo_hostname, + database: smf_repo_database, + port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), + stacktrace: System.get_env("SMF_REPO_STACKTRACE") == "TRUE" || true, + show_sensitive_data_on_connection_error: + System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") == "TRUE" || false, + pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") + ] + _ -> + [ + username: smf_repo_username, + password: smf_repo_password, + hostname: smf_repo_hostname, + database: smf_repo_database, + port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), + stacktrace: true, + show_sensitive_data_on_connection_error: true, + pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") + ] + end - # Configure proxy - config :epochtalk_server, - proxy_config: %{ - threads_seq: System.get_env("THREADS_SEQ") || "6000000", - boards_seq: System.get_env("BOARDS_SEQ") || "500", - id_board_blacklist: id_board_blacklist - } + config :epochtalk_server, EpochtalkServer.SmfRepo, proxy_repo_config end From ec3eb15ddde170de774f5df53f9b965671497f2d Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 17 Oct 2024 08:50:23 -0700 Subject: [PATCH 103/231] refactor(application): remove smf repo from supervisor tree in :test env --- lib/epochtalk_server/application.ex | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 57799cb7..c88ba198 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -40,15 +40,19 @@ defmodule EpochtalkServer.Application do # {EpochtalkServer.Worker, arg} ] - # don't run config_warmer during tests + # adjust supervised processes for testing children = - if Application.get_env(:epochtalk_server, :env) == :test, - do: - List.delete( - children, - {Task, &EpochtalkServer.Models.Configuration.warm_frontend_config/0} - ), - else: children + if Application.get_env(:epochtalk_server, :env) == :test do + children + # don't run config warmer during tests + |> List.delete( + {Task, &EpochtalkServer.Models.Configuration.warm_frontend_config/0} + ) + # don't run SmfRepo during tests + |> List.delete(EpochtalkServer.SmfRepo) + else + children + end # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options From 0a6fe890a15f7b560935e4d068afc684ef7903ab Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 17 Oct 2024 08:51:44 -0700 Subject: [PATCH 104/231] style(): mix format --- config/runtime.exs | 119 +++++++++++++++------------- lib/epochtalk_server/application.ex | 4 +- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 2f6d319c..5ef53116 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -420,38 +420,41 @@ if config_env() == :prod do end # Configure proxy sequences and boards blacklist -proxy_config = case config_env() do - :prod -> - id_board_blacklist = - System.get_env("ID_BOARD_BLACKLIST") || - raise """ - environment variable ID_BOARD_BLACKLIST is missing. - """ - - id_board_blacklist = - id_board_blacklist - |> String.split() - |> Enum.map(&String.to_integer(&1)) - - %{ - threads_seq: System.get_env("THREADS_SEQ") || "6000000", - boards_seq: System.get_env("BOARDS_SEQ") || "500", - id_board_blacklist: id_board_blacklist - } - :dev -> - %{ - threads_seq: "6000000", - boards_seq: "500", - id_board_blacklist: [] - } - # default thread/board seq and empty blacklist - _ -> - %{ - threads_seq: "0", - boards_seq: "0", - id_board_blacklist: [] - } -end +proxy_config = + case config_env() do + :prod -> + id_board_blacklist = + System.get_env("ID_BOARD_BLACKLIST") || + raise """ + environment variable ID_BOARD_BLACKLIST is missing. + """ + + id_board_blacklist = + id_board_blacklist + |> String.split() + |> Enum.map(&String.to_integer(&1)) + + %{ + threads_seq: System.get_env("THREADS_SEQ") || "6000000", + boards_seq: System.get_env("BOARDS_SEQ") || "500", + id_board_blacklist: id_board_blacklist + } + + :dev -> + %{ + threads_seq: "6000000", + boards_seq: "500", + id_board_blacklist: [] + } + + # default thread/board seq and empty blacklist + _ -> + %{ + threads_seq: "0", + boards_seq: "0", + id_board_blacklist: [] + } + end config :epochtalk_server, proxy_config: proxy_config @@ -482,31 +485,33 @@ if config_env() != :test do environment variable SMF_REPO_DATABASE is missing. """ - proxy_repo_config = case config_env() do - :prod -> - [ - username: smf_repo_username, - password: smf_repo_password, - hostname: smf_repo_hostname, - database: smf_repo_database, - port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), - stacktrace: System.get_env("SMF_REPO_STACKTRACE") == "TRUE" || true, - show_sensitive_data_on_connection_error: - System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") == "TRUE" || false, - pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") - ] - _ -> - [ - username: smf_repo_username, - password: smf_repo_password, - hostname: smf_repo_hostname, - database: smf_repo_database, - port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), - stacktrace: true, - show_sensitive_data_on_connection_error: true, - pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") - ] - end + proxy_repo_config = + case config_env() do + :prod -> + [ + username: smf_repo_username, + password: smf_repo_password, + hostname: smf_repo_hostname, + database: smf_repo_database, + port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), + stacktrace: System.get_env("SMF_REPO_STACKTRACE") == "TRUE" || true, + show_sensitive_data_on_connection_error: + System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") == "TRUE" || false, + pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") + ] + + _ -> + [ + username: smf_repo_username, + password: smf_repo_password, + hostname: smf_repo_hostname, + database: smf_repo_database, + port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), + stacktrace: true, + show_sensitive_data_on_connection_error: true, + pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") + ] + end config :epochtalk_server, EpochtalkServer.SmfRepo, proxy_repo_config end diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index c88ba198..4535c59e 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -45,9 +45,7 @@ defmodule EpochtalkServer.Application do if Application.get_env(:epochtalk_server, :env) == :test do children # don't run config warmer during tests - |> List.delete( - {Task, &EpochtalkServer.Models.Configuration.warm_frontend_config/0} - ) + |> List.delete({Task, &EpochtalkServer.Models.Configuration.warm_frontend_config/0}) # don't run SmfRepo during tests |> List.delete(EpochtalkServer.SmfRepo) else From 9392713a9263c731e6e2a18eaf2ed5a59a7f58aa Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 18 Oct 2024 12:53:55 -0700 Subject: [PATCH 105/231] refactor(config/dev): remove config for smf repo consolidated into config/runtime proxy :dev section --- config/dev.exs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 0fc4128b..1b40ac3a 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,16 +1,5 @@ import Config -config :epochtalk_server, EpochtalkServer.SmfRepo, - username: System.get_env("SMF_REPO_USERNAME"), - password: System.get_env("SMF_REPO_PASSWORD"), - hostname: System.get_env("SMF_REPO_HOSTNAME"), - database: System.get_env("SMF_REPO_DATABASE"), - port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), - stacktrace: System.get_env("SMF_REPO_STACKTRACE") || true, - show_sensitive_data_on_connection_error: - System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") || true, - pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") - # For Development `config/dev.exs` loads the "Local" adapter which allows preview # of sent emails at the url `/dev/mailbox`. To test SMTP in Development mode, # mailer configurations for adapter, credentials and other options can be From 8d82eb4ccb5da330b11776b3429a904fc50634f3 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 18 Oct 2024 13:05:56 -0700 Subject: [PATCH 106/231] refactor(config/runtime): use convenience methods for loading env variables --- config/runtime.exs | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 5ef53116..5e14d1ca 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -424,10 +424,10 @@ proxy_config = case config_env() do :prod -> id_board_blacklist = - System.get_env("ID_BOARD_BLACKLIST") || - raise """ - environment variable ID_BOARD_BLACKLIST is missing. - """ + get_env_or_raise_with_message.( + "ID_BOARD_BLACKLIST", + "environment variable ID_BOARD_BLACKLIST is missing." + ) id_board_blacklist = id_board_blacklist @@ -462,28 +462,28 @@ config :epochtalk_server, proxy_config: proxy_config # - do not configure for testing if config_env() != :test do smf_repo_username = - System.get_env("SMF_REPO_USERNAME") || - raise """ - environment variable SMF_REPO_USERNAME is missing. - """ + get_env_or_raise_with_message.( + "SMF_REPO_USERNAME", + "environment variable SMF_REPO_USERNAME is missing." + ) smf_repo_password = - System.get_env("SMF_REPO_PASSWORD") || - raise """ - environment variable SMF_REPO_PASSWORD is missing. - """ + get_env_or_raise_with_message.( + "SMF_REPO_PASSWORD", + "environment variable SMF_REPO_PASSWORD is missing." + ) smf_repo_hostname = - System.get_env("SMF_REPO_HOSTNAME") || - raise """ - environment variable SMF_REPO_HOSTNAME is missing. - """ + get_env_or_raise_with_message.( + "SMF_REPO_HOSTNAME", + "environment variable SMF_REPO_HOSTNAME is missing." + ) smf_repo_database = - System.get_env("SMF_REPO_DATABASE") || - raise """ - environment variable SMF_REPO_DATABASE is missing. - """ + get_env_or_raise_with_message.( + "SMF_REPO_DATABASE", + "environment variable SMF_REPO_DATABASE is missing." + ) proxy_repo_config = case config_env() do @@ -493,11 +493,11 @@ if config_env() != :test do password: smf_repo_password, hostname: smf_repo_hostname, database: smf_repo_database, - port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), - stacktrace: System.get_env("SMF_REPO_STACKTRACE") == "TRUE" || true, + port: get_env_cast_integer_with_default.("SMF_REPO_PORT", "3306"), + stacktrace: get_env_cast_bool_with_default.("SMF_REPO_STACKTRACE", "TRUE"), show_sensitive_data_on_connection_error: - System.get_env("SMF_REPO_SENSITIVE_DATA_ON_ERROR") == "TRUE" || false, - pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") + get_env_cast_bool_with_default.("SMF_REPO_SENSITIVE_DATA_ON_ERROR", "FALSE"), + pool_size: get_env_cast_integer_with_default.("SMF_REPO_POOL_SIZE", "10") ] _ -> @@ -506,10 +506,10 @@ if config_env() != :test do password: smf_repo_password, hostname: smf_repo_hostname, database: smf_repo_database, - port: String.to_integer(System.get_env("SMF_REPO_PORT") || "3306"), + port: get_env_cast_integer_with_default.("SMF_REPO_PORT", "3306"), stacktrace: true, show_sensitive_data_on_connection_error: true, - pool_size: String.to_integer(System.get_env("SMF_REPO_POOL_SIZE") || "10") + pool_size: get_env_cast_integer_with_default.("SMF_REPO_POOL_SIZE", "10") ] end From a969a71f5dc10a7643df965ec975d53d1ca2668b Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 18 Oct 2024 13:45:38 -0700 Subject: [PATCH 107/231] refactor(config/runtime): modify board/thread seq to start at -10 for test prevent test from trying to run proxy code --- config/runtime.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 5e14d1ca..492f30fd 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -450,8 +450,8 @@ proxy_config = # default thread/board seq and empty blacklist _ -> %{ - threads_seq: "0", - boards_seq: "0", + threads_seq: "-10", + boards_seq: "-10", id_board_blacklist: [] } end From ff97c4739891dd8d66bd26436668d7f57d0c2826 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 18 Oct 2024 13:47:53 -0700 Subject: [PATCH 108/231] refactor(controllers/board): check boards seq in proxy :by_category don't run proxy by_category if boards seq is lower than 0 (test mode) --- lib/epochtalk_server_web/controllers/board.ex | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 7aa3c4ef..0106de91 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -110,14 +110,15 @@ defmodule EpochtalkServerWeb.Controllers.Board do end defp check_proxy(conn, _) do + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() + conn = case conn.private.phoenix_action do :slug_to_id -> case Integer.parse(conn.params["slug"]) do {_, ""} -> slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) - boards_seq = boards_seq |> String.to_integer() if slug_as_id < boards_seq do conn @@ -132,9 +133,13 @@ defmodule EpochtalkServerWeb.Controllers.Board do end :by_category -> - conn - |> proxy_by_category(conn.params) - |> halt() + if boards_seq > 0 do + conn + |> proxy_by_category(conn.params) + |> halt() + else + conn + end _ -> conn From b5a6d91a08a7a137b4a90ee614ccdb41589e2c41 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 18 Oct 2024 13:49:34 -0700 Subject: [PATCH 109/231] fix(json/board_json): remove proxy options from non proxy :by_category --- lib/epochtalk_server_web/json/board_json.ex | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 2d1ae252..daf9e72e 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -44,10 +44,8 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, - user_priority: user_priority, + user_priority: user_priority # board counts and last post info for proxy version - board_counts: board_counts, - board_last_post_info: board_last_post_info }) do # append board moderators to each board in board mapping board_mapping = @@ -77,10 +75,8 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_mapping, :boards, category, - user_priority, + user_priority # board counts and last post info for proxy version - board_counts, - board_last_post_info ) acc ++ [category] From 1a8bfe7a580bc70300104004b04aca5e1cf48277 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 21 Oct 2024 15:31:09 -1000 Subject: [PATCH 110/231] fix(static-testing): resolve issues with credo and formatting for testing --- config/runtime.exs | 2 +- lib/epochtalk_server/smf_loader.ex | 114 +++++++++++------- .../controllers/thread.ex | 64 +++++----- .../helpers/breadcrumbs.ex | 1 + .../helpers/proxy_conversion.ex | 99 +++++++++------ lib/epochtalk_server_web/json/board_json.ex | 34 +++--- lib/epochtalk_server_web/json/post_json.ex | 15 ++- 7 files changed, 200 insertions(+), 129 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 665dc1c6..3ae66c39 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -460,7 +460,7 @@ if config_env() == :prod do id_board_blacklist = id_board_blacklist |> String.split() - |> Enum.map(&(String.to_integer(&1))) + |> Enum.map(&String.to_integer(&1)) # Configure proxy config :epochtalk_server, diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index 9c0655e3..ffcd2522 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -1,10 +1,15 @@ defmodule EpochtalkServer.SMFLoader do alias EpochtalkServer.Models.BoardMapping + @moduledoc """ + SMFLoader, for pulling bitcointalk boards and categories + """ + # converts smf_boards tsv file to epochtalk boards tsv file def convert_smf_boards_tsv_file(path) do - {boards, board_mappings} = load_from_tsv_file(path) - |> map_boards_stream() + {boards, board_mappings} = + load_from_tsv_file(path) + |> map_boards_stream() # write boards to file for import boards @@ -14,6 +19,7 @@ defmodule EpochtalkServer.SMFLoader do # return board mappings for later insertion board_mappings end + def load_categories_mappings_from_tsv_file(path) do # load categories (in postgres format) # and return category mappings for later insertion @@ -28,42 +34,46 @@ defmodule EpochtalkServer.SMFLoader do } end) end + def insert_board_mappings(board_mappings) do # board mappings board_mappings |> BoardMapping.update() end + # loads smf data from a tsv file def load_from_tsv_file(path) do with true <- if(File.exists?(path), do: true, else: "ファイルがない"), - {:ok, file} <- File.read(path), - file_lines <- file |> String.trim() |> String.split("\n"), - [header_line | file_lines] <- file_lines, - headers <- - # clean line - header_line - |> String.trim() - |> String.split("\t"), - smf_boards <- - file_lines - |> Enum.map(&( - &1 - # clean line - |> String.trim() - |> String.split("\t") - )) - |> Enum.map(fn line -> - # map each line to headers - Enum.zip(headers, line) - |> Enum.into(%{}) - end) do - smf_boards + {:ok, file} <- File.read(path), + file_lines <- file |> String.trim() |> String.split("\n"), + [header_line | file_lines] <- file_lines, + # clean line + headers <- + header_line + |> String.trim() + |> String.split("\t"), + smf_boards <- + file_lines + |> Enum.map( + &(&1 + # clean line + |> String.trim() + |> String.split("\t")) + ) + |> Enum.map(fn line -> + # map each line to headers + Enum.zip(headers, line) + |> Enum.into(%{}) + end) do + smf_boards else problem -> IO.puts("問題がある： #{inspect(problem)}") end end + def map_boards_stream(boards_stream) do now = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + {board_mapping_pairs, _slug_duplicate_index} = boards_stream |> Enum.map_reduce(%{}, fn smf_board, slugs -> @@ -72,19 +82,24 @@ defmodule EpochtalkServer.SMFLoader do |> HtmlEntities.decode() |> String.replace(~r{[ /]}, "-") |> String.slice(0..99) + # handle duplicate slugs slug_duplicate_index = Map.get(slugs, slug) + {slugs, slug} = if slug_duplicate_index == nil do # keep track of used slugs, return original slug {Map.put(slugs, slug, 0), slug} else # replace last characters with index - new_slug = slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) + new_slug = + slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) + new_slug = new_slug <> to_string(slug_duplicate_index) # increment used slugs, return new slug {Map.put(slugs, slug, slug_duplicate_index + 1), new_slug} end + # build board board = %{} @@ -98,7 +113,10 @@ defmodule EpochtalkServer.SMFLoader do |> Map.put(:created_at, now) |> Map.put(:imported_at, now) |> Map.put(:updated_at, now) - |> Map.put(:meta, "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"") + |> Map.put( + :meta, + "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"" + ) |> Map.put(:right_to_left, "f") |> Map.put(:slug, slug) @@ -114,6 +132,7 @@ defmodule EpochtalkServer.SMFLoader do category_id: smf_board["ID_CAT"] |> String.to_integer(), view_order: smf_board["boardOrder"] |> String.to_integer() } + _ -> %{ type: "board", @@ -123,10 +142,13 @@ defmodule EpochtalkServer.SMFLoader do view_order: smf_board["boardOrder"] |> String.to_integer() } end + {{board, board_mapping}, slugs} end) + Enum.unzip(board_mapping_pairs) end + def tabulate_boards_map(boards_map) do data = boards_map @@ -149,27 +171,31 @@ defmodule EpochtalkServer.SMFLoader do |> Enum.join("\t") end) - header = [ - "id", - "name", - "description", - "post_count", - "thread_count", - "viewable_by", - "postable_by", - "created_at", - "imported_at", - "updated_at", - "meta", - "right_to_left", - "slug" - ] |> Enum.join("\t") - [ header | data ] + header = + [ + "id", + "name", + "description", + "post_count", + "thread_count", + "viewable_by", + "postable_by", + "created_at", + "imported_at", + "updated_at", + "meta", + "right_to_left", + "slug" + ] + |> Enum.join("\t") + + [header | data] end + def write_to_tsv_file(data, path) do with false <- if(File.exists?(path), do: "ファイルがもうある", else: false), - file <- File.stream!(path) do - data |> Enum.into(file, fn line -> line <> "\n" end) + file <- File.stream!(path) do + data |> Enum.into(file, fn line -> line <> "\n" end) else problem -> IO.puts("問題がある： #{inspect(problem)}") end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index acb92094..9b3e15a1 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -754,36 +754,10 @@ defmodule EpochtalkServerWeb.Controllers.Thread do defp check_proxy(conn, _) do case conn.private.phoenix_action do :by_board -> - %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) - boards_seq = boards_seq |> String.to_integer() - - if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do - conn - |> proxy_by_board(conn.params) - |> halt() - else - conn - end + maybe_proxy_by_board(conn, conn.params) :slug_to_id -> - case Integer.parse(conn.params["slug"]) do - {_, ""} -> - slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) - - %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) - threads_seq = threads_seq |> String.to_integer() - - if slug_as_id < threads_seq do - conn - |> render(:slug_to_id, id: slug_as_id) - |> halt() - else - conn - end - - _ -> - conn - end + maybe_proxy_slug_to_id(conn, conn.params) :viewed -> conn @@ -800,6 +774,40 @@ defmodule EpochtalkServerWeb.Controllers.Thread do end end + defp maybe_proxy_by_board(conn, attrs) do + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() + + if Validate.cast(attrs, "board_id", :integer, required: true) < boards_seq do + conn + |> proxy_by_board(attrs) + |> halt() + else + conn + end + end + + defp maybe_proxy_slug_to_id(conn, attrs) do + case Integer.parse(attrs["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(attrs, "slug", :integer, required: true) + + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() + + if slug_as_id < threads_seq do + conn + |> render(:slug_to_id, id: slug_as_id) + |> halt() + else + conn + end + + _ -> + conn + end + end + defp proxy_by_board(conn, attrs) do with board_id <- Validate.cast(attrs, "board_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 737f1324..4e9e84ad 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -116,6 +116,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do "thread" -> {:ok, board} = Board.find_by_id(object.board_id) %{id: board.slug, type: "board"} + _ -> nil end diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index e55d4d23..ae4e291b 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -77,7 +77,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do display_mode: "always", expiration: p.expireTime * 1000, has_voted: false, - locked: (p.votingLocked == 1), + locked: p.votingLocked == 1, max_answers: p.maxVotes, question: p.question }) @@ -109,7 +109,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_recent_threads() do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(t in "smf_topics", limit: 5, where: t.id_board not in ^id_board_blacklist, @@ -148,7 +150,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Recent threads not found"} - threads -> threads + threads -> + threads end end @@ -172,7 +175,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_board_counts() do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist, select: %{ @@ -186,12 +191,15 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Boards not found"} - boards -> return_tuple(boards) + boards -> + return_tuple(boards) end end def build_board_last_post_info() do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist ) @@ -214,11 +222,15 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Boards not found"} - boards -> return_tuple(boards) + boards -> + return_tuple(boards) end end + def build_board(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(b in "smf_boards", where: b.id_board == ^id and b.id_board not in ^id_board_blacklist, select: %{ @@ -242,9 +254,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_threads_by_board(id, page, per_page) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + count_query = - from t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, select: %{count: count(t.id_topic)} + from t in "smf_topics", + where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, + select: %{count: count(t.id_topic)} from(t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, @@ -293,7 +309,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_thread(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(t in "smf_topics", where: t.id_topic == ^id and t.id_board not in ^id_board_blacklist ) @@ -336,7 +354,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_post(id) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + from(m in "smf_messages", where: m.id_msg == ^id and m.id_board not in ^id_board_blacklist, select: %{ @@ -360,9 +380,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_posts_by_thread(id, page, per_page) do - %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + count_query = - from m in "smf_messages", where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, select: %{count: count(m.id_topic)} + from m in "smf_messages", + where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, + select: %{count: count(m.id_topic)} from(m in "smf_messages", limit: ^per_page, @@ -374,32 +398,31 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do on: u.id_member == a.id_member and a.attachmentType == 1 ) |> select([m, u, a], %{ - id: m.id_msg, - thread_id: m.id_topic, - board_id: m.id_board, - title: m.subject, - body: m.body, - updated_at: m.modifiedTime, + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + title: m.subject, + body: m.body, + updated_at: m.modifiedTime, + username: m.posterName, + created_at: m.posterTime * 1000, + modified_time: m.modifiedTime, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ), + user: %{ + id: m.id_member, username: m.posterName, - created_at: m.posterTime * 1000, - modified_time: m.modifiedTime, - avatar: - fragment( - "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", - u.avatar, - u.avatar, - a.filename - ), - user: %{ - id: m.id_member, - username: m.posterName, - signature: u.signature, - activity: u.activity, - merit: u.merit, - title: u.usertitle - } + signature: u.signature, + activity: u.activity, + merit: u.merit, + title: u.usertitle } - ) + }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do {:ok, [], _} -> diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 6ffb2e75..2d1ae252 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -12,16 +12,18 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do Renders proxy version of `Board` data by `Category`. """ def proxy_by_category(%{ - categories: categories, - board_moderators: board_moderators, - board_mapping: board_mapping, - user_priority: user_priority, - board_counts: board_counts, - board_last_post_info: board_last_post_info - }) do + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: board_counts, + board_last_post_info: board_last_post_info + }) do board_counts = map_to_id(board_counts) board_last_post_info = map_to_id(board_last_post_info) - data = by_category(%{ + + data = + by_category(%{ categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, @@ -32,6 +34,7 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do data end + defp map_to_id(data), do: Enum.reduce(data, %{}, &(&2 |> Map.put(&1.id, Map.delete(&1, :id)))) @doc """ @@ -197,13 +200,16 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do |> Map.merge(board.thread) # add board counts for proxy version - board = if board_counts != nil, - do: Map.merge(board, board_counts[board.id]), - else: board + board = + if board_counts != nil, + do: Map.merge(board, board_counts[board.id]), + else: board + # add board last post info for proxy version - board = if board_last_post_info != nil, - do: Map.merge(board, board_last_post_info[board.id]), - else: board + board = + if board_last_post_info != nil, + do: Map.merge(board, board_last_post_info[board.id]), + else: board # delete unneeded properties board = diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index abf01a21..d260a861 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -409,14 +409,21 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(post.body || post.body_html, "'", "\'") - %Porcelain.Result{out: parsed_body, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + %Porcelain.Result{out: parsed_body, status: _status} = + Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - signature = if post.user.signature, + signature = + if post.user.signature, do: String.replace(post.user.signature, "'", "\'"), else: nil - parsed_signature = if signature do - %Porcelain.Result{out: parsed_sig, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"") + parsed_signature = + if signature do + %Porcelain.Result{out: parsed_sig, status: _status} = + Porcelain.shell( + "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"" + ) + parsed_sig else nil From fa6bfe5f28075b7c1a9150e2bee24b46ea173ff8 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 22 Oct 2024 12:46:10 -1000 Subject: [PATCH 111/231] feat(user-find): initial db query for user find, updated to query by user id instead of username --- .../helpers/proxy_conversion.ex | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ae4e291b..248cbd50 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -45,6 +45,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "poll.by_thread" -> build_poll(id) + "user.find" -> + build_user(id) + _ -> build_model(nil, nil, nil, nil) end @@ -66,6 +69,31 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_user(user_id) do + from(u in "smf_members", where: u.id_member == ^user_id) + |> select([u], %{ + activity: u.activity, + avatar: u.avatar, + created_at: u.dateRegistered * 1000, + dob: u.birthdate, + gender: u.gender, + id: u.id_member, + language: nil, + last_active: nil, + location: u.location, + merit: u.merit, + id_group: u.id_group, + id_post_group: u.id_post_group, + signature: u.signature, + posts: u.posts, + name: u.realName, + username: u.realName, + title: u.usertitle, + website: u.websiteUrl + }) + |> SmfRepo.one() + end + def build_poll(thread_id) do from(t in "smf_topics", where: t.id_topic == ^thread_id From f11ff6ed4bcdf869d8e9f71b007544602ebb292f Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 22 Oct 2024 15:19:23 -1000 Subject: [PATCH 112/231] feat(user-find-route): implement initial route for user find proxy --- lib/epochtalk_server_web/controllers/user.ex | 26 ++++++++++++++++++++ lib/epochtalk_server_web/json/user_json.ex | 7 ++++++ 2 files changed, 33 insertions(+) diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index 07f2f452..4c00fced 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -17,6 +17,9 @@ defmodule EpochtalkServerWeb.Controllers.User do alias EpochtalkServerWeb.CustomErrors.InvalidPayload alias EpochtalkServerWeb.Helpers.ACL alias EpochtalkServerWeb.Helpers.Validate + alias EpochtalkServerWeb.Helpers.ProxyConversion + + plug :check_proxy when action in [:find] @doc """ Used to check if a username has already been taken @@ -269,4 +272,27 @@ defmodule EpochtalkServerWeb.Controllers.User do end def login(_conn, _attrs), do: raise(InvalidPayload) + + ## === Private Helper Functions === + + defp check_proxy(conn, _) do + case conn.private.phoenix_action do + :find -> + conn + |> proxy_find(conn.params) + |> halt() + + _ -> + conn + end + end + + defp proxy_find(conn, attrs) do + with user <- ProxyConversion.build_model("user.find", attrs["id"]) do + render(conn, :find_proxy, %{user: user}) + else + _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot find specified user") + end + end + end diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 139d313a..aa3c0b48 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -13,6 +13,13 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do """ def user(%{user: user, token: token}), do: format_user_reply(user, token) + @doc """ + Renders formatted user JSON for find proxy + """ + def find_proxy(%{user: user}) do + user + end + @doc """ Renders formatted user JSON for find """ From d698369d815f6445e97940644cf6214be236ed2b Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 22 Oct 2024 19:35:06 -0700 Subject: [PATCH 113/231] ci(github/workflows/main): add build and push image step push docker image to github container registry --- .github/workflows/main.yml | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 74abbb57..0cb0aca4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,12 @@ on: branches: - main +# env settings for github releases +# docker image push +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: dependencies: runs-on: ubuntu-latest @@ -203,3 +209,41 @@ jobs: # external_repository: epochtalk/server.epochtalk.github.io # publish_dir: ./doc # publish_branch: gh-pages + + # build and push image to github container registry + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.1 + - name: Log in to the Container registry + uses: docker/login-action@3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@5.5.1 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + id: push + uses: docker/build-push-action@6.9.0 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # https://docs.github.com/en/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1.4.3 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true From 96a165e5a104cba01c919c087511bf95a9440852 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 22 Oct 2024 19:38:00 -0700 Subject: [PATCH 114/231] ci(github/workflows/main): depend on static analysis before pushing to image repo --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0cb0aca4..5609f1b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -212,6 +212,7 @@ jobs: # build and push image to github container registry build-and-push-image: + needs: [static_code_analysis] runs-on: ubuntu-latest permissions: contents: read From fe66fd8d1cfa6f806bd8da3852a08d817c976183 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 22 Oct 2024 19:46:11 -0700 Subject: [PATCH 115/231] ci(github/workflows/main): update action versions (add v) --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5609f1b0..6c09f853 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -223,19 +223,19 @@ jobs: - name: Checkout repository uses: actions/checkout@v4.2.1 - name: Log in to the Container registry - uses: docker/login-action@3.3.0 + uses: docker/login-action@v3.3.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@5.5.1 + uses: docker/metadata-action@v5.5.1 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push Docker image id: push - uses: docker/build-push-action@6.9.0 + uses: docker/build-push-action@v6.9.0 with: context: . push: true From a1ff6571bec8418f3118a94f51f6d3e8b7092bf2 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 24 Oct 2024 09:25:10 -0700 Subject: [PATCH 116/231] fix(helpers/breadcrumbs): return correctly for nonexistant thread refactor for edge case; code was failing when thread did not exist --- .../helpers/breadcrumbs.ex | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 4e9e84ad..a74a1c5f 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -73,29 +73,29 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) threads_seq = threads_seq |> String.to_integer() - thread = - cond do - is_integer(id) && id < threads_seq -> - ProxyConversion.build_model("thread", id) - - is_binary(id) -> - Thread.find(id) - - true -> - build_crumbs(nil, nil, crumbs) - end - - crumbs = [ - %{ - label: thread.title, - routeName: "Posts", - opts: %{slug: id, locked: thread.locked} - } - | crumbs - ] - - next_data = get_next_data(thread, "thread") - build_crumbs(next_data[:id], next_data[:type], crumbs) + thread = cond do + is_integer(id) && id < threads_seq -> + ProxyConversion.build_model("thread", id) + is_binary(id) -> + Thread.find(id) + true -> nil + end + + if is_nil(thread) do + build_crumbs(nil, nil, crumbs) + else + crumbs = [ + %{ + label: thread.title, + routeName: "Posts", + opts: %{slug: id, locked: thread.locked} + } + | crumbs + ] + + next_data = get_next_data(thread, "thread") + build_crumbs(next_data[:id], next_data[:type], crumbs) + end end defp get_next_data(object, type) do From 31da5bd72534d943be36f82e1f133823b90fb02b Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 24 Oct 2024 09:29:49 -0700 Subject: [PATCH 117/231] style(helpers/breadcrumbs): mix format --- .../helpers/breadcrumbs.ex | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index a74a1c5f..23e07bc5 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -73,13 +73,17 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) threads_seq = threads_seq |> String.to_integer() - thread = cond do - is_integer(id) && id < threads_seq -> - ProxyConversion.build_model("thread", id) - is_binary(id) -> - Thread.find(id) - true -> nil - end + thread = + cond do + is_integer(id) && id < threads_seq -> + ProxyConversion.build_model("thread", id) + + is_binary(id) -> + Thread.find(id) + + true -> + nil + end if is_nil(thread) do build_crumbs(nil, nil, crumbs) From 12d4d0625fa516135913faa49115dfdce8b87ee9 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 24 Oct 2024 09:31:54 -0700 Subject: [PATCH 118/231] ci(github/workflows/main): re-enable test job --- .github/workflows/main.yml | 120 ++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c09f853..2909b3f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -108,66 +108,66 @@ jobs: priv/plts - name: Run dialyzer run: mix dialyzer --format github - # test: - # needs: dependencies - # name: Run tests - # runs-on: ubuntu-latest - # services: - # postgres: - # image: postgres:14.2 - # env: - # POSTGRES_USER: postgres - # POSTGRES_PASSWORD: postgres - # POSTGRES_DB: epochtalk_server_test - # ports: - # - 5432:5432 - # # Set health checks to wait until postgres has started - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # redis: - # ports: - # - 6379:6379 - # image: redis:7.0.4 - # # Set health checks to wait until redis has started - # options: >- - # --health-cmd "redis-cli ping" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # steps: - # - name: Cancel previous runs - # uses: styfle/cancel-workflow-action@0.11.0 - # with: - # access_token: ${{ github.token }} - # - name: Checkout - # uses: actions/checkout@v3.6.0 - # with: - # ref: ${{ github.event.client_payload.branch }} - # - name: Parse .tool-versions - # id: tool-versions - # uses: paulo-ferraz-oliveira/parse-tool-versions@v1 - # - name: Sets up an Erlang/OTP environment - # uses: erlef/setup-beam@v1 - # with: - # version-file: .tool-versions - # version-type: strict - # - name: Retrieve cached dependencies - # uses: actions/cache@v3.3.1 - # id: mix-cache - # with: - # path: | - # deps - # _build - # key: ${{ runner.os }}-${{ steps.tool-versions.outputs.erlang }}-${{ steps.tool-versions.outputs.elixir }}-${{ hashFiles('mix.lock') }} - # - name: Run tests - # run: mix test - # env: - # AWS_REGION: ${{ secrets.AWS_REGION }} - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + test: + needs: dependencies + name: Run tests + runs-on: ubuntu-latest + services: + postgres: + image: postgres:14.2 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: epochtalk_server_test + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + ports: + - 6379:6379 + image: redis:7.0.4 + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + - name: Checkout + uses: actions/checkout@v3.6.0 + with: + ref: ${{ github.event.client_payload.branch }} + - name: Parse .tool-versions + id: tool-versions + uses: paulo-ferraz-oliveira/parse-tool-versions@v1 + - name: Sets up an Erlang/OTP environment + uses: erlef/setup-beam@v1 + with: + version-file: .tool-versions + version-type: strict + - name: Retrieve cached dependencies + uses: actions/cache@v3.3.1 + id: mix-cache + with: + path: | + deps + _build + key: ${{ runner.os }}-${{ steps.tool-versions.outputs.erlang }}-${{ steps.tool-versions.outputs.elixir }}-${{ hashFiles('mix.lock') }} + - name: Run tests + run: mix test + env: + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # release: # needs: [test, static_code_analysis] # name: Release From 718cca8c0961dc23f9aa63448c035ee80a6032a0 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 24 Oct 2024 10:36:14 -0700 Subject: [PATCH 119/231] Revert "fix(static-testing): resolve issues with credo and formatting for testing" Resolved in CI branch This reverts commit 1a8bfe7a580bc70300104004b04aca5e1cf48277. --- config/runtime.exs | 2 +- lib/epochtalk_server/smf_loader.ex | 114 +++++++----------- .../controllers/thread.ex | 64 +++++----- .../helpers/breadcrumbs.ex | 1 - .../helpers/proxy_conversion.ex | 99 ++++++--------- lib/epochtalk_server_web/json/board_json.ex | 34 +++--- lib/epochtalk_server_web/json/post_json.ex | 15 +-- 7 files changed, 129 insertions(+), 200 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 3ae66c39..665dc1c6 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -460,7 +460,7 @@ if config_env() == :prod do id_board_blacklist = id_board_blacklist |> String.split() - |> Enum.map(&String.to_integer(&1)) + |> Enum.map(&(String.to_integer(&1))) # Configure proxy config :epochtalk_server, diff --git a/lib/epochtalk_server/smf_loader.ex b/lib/epochtalk_server/smf_loader.ex index ffcd2522..9c0655e3 100644 --- a/lib/epochtalk_server/smf_loader.ex +++ b/lib/epochtalk_server/smf_loader.ex @@ -1,15 +1,10 @@ defmodule EpochtalkServer.SMFLoader do alias EpochtalkServer.Models.BoardMapping - @moduledoc """ - SMFLoader, for pulling bitcointalk boards and categories - """ - # converts smf_boards tsv file to epochtalk boards tsv file def convert_smf_boards_tsv_file(path) do - {boards, board_mappings} = - load_from_tsv_file(path) - |> map_boards_stream() + {boards, board_mappings} = load_from_tsv_file(path) + |> map_boards_stream() # write boards to file for import boards @@ -19,7 +14,6 @@ defmodule EpochtalkServer.SMFLoader do # return board mappings for later insertion board_mappings end - def load_categories_mappings_from_tsv_file(path) do # load categories (in postgres format) # and return category mappings for later insertion @@ -34,46 +28,42 @@ defmodule EpochtalkServer.SMFLoader do } end) end - def insert_board_mappings(board_mappings) do # board mappings board_mappings |> BoardMapping.update() end - # loads smf data from a tsv file def load_from_tsv_file(path) do with true <- if(File.exists?(path), do: true, else: "ファイルがない"), - {:ok, file} <- File.read(path), - file_lines <- file |> String.trim() |> String.split("\n"), - [header_line | file_lines] <- file_lines, - # clean line - headers <- - header_line - |> String.trim() - |> String.split("\t"), - smf_boards <- - file_lines - |> Enum.map( - &(&1 - # clean line - |> String.trim() - |> String.split("\t")) - ) - |> Enum.map(fn line -> - # map each line to headers - Enum.zip(headers, line) - |> Enum.into(%{}) - end) do - smf_boards + {:ok, file} <- File.read(path), + file_lines <- file |> String.trim() |> String.split("\n"), + [header_line | file_lines] <- file_lines, + headers <- + # clean line + header_line + |> String.trim() + |> String.split("\t"), + smf_boards <- + file_lines + |> Enum.map(&( + &1 + # clean line + |> String.trim() + |> String.split("\t") + )) + |> Enum.map(fn line -> + # map each line to headers + Enum.zip(headers, line) + |> Enum.into(%{}) + end) do + smf_boards else problem -> IO.puts("問題がある： #{inspect(problem)}") end end - def map_boards_stream(boards_stream) do now = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - {board_mapping_pairs, _slug_duplicate_index} = boards_stream |> Enum.map_reduce(%{}, fn smf_board, slugs -> @@ -82,24 +72,19 @@ defmodule EpochtalkServer.SMFLoader do |> HtmlEntities.decode() |> String.replace(~r{[ /]}, "-") |> String.slice(0..99) - # handle duplicate slugs slug_duplicate_index = Map.get(slugs, slug) - {slugs, slug} = if slug_duplicate_index == nil do # keep track of used slugs, return original slug {Map.put(slugs, slug, 0), slug} else # replace last characters with index - new_slug = - slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) - + new_slug = slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10)))) new_slug = new_slug <> to_string(slug_duplicate_index) # increment used slugs, return new slug {Map.put(slugs, slug, slug_duplicate_index + 1), new_slug} end - # build board board = %{} @@ -113,10 +98,7 @@ defmodule EpochtalkServer.SMFLoader do |> Map.put(:created_at, now) |> Map.put(:imported_at, now) |> Map.put(:updated_at, now) - |> Map.put( - :meta, - "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"" - ) + |> Map.put(:meta, "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"") |> Map.put(:right_to_left, "f") |> Map.put(:slug, slug) @@ -132,7 +114,6 @@ defmodule EpochtalkServer.SMFLoader do category_id: smf_board["ID_CAT"] |> String.to_integer(), view_order: smf_board["boardOrder"] |> String.to_integer() } - _ -> %{ type: "board", @@ -142,13 +123,10 @@ defmodule EpochtalkServer.SMFLoader do view_order: smf_board["boardOrder"] |> String.to_integer() } end - {{board, board_mapping}, slugs} end) - Enum.unzip(board_mapping_pairs) end - def tabulate_boards_map(boards_map) do data = boards_map @@ -171,31 +149,27 @@ defmodule EpochtalkServer.SMFLoader do |> Enum.join("\t") end) - header = - [ - "id", - "name", - "description", - "post_count", - "thread_count", - "viewable_by", - "postable_by", - "created_at", - "imported_at", - "updated_at", - "meta", - "right_to_left", - "slug" - ] - |> Enum.join("\t") - - [header | data] + header = [ + "id", + "name", + "description", + "post_count", + "thread_count", + "viewable_by", + "postable_by", + "created_at", + "imported_at", + "updated_at", + "meta", + "right_to_left", + "slug" + ] |> Enum.join("\t") + [ header | data ] end - def write_to_tsv_file(data, path) do with false <- if(File.exists?(path), do: "ファイルがもうある", else: false), - file <- File.stream!(path) do - data |> Enum.into(file, fn line -> line <> "\n" end) + file <- File.stream!(path) do + data |> Enum.into(file, fn line -> line <> "\n" end) else problem -> IO.puts("問題がある： #{inspect(problem)}") end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 9b3e15a1..acb92094 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -754,10 +754,36 @@ defmodule EpochtalkServerWeb.Controllers.Thread do defp check_proxy(conn, _) do case conn.private.phoenix_action do :by_board -> - maybe_proxy_by_board(conn, conn.params) + %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) + boards_seq = boards_seq |> String.to_integer() + + if Validate.cast(conn.params, "board_id", :integer, required: true) < boards_seq do + conn + |> proxy_by_board(conn.params) + |> halt() + else + conn + end :slug_to_id -> - maybe_proxy_slug_to_id(conn, conn.params) + case Integer.parse(conn.params["slug"]) do + {_, ""} -> + slug_as_id = Validate.cast(conn.params, "slug", :integer, required: true) + + %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) + threads_seq = threads_seq |> String.to_integer() + + if slug_as_id < threads_seq do + conn + |> render(:slug_to_id, id: slug_as_id) + |> halt() + else + conn + end + + _ -> + conn + end :viewed -> conn @@ -774,40 +800,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do end end - defp maybe_proxy_by_board(conn, attrs) do - %{boards_seq: boards_seq} = Application.get_env(:epochtalk_server, :proxy_config) - boards_seq = boards_seq |> String.to_integer() - - if Validate.cast(attrs, "board_id", :integer, required: true) < boards_seq do - conn - |> proxy_by_board(attrs) - |> halt() - else - conn - end - end - - defp maybe_proxy_slug_to_id(conn, attrs) do - case Integer.parse(attrs["slug"]) do - {_, ""} -> - slug_as_id = Validate.cast(attrs, "slug", :integer, required: true) - - %{threads_seq: threads_seq} = Application.get_env(:epochtalk_server, :proxy_config) - threads_seq = threads_seq |> String.to_integer() - - if slug_as_id < threads_seq do - conn - |> render(:slug_to_id, id: slug_as_id) - |> halt() - else - conn - end - - _ -> - conn - end - end - defp proxy_by_board(conn, attrs) do with board_id <- Validate.cast(attrs, "board_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 4e9e84ad..737f1324 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -116,7 +116,6 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do "thread" -> {:ok, board} = Board.find_by_id(object.board_id) %{id: board.slug, type: "board"} - _ -> nil end diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ae4e291b..e55d4d23 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -77,7 +77,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do display_mode: "always", expiration: p.expireTime * 1000, has_voted: false, - locked: p.votingLocked == 1, + locked: (p.votingLocked == 1), max_answers: p.maxVotes, question: p.question }) @@ -109,9 +109,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_recent_threads() do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(t in "smf_topics", limit: 5, where: t.id_board not in ^id_board_blacklist, @@ -150,8 +148,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Recent threads not found"} - threads -> - threads + threads -> threads end end @@ -175,9 +172,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_board_counts() do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist, select: %{ @@ -191,15 +186,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Boards not found"} - boards -> - return_tuple(boards) + boards -> return_tuple(boards) end end def build_board_last_post_info() do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist ) @@ -222,15 +214,11 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do [] -> {:error, "Boards not found"} - boards -> - return_tuple(boards) + boards -> return_tuple(boards) end end - def build_board(id) do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(b in "smf_boards", where: b.id_board == ^id and b.id_board not in ^id_board_blacklist, select: %{ @@ -254,13 +242,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_threads_by_board(id, page, per_page) do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) count_query = - from t in "smf_topics", - where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, - select: %{count: count(t.id_topic)} + from t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, select: %{count: count(t.id_topic)} from(t in "smf_topics", where: t.id_board == ^id and t.id_board not in ^id_board_blacklist, @@ -309,9 +293,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_thread(id) do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(t in "smf_topics", where: t.id_topic == ^id and t.id_board not in ^id_board_blacklist ) @@ -354,9 +336,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_post(id) do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) from(m in "smf_messages", where: m.id_msg == ^id and m.id_board not in ^id_board_blacklist, select: %{ @@ -380,13 +360,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_posts_by_thread(id, page, per_page) do - %{id_board_blacklist: id_board_blacklist} = - Application.get_env(:epochtalk_server, :proxy_config) - + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) count_query = - from m in "smf_messages", - where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, - select: %{count: count(m.id_topic)} + from m in "smf_messages", where: m.id_topic == ^id and m.id_board not in ^id_board_blacklist, select: %{count: count(m.id_topic)} from(m in "smf_messages", limit: ^per_page, @@ -398,31 +374,32 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do on: u.id_member == a.id_member and a.attachmentType == 1 ) |> select([m, u, a], %{ - id: m.id_msg, - thread_id: m.id_topic, - board_id: m.id_board, - title: m.subject, - body: m.body, - updated_at: m.modifiedTime, - username: m.posterName, - created_at: m.posterTime * 1000, - modified_time: m.modifiedTime, - avatar: - fragment( - "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", - u.avatar, - u.avatar, - a.filename - ), - user: %{ - id: m.id_member, + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + title: m.subject, + body: m.body, + updated_at: m.modifiedTime, username: m.posterName, - signature: u.signature, - activity: u.activity, - merit: u.merit, - title: u.usertitle + created_at: m.posterTime * 1000, + modified_time: m.modifiedTime, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ), + user: %{ + id: m.id_member, + username: m.posterName, + signature: u.signature, + activity: u.activity, + merit: u.merit, + title: u.usertitle + } } - }) + ) |> ProxyPagination.page_simple(count_query, page, per_page: per_page) |> case do {:ok, [], _} -> diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 2d1ae252..6ffb2e75 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -12,18 +12,16 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do Renders proxy version of `Board` data by `Category`. """ def proxy_by_category(%{ - categories: categories, - board_moderators: board_moderators, - board_mapping: board_mapping, - user_priority: user_priority, - board_counts: board_counts, - board_last_post_info: board_last_post_info - }) do + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: board_counts, + board_last_post_info: board_last_post_info + }) do board_counts = map_to_id(board_counts) board_last_post_info = map_to_id(board_last_post_info) - - data = - by_category(%{ + data = by_category(%{ categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, @@ -34,7 +32,6 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do data end - defp map_to_id(data), do: Enum.reduce(data, %{}, &(&2 |> Map.put(&1.id, Map.delete(&1, :id)))) @doc """ @@ -200,16 +197,13 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do |> Map.merge(board.thread) # add board counts for proxy version - board = - if board_counts != nil, - do: Map.merge(board, board_counts[board.id]), - else: board - + board = if board_counts != nil, + do: Map.merge(board, board_counts[board.id]), + else: board # add board last post info for proxy version - board = - if board_last_post_info != nil, - do: Map.merge(board, board_last_post_info[board.id]), - else: board + board = if board_last_post_info != nil, + do: Map.merge(board, board_last_post_info[board.id]), + else: board # delete unneeded properties board = diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index d260a861..abf01a21 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -409,21 +409,14 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(post.body || post.body_html, "'", "\'") - %Porcelain.Result{out: parsed_body, status: _status} = - Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + %Porcelain.Result{out: parsed_body, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - signature = - if post.user.signature, + signature = if post.user.signature, do: String.replace(post.user.signature, "'", "\'"), else: nil - parsed_signature = - if signature do - %Porcelain.Result{out: parsed_sig, status: _status} = - Porcelain.shell( - "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"" - ) - + parsed_signature = if signature do + %Porcelain.Result{out: parsed_sig, status: _status} = Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"") parsed_sig else nil From ffd69750c99dc71149c38898de51823854ee5ab0 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 24 Oct 2024 10:48:37 -0700 Subject: [PATCH 120/231] ci(github/workflows/main): depend on test before build and push --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2909b3f1..35468b11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -212,7 +212,7 @@ jobs: # build and push image to github container registry build-and-push-image: - needs: [static_code_analysis] + needs: [static_code_analysis, test] runs-on: ubuntu-latest permissions: contents: read From 30a4e191e696cddddff81c36e7b2e9d0d4cf8dc4 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 24 Oct 2024 10:49:37 -1000 Subject: [PATCH 121/231] feat(profile): bring back user id in last post info for linking to profile, parse signature user find --- .../helpers/proxy_conversion.ex | 1 + lib/epochtalk_server_web/json/user_json.ex | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 248cbd50..8bbf9e63 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -238,6 +238,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_post_created_at: m.posterTime * 1000, last_post_position: t.numReplies, last_post_username: m.posterName, + last_post_user_id: m.id_member, last_thread_created_at: t.id_member_started, last_thread_id: t.id_topic, last_thread_post_count: t.numReplies, diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index aa3c0b48..f09d23d1 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -17,7 +17,19 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do Renders formatted user JSON for find proxy """ def find_proxy(%{user: user}) do - user + parsed_signature = + if user.signature do + %Porcelain.Result{out: parsed_sig, status: _status} = + Porcelain.shell( + "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> user.signature <> "');\"" + ) + + parsed_sig + else + nil + end + + user |> Map.put(:signature, parsed_signature) end @doc """ From ebdb77cd2e6899a65834c8ff215f2648e5fdbf24 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 25 Oct 2024 10:26:57 -1000 Subject: [PATCH 122/231] feat(posts-by-username): implement db query for posts by username proxy --- .../helpers/proxy_conversion.ex | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 8bbf9e63..f02cc8fa 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -23,6 +23,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "posts.by_thread" -> build_posts_by_thread(id, page, per_page) + "posts.by_user" -> + build_posts_by_user(id, page, per_page) + _ -> build_model(nil, nil, nil, nil) end @@ -462,6 +465,44 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_posts_by_user(id, page, per_page) do + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + + count_query = + from u in "smf_members", + where: u.id_member == ^id, + select: %{count: u.posts} + + from(m in "smf_messages", + limit: ^per_page, + where: m.id_member == ^id and m.id_board not in ^id_board_blacklist, + order_by: [desc: m.id_msg] + ) + |> select([m], %{ + id: m.id_msg, + thread_id: m.id_topic, + board_id: m.id_board, + thread_title: m.subject, + thread_slug: m.id_topic, + position: 0, + body_html: m.body, + updated_at: m.modifiedTime, + created_at: m.posterTime * 1000, + user: %{ + id: m.id_member, + } + }) + |> ProxyPagination.page_simple(count_query, page, per_page: per_page) + |> case do + {:ok, [], _} -> + {:error, "Posts not found for user_id: #{id}"} + + {:ok, posts, data} -> + return_tuple(posts, data) + end + end + defp return_tuple(object) do if length(object) > 1 do {:ok, object} From bd247d813c7bd98e99fb5ce3a3c87c2cb225bcd9 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 28 Oct 2024 11:40:15 -1000 Subject: [PATCH 123/231] feat(post-by-username): route for post by username --- lib/epochtalk_server_web/controllers/post.ex | 29 ++++++++++++++++++- .../helpers/proxy_conversion.ex | 2 +- lib/epochtalk_server_web/json/post_json.ex | 21 ++++++++++++-- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 667fd2a1..66a1767b 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -32,7 +32,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do @max_post_title_length 255 - plug :check_proxy when action in [:by_thread] + plug :check_proxy when action in [:by_thread, :by_username] @doc """ Used to create posts @@ -542,6 +542,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do conn end + :by_username -> + conn + |> proxy_by_username(conn.params) + |> halt() + _ -> conn end @@ -549,6 +554,28 @@ defmodule EpochtalkServerWeb.Controllers.Post do conn end + defp proxy_by_username(conn, attrs) do + # Parameter Validation + with user_id <- Validate.cast(attrs, "id", :integer, required: true), + page <- Validate.cast(attrs, "page", :integer, default: 1, min: 1), + limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), + {:ok, posts, data} <- ProxyConversion.build_model("posts.by_user", user_id, page, limit) do + render(conn, :proxy_by_username, %{ + posts: posts, + count: data.total_records, + limit: data.per_page, + page: data.page, + desc: true + }) + else + {:view_deleted_users, false} -> + ErrorHelpers.render_json_error(conn, 400, "Account not found") + + _ -> + ErrorHelpers.render_json_error(conn, 400, "Error, cannot get posts by username") + end + end + defp proxy_by_thread(conn, attrs) do with thread_id <- Validate.cast(attrs, "thread_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index f02cc8fa..c1ba7383 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -88,7 +88,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do id_group: u.id_group, id_post_group: u.id_post_group, signature: u.signature, - posts: u.posts, + post_count: u.posts, name: u.realName, username: u.realName, title: u.usertitle, diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index d260a861..de5474d6 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -152,8 +152,6 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do } end - ## === Private Helper Functions === - @doc """ Renders all `Post` for a particular `User`. """ @@ -181,6 +179,25 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do } end + @doc """ + Renders all `Post` for a particular `User`. + """ + def proxy_by_username(%{ + posts: posts, + count: count, + limit: limit, + page: page, + desc: desc + }) do + %{ + posts: posts, + count: count, + limit: limit, + page: page, + desc: desc + } + end + ## === Public Helper Functions === def handle_deleted_posts(posts, thread, user, authed_user_priority, view_deleted_posts) do From e9a3a74d725bfe7f0cfe8838af986560e85cc10d Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 29 Oct 2024 13:58:46 -0700 Subject: [PATCH 124/231] Revert "fix(json/board_json): remove proxy options from non proxy :by_category" This reverts commit b5a6d91a08a7a137b4a90ee614ccdb41589e2c41. --- lib/epochtalk_server_web/json/board_json.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index daf9e72e..2d1ae252 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -44,8 +44,10 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do categories: categories, board_moderators: board_moderators, board_mapping: board_mapping, - user_priority: user_priority + user_priority: user_priority, # board counts and last post info for proxy version + board_counts: board_counts, + board_last_post_info: board_last_post_info }) do # append board moderators to each board in board mapping board_mapping = @@ -75,8 +77,10 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_mapping, :boards, category, - user_priority + user_priority, # board counts and last post info for proxy version + board_counts, + board_last_post_info ) acc ++ [category] From aa1663002a1eb95c85647fd9b32f2aad798cf2ea Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 29 Oct 2024 14:07:57 -0700 Subject: [PATCH 125/231] fix(json/board): handle base case for test add function head for case where board_counts and board_last_post_info don't exist fixes broken test --- lib/epochtalk_server_web/json/board_json.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 2d1ae252..75ed8f30 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -90,6 +90,22 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do %{boards: categories} end + def by_category(%{ + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority + }) do + by_category(%{ + categories: categories, + board_moderators: board_moderators, + board_mapping: board_mapping, + user_priority: user_priority, + board_counts: nil, + board_last_post_info: nil + }) + end + @doc """ Renders `Board` for find query. """ From f141bdf7d067540cab02c8fd27700512cdeaedac Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 29 Oct 2024 11:23:01 -1000 Subject: [PATCH 126/231] refactor(post-by-username): remove unneeded error check --- lib/epochtalk_server_web/controllers/post.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 66a1767b..ebf71ffb 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -568,9 +568,6 @@ defmodule EpochtalkServerWeb.Controllers.Post do desc: true }) else - {:view_deleted_users, false} -> - ErrorHelpers.render_json_error(conn, 400, "Account not found") - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot get posts by username") end From 6f414bce671040be62516998cdb6862ba4a3ef7c Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 29 Oct 2024 11:24:10 -1000 Subject: [PATCH 127/231] feat(thread-by-username): initial implementation of thread by username, update proxy pagination to support next/prev --- lib/epochtalk_server_web/controllers/post.ex | 3 +- .../controllers/thread.ex | 30 +++++++++- .../helpers/proxy_conversion.ex | 55 +++++++++++++++++-- .../helpers/proxy_pagination.ex | 54 +++++++++++++----- lib/epochtalk_server_web/json/thread_json.ex | 22 ++++++++ 5 files changed, 142 insertions(+), 22 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index ebf71ffb..e7e7cef1 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -559,7 +559,8 @@ defmodule EpochtalkServerWeb.Controllers.Post do with user_id <- Validate.cast(attrs, "id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1, min: 1), limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), - {:ok, posts, data} <- ProxyConversion.build_model("posts.by_user", user_id, page, limit) do + desc <- Validate.cast(attrs, "desc", :boolean, default: true), + {:ok, posts, data} <- ProxyConversion.build_model("posts.by_user", user_id, page, limit, desc) do render(conn, :proxy_by_username, %{ posts: posts, count: data.total_records, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 909a7e06..0be741dd 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -28,7 +28,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do alias EpochtalkServer.Models.Mention alias EpochtalkServerWeb.Helpers.ProxyConversion - plug :check_proxy when action in [:by_board, :slug_to_id, :viewed, :recent] + plug :check_proxy when action in [:by_board, :by_username, :slug_to_id, :viewed, :recent] @doc """ Used to retrieve recent threads @@ -801,11 +801,39 @@ defmodule EpochtalkServerWeb.Controllers.Thread do |> halt() end + # check proxy for :recent action + defp check_proxy(%{private: %{phoenix_action: :by_username}} = conn, _) do + conn + |> proxy_by_username(conn.params) + |> halt() + end + # check proxy default defp check_proxy(%{private: %{phoenix_action: _}} = conn, _) do conn end + defp proxy_by_username(conn, attrs) do + # Parameter Validation + with user_id <- Validate.cast(attrs, "id", :integer, required: true), + page <- Validate.cast(attrs, "page", :integer, default: 1, min: 1), + limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), + desc <- Validate.cast(attrs, "desc", :boolean, default: true), + {:ok, threads, data} <- ProxyConversion.build_model("threads.by_user", user_id, page, limit, desc) do + render(conn, :proxy_by_username, %{ + threads: threads, + next: data.next, + prev: data.prev, + limit: data.per_page, + page: data.page, + desc: desc + }) + else + _ -> + ErrorHelpers.render_json_error(conn, 400, "Error, cannot get threads by username") + end + end + defp proxy_by_board(conn, attrs) do with board_id <- Validate.cast(attrs, "board_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index c1ba7383..a2e3bcb3 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -23,8 +23,18 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "posts.by_thread" -> build_posts_by_thread(id, page, per_page) + _ -> + build_model(nil, nil, nil, nil) + end + end + + def build_model(model_type, id, page, per_page, desc) when is_integer(id) do + case model_type do + "threads.by_user" -> + build_threads_by_user(id, page, per_page, desc) + "posts.by_user" -> - build_posts_by_user(id, page, per_page) + build_posts_by_user(id, page, per_page, desc) _ -> build_model(nil, nil, nil, nil) @@ -337,7 +347,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_viewed: nil, is_proxy: true }) - |> ProxyPagination.page_simple(count_query, page, per_page: per_page) + |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: true) end def build_thread(id) do @@ -455,7 +465,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do title: u.usertitle } }) - |> ProxyPagination.page_simple(count_query, page, per_page: per_page) + |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: true) |> case do {:ok, [], _} -> {:error, "Posts not found for thread_id: #{id}"} @@ -465,10 +475,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end - def build_posts_by_user(id, page, per_page) do + def build_posts_by_user(id, page, per_page, desc) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) + direction = if desc, do: :desc, else: :asc + count_query = from u in "smf_members", where: u.id_member == ^id, @@ -477,7 +489,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do from(m in "smf_messages", limit: ^per_page, where: m.id_member == ^id and m.id_board not in ^id_board_blacklist, - order_by: [desc: m.id_msg] + order_by: [{^direction, m.id_msg}] ) |> select([m], %{ id: m.id_msg, @@ -493,7 +505,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do id: m.id_member, } }) - |> ProxyPagination.page_simple(count_query, page, per_page: per_page) + |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: desc) |> case do {:ok, [], _} -> {:error, "Posts not found for user_id: #{id}"} @@ -503,6 +515,37 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_threads_by_user(id, page, per_page, desc) do + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + + direction = if desc, do: :desc, else: :asc + + from(t in "smf_topics", + where: t.id_member_started == ^id and t.id_board not in ^id_board_blacklist, + order_by: [{^direction, t.id_topic}] + ) + |> join(:left, [t], f in "smf_messages", on: t.id_first_msg == f.id_msg) + |> join(:left, [t], l in "smf_messages", on: t.id_last_msg == l.id_msg) + |> select([t, f, l], %{ + thread_id: t.id_topic, + thread_slug: t.id_topic, + board_id: t.id_board, + sticky: t.isSticky, + locked: t.locked, + user: %{id: t.id_member_started, deleted: false }, + moderated: t.selfModerated, + post_count: t.numReplies, + thread_title: f.subject, + body: f.body, + created_at: f.posterTime * 1000, + updated_at: l.posterTime * 1000, + board_visible: true, + is_proxy: true + }) + |> ProxyPagination.page_next_prev(page, per_page: per_page, desc: desc) + end + defp return_tuple(object) do if length(object) > 1 do {:ok, object} diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index 6769893f..4e819ed9 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -39,35 +39,36 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do page :: integer | String.t() | nil, per_page: integer | String.t() | nil ) :: {:ok, list :: [term()] | [], pagination_data :: map()} - def page_simple(query, count_query, nil, per_page: nil), - do: page_simple(query, count_query, 1, per_page: 15) + def page_simple(query, count_query, nil, per_page: nil, desc: desc), + do: page_simple(query, count_query, 1, per_page: 15, desc: desc) - def page_simple(query, count_query, page, per_page: nil) when is_integer(page), - do: page_simple(query, count_query, page, per_page: 15) + def page_simple(query, count_query, page, per_page: nil, desc: desc) when is_integer(page), + do: page_simple(query, count_query, page, per_page: 15, desc: desc) - def page_simple(query, count_query, nil, per_page: per_page) when is_integer(per_page), - do: page_simple(query, count_query, 1, per_page: per_page) + def page_simple(query, count_query, nil, per_page: per_page, desc: desc) when is_integer(per_page), + do: page_simple(query, count_query, 1, per_page: per_page, desc: desc) - def page_simple(query, count_query, nil, per_page: per_page) when is_binary(per_page), + def page_simple(query, count_query, nil, per_page: per_page, desc: desc) when is_binary(per_page), do: page_simple(query, count_query, 1, - per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1) + per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1), + desc: desc ) - def page_simple(query, count_query, page, per_page: nil) when is_binary(page), + def page_simple(query, count_query, page, per_page: nil, desc: desc) when is_binary(page), do: page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), - per_page: 15 + per_page: 15, desc: desc ) - def page_simple(query, count_query, page, per_page: per_page) + def page_simple(query, count_query, page, per_page: per_page, desc: desc) when is_binary(page) and is_binary(per_page), do: page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), - per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1) + per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1), desc: desc ) - def page_simple(query, count_query, page, per_page: per_page) do + def page_simple(query, count_query, page, per_page: per_page, desc: desc) do options = [prefix: "public"] total_records = @@ -85,12 +86,37 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do page: page, per_page: per_page, total_records: total_records, - total_pages: total_pages + total_pages: total_pages, + desc: desc } {:ok, result, pagination_data} end + def page_next_prev(query, page, per_page: per_page, desc: desc) do + # query one more page to calculate if next page exists + result = records(query, page, nil, per_page + 1) + + pagination_data = %{ + next: length(result) == per_page + 1, + prev: page > 1, + page: page, + per_page: per_page, + desc: desc + } + + # remove extra element + result = result |> Enum.reverse() |> tl() |> Enum.reverse() + {:ok, result, pagination_data} + end + + defp records(query, page, total_pages, per_page) when is_nil(total_pages) do + query + |> limit(^per_page) + |> offset(^(per_page * (page - 1))) + |> SmfRepo.all() + end + defp records(_, page, total_pages, _) when page > total_pages, do: [] defp records(query, page, _, per_page) do diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index feb2b94c..f085a921 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -195,6 +195,28 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do } end + @doc """ + Renders all `Post` for a particular `User`. + """ + def proxy_by_username(%{ + threads: threads, + next: next, + prev: prev, + limit: limit, + page: page, + desc: desc + }) do + %{ + posts: threads, + next: next, + prev: prev, + limit: limit, + page: page, + desc: desc + } + end + + @doc """ Renders sticky `Thread`. From bf3d7263ec271509eb0fbdb0e9b86ca177b336c5 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 31 Oct 2024 13:26:45 -1000 Subject: [PATCH 128/231] yolo(parser): poolboy parser implementation wip --- config/runtime.exs | 9 +++ lib/epochtalk_server/application.ex | 4 + lib/epochtalk_server/bbc_parser.ex | 78 ++++++++++++++++++ lib/epochtalk_server_web/controllers/post.ex | 2 +- lib/epochtalk_server_web/json/post_json.ex | 85 +++++++++++++++++--- mix.exs | 1 + 6 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 lib/epochtalk_server/bbc_parser.ex diff --git a/config/runtime.exs b/config/runtime.exs index 492f30fd..45dd7bbf 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -407,6 +407,15 @@ end ##### PROXY REPO CONFIGURATIONS ##### +poolboy_config = [ + name: {:local, :bbc_parser}, + worker_module: EpochtalkServer.BBCParser, + size: 5, + max_overflow: 2, + strategy: :fifo +] +config :epochtalk_server, poolboy_config: poolboy_config + # conditionally show debug logs in prod if config_env() == :prod do logger_level = diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 4535c59e..13507d66 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -21,6 +21,8 @@ defmodule EpochtalkServer.Application do EpochtalkServer.Repo, # Start the Smf repository EpochtalkServer.SmfRepo, + # Start the BBC Parser + :poolboy.child_spec(:bbc_parser, poolboy_config()), # Start Role Cache EpochtalkServer.Cache.Role, # Warm frontend_config variable (referenced by api controllers) @@ -68,4 +70,6 @@ defmodule EpochtalkServer.Application do # fetch redix config defp redix_config(), do: Application.get_env(:epochtalk_server, :redix) + + defp poolboy_config, do: Application.get_env(:epochtalk_server, :poolboy_config) end diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex new file mode 100644 index 00000000..755c1e4a --- /dev/null +++ b/lib/epochtalk_server/bbc_parser.ex @@ -0,0 +1,78 @@ +defmodule EpochtalkServer.BBCParser do + use GenServer + require Logger + alias Porcelain.Process, as: Proc + + @moduledoc """ + `BBCParser` genserver, runs interactive php shell to call bbcode parser + """ + + ## === genserver functions ==== + + @impl true + def init(:ok), do: {:ok, load()} + + @impl true + def handle_call({:parse, bbcode_data}, _from, {proc, pid}) do + Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") + Logger.debug("FUCKFUCKFUCK PID: #{inspect(pid)}") + parsed = receive do + {^pid, :data, :out, data} -> data + end + + {:reply, parsed, {proc, pid}} + end + + ## === parser api functions ==== + + @doc """ + Start genserver and create a reference for supervision tree + """ + def start_link(_opts) do + GenServer.start_link(__MODULE__, :ok) + end + + @doc """ + Returns parsed bbcode input + """ + @spec parse(bbcodea_data :: any) :: String.t() + def parse(bbcode_data) do + GenServer.call(__MODULE__, {:parse, bbcode_data}) + end + + @doc """ + Uses poolboy to call parser + """ + def async_parse(bbcode_data) do + :poolboy.transaction( + :bbc_parser, + fn pid -> + # Let's wrap the genserver call in a try - catch block. This allows us to trap any exceptions + # that might be thrown and return the worker back to poolboy in a clean manner. It also allows + # the programmer to retrieve the error and potentially fix it. + try do + Logger.debug "#{__MODULE__}(ASYNC PARSE): #{inspect(pid)}" + GenServer.call(pid, {:parse, bbcode_data}, 10000) |> IO.inspect + catch + e, r -> IO.inspect("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") + :ok + end + end, + 50000 + ) + end + + ## === private functions ==== + + # returns loaded interactive php shell + defp load() do + proc = %Proc{pid: pid} = Porcelain.spawn_shell("php -a",in: :receive, out: {:send, self()}) + Proc.send_input(proc, "require 'parsing.php';\n") + Logger.debug "#{__MODULE__}(LOAD): #{inspect(pid)}" + # clear initial message + receive do + {^pid, :data, :out, data} -> Logger.debug "#{__MODULE__}: #{inspect(data)}" + end + {proc, pid} + end +end diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index e7e7cef1..05653d5f 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -566,7 +566,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do count: data.total_records, limit: data.per_page, page: data.page, - desc: true + desc: desc }) else _ -> diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index de5474d6..16807bfc 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -188,7 +188,10 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do limit: limit, page: page, desc: desc - }) do + }) when is_list(posts) do + posts = + posts + |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) %{ posts: posts, count: count, @@ -198,6 +201,21 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do } end + def proxy_by_username(%{ + posts: posts, + count: count, + limit: limit, + page: page, + desc: desc + }), + do: proxy_by_username(%{ + posts: [posts], + count: count, + limit: limit, + page: page, + desc: desc + }) + ## === Public Helper Functions === def handle_deleted_posts(posts, thread, user, authed_user_priority, view_deleted_posts) do @@ -424,24 +442,29 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do end defp format_proxy_post_data_for_by_thread(post) do - body = String.replace(post.body || post.body_html, "'", "\'") + body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") + + # %Porcelain.Result{out: parsed_body, status: _status} = + # Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - %Porcelain.Result{out: parsed_body, status: _status} = - Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + parsed_body = EpochtalkServer.BBCParser.async_parse(body) + + parsed_body = Enum.join(for <>, do: <>) + # IO.inspect parsed_body signature = - if post.user.signature, + if Map.get(post.user, :signature), do: String.replace(post.user.signature, "'", "\'"), else: nil parsed_signature = if signature do - %Porcelain.Result{out: parsed_sig, status: _status} = - Porcelain.shell( - "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"" - ) - - parsed_sig + # %Porcelain.Result{out: parsed_sig, status: _status} = + # Porcelain.shell( + # "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"" + # ) + # parsed_sig + EpochtalkServer.BBCParser.async_parse(signature) else nil end @@ -449,4 +472,44 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do user = post.user |> Map.put(:signature, parsed_signature) post |> Map.put(:body_html, parsed_body) |> Map.put(:user, user) end + + alias Porcelain.Process, as: Proc + alias Porcelain.Result + + def test() do + body = File.read!("./broken_ouput.txt") + + # %Porcelain.Result{out: parsed_body, status: _status} = + # Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + # parsed_body + + proc = %Proc{out: outstream} = Porcelain.spawn("php", ["-a"], [in: :receive, out: :stream]) + Proc.send_input(proc, "require 'parsing.php';\n") ; + Proc.send_input(proc, "echo parse_bbc('[b]hello[/b]');\n") + Proc.send_input(proc, "exit\n"); + Enum.into(proc.out, "") + end + + def test2() do + body = File.read!("./broken_ouput.txt") + + # %Porcelain.Result{out: parsed_body, status: _status} = + # Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") + # parsed_body + + + proc = %Proc{pid: pid} = + Porcelain.spawn_shell("php -a",in: :receive, out: {:send, self()}) + Proc.send_input(proc, "require 'parsing.php';\n") ; + Proc.send_input(proc, "echo parse_bbc('[b]hello[/b]');\n") + + receive do + {^pid, :data, :out, data} -> data + end + test = receive do + {^pid, :data, :out, data} -> data + end + IO.inspect test + IO.inspect test + end end diff --git a/mix.exs b/mix.exs index 7c61b22c..4586a4fe 100644 --- a/mix.exs +++ b/mix.exs @@ -65,6 +65,7 @@ defmodule EpochtalkServer.MixProject do {:phoenix_ecto, "~> 4.4"}, {:phoenix_html, "~> 3.0"}, {:plug_cowboy, "~> 2.5"}, + {:poolboy, "~> 1.5.1"}, {:porcelain, "~> 2.0"}, {:poison, "~> 3.0"}, {:postgrex, "~> 0.17.1"}, From b043e7a65299eee796994a4ce7bbd8f175b37b07 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 31 Oct 2024 16:50:18 -0700 Subject: [PATCH 129/231] fix(bbcode_parser): check for empty string before running bbcode parser --- lib/epochtalk_server/bbc_parser.ex | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 755c1e4a..33a8aaab 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -14,13 +14,17 @@ defmodule EpochtalkServer.BBCParser do @impl true def handle_call({:parse, bbcode_data}, _from, {proc, pid}) do - Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") - Logger.debug("FUCKFUCKFUCK PID: #{inspect(pid)}") - parsed = receive do - {^pid, :data, :out, data} -> data - end + if bbcode_data == "" do + {:reply, "", {proc, pid}} + else + Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") + Logger.debug("FUCKFUCKFUCK PID: #{inspect(pid)}") + parsed = receive do + {^pid, :data, :out, data} -> data + end - {:reply, parsed, {proc, pid}} + {:reply, parsed, {proc, pid}} + end end ## === parser api functions ==== From e271bf8fe81fa0f8fcd8089bdb01ac522fcaa83f Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 1 Nov 2024 11:46:07 -1000 Subject: [PATCH 130/231] feat(upgrade-elixir): upgrade to latest elixir --- .tool-versions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tool-versions b/.tool-versions index 0f420050..95179337 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -elixir 1.14.4-otp-25 -erlang 25.3.1 +elixir 1.17.3-otp-27 +erlang 27.1.2 php 8.3.7 From b3de4824947e5a5fe5301374c6536bd4b8ee60c2 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 1 Nov 2024 12:40:06 -1000 Subject: [PATCH 131/231] refactor(bbc-parser): clean up bbcode parser gen server --- lib/epochtalk_server/bbc_parser.ex | 41 ++++++++++-------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 33a8aaab..e6aecd66 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -2,6 +2,7 @@ defmodule EpochtalkServer.BBCParser do use GenServer require Logger alias Porcelain.Process, as: Proc + @timeout 10000 @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser @@ -13,18 +14,17 @@ defmodule EpochtalkServer.BBCParser do def init(:ok), do: {:ok, load()} @impl true - def handle_call({:parse, bbcode_data}, _from, {proc, pid}) do - if bbcode_data == "" do - {:reply, "", {proc, pid}} - else - Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") - Logger.debug("FUCKFUCKFUCK PID: #{inspect(pid)}") - parsed = receive do - {^pid, :data, :out, data} -> data - end + def handle_call({:parse, ""}, _from, {proc, pid}), + do: {:reply, "", {proc, pid}} - {:reply, parsed, {proc, pid}} + def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do + Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") + parsed = receive do + {^pid, :data, :out, data} -> + Logger.debug data + data end + {:reply, parsed, {proc, pid}} end ## === parser api functions ==== @@ -32,17 +32,7 @@ defmodule EpochtalkServer.BBCParser do @doc """ Start genserver and create a reference for supervision tree """ - def start_link(_opts) do - GenServer.start_link(__MODULE__, :ok) - end - - @doc """ - Returns parsed bbcode input - """ - @spec parse(bbcodea_data :: any) :: String.t() - def parse(bbcode_data) do - GenServer.call(__MODULE__, {:parse, bbcode_data}) - end + def start_link(_opts), do: GenServer.start_link(__MODULE__, :ok) @doc """ Uses poolboy to call parser @@ -51,18 +41,15 @@ defmodule EpochtalkServer.BBCParser do :poolboy.transaction( :bbc_parser, fn pid -> - # Let's wrap the genserver call in a try - catch block. This allows us to trap any exceptions - # that might be thrown and return the worker back to poolboy in a clean manner. It also allows - # the programmer to retrieve the error and potentially fix it. try do Logger.debug "#{__MODULE__}(ASYNC PARSE): #{inspect(pid)}" - GenServer.call(pid, {:parse, bbcode_data}, 10000) |> IO.inspect + GenServer.call(pid, {:parse, bbcode_data}, @timeout) catch e, r -> IO.inspect("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") :ok end end, - 50000 + @timeout ) end @@ -73,7 +60,7 @@ defmodule EpochtalkServer.BBCParser do proc = %Proc{pid: pid} = Porcelain.spawn_shell("php -a",in: :receive, out: {:send, self()}) Proc.send_input(proc, "require 'parsing.php';\n") Logger.debug "#{__MODULE__}(LOAD): #{inspect(pid)}" - # clear initial message + # clear initial php interactive shell message receive do {^pid, :data, :out, data} -> Logger.debug "#{__MODULE__}: #{inspect(data)}" end From e8bddb45b099aaae44760645bc3fa22e06b2a2d5 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 1 Nov 2024 12:43:45 -1000 Subject: [PATCH 132/231] refactor(general): update code syntax for latest elixir updatE --- .../controllers/image_reference.ex | 2 -- lib/epochtalk_server_web/controllers/post.ex | 3 --- lib/epochtalk_server_web/controllers/thread.ex | 6 ++---- lib/epochtalk_server_web/controllers/user.ex | 10 ++++------ lib/epochtalk_server_web/json/thread_json.ex | 2 +- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/image_reference.ex b/lib/epochtalk_server_web/controllers/image_reference.ex index 1f73917c..9a40520a 100644 --- a/lib/epochtalk_server_web/controllers/image_reference.ex +++ b/lib/epochtalk_server_web/controllers/image_reference.ex @@ -63,8 +63,6 @@ defmodule EpochtalkServerWeb.Controllers.ImageReference do # checksum <- Validate.cast(attrs, "checksum", :string, required: true), file_type <- Validate.cast(attrs, "file_type", :string, required: true) do %{length: length, type: file_type} - else - _ -> %{error: "Invalid attrs"} end end end diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 05653d5f..5673de0a 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -420,9 +420,6 @@ defmodule EpochtalkServerWeb.Controllers.Post do Validate.cast(attrs, "body", :string, required: true, max: post_max_length, min: 1), parsed_body <- Parse.markdown(body) do render(conn, :preview, %{parsed_body: parsed_body}) - else - _ -> - ErrorHelpers.render_json_error(conn, 400, "Error, cannot generate preview") end end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 0be741dd..89512b3c 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -38,8 +38,8 @@ defmodule EpochtalkServerWeb.Controllers.Thread do user_priority <- ACL.get_user_priority(conn), threads <- Thread.recent(user, user_priority) do render(conn, :recent, %{threads: threads}) - else - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch recent threads") + # else + # _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch recent threads") end end @@ -865,8 +865,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do defp proxy_recent(conn, _attrs) do with threads <- ProxyConversion.build_model("threads.recent") do render(conn, :recent, %{threads: threads}) - else - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch recent threads") end end diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index 4c00fced..36c1bf7f 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -97,10 +97,6 @@ defmodule EpochtalkServerWeb.Controllers.User do {:ok, _email} <- Mailer.send_confirm_account(user) do render(conn, :register_with_verify, user: user) else - # error in user.create - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - # error email failed to send {:error, :not_delivered} -> ErrorHelpers.render_json_error( @@ -109,6 +105,10 @@ defmodule EpochtalkServerWeb.Controllers.User do "Sending of account confirmation email failed, mailer is not properly configured." ) + # error in user.create + {:error, data} -> + ErrorHelpers.render_json_error(conn, 400, data) + # Catch all for any other errors _ -> ErrorHelpers.render_json_error(conn, 500, "There was an issue registering") @@ -290,8 +290,6 @@ defmodule EpochtalkServerWeb.Controllers.User do defp proxy_find(conn, attrs) do with user <- ProxyConversion.build_model("user.find", attrs["id"]) do render(conn, :find_proxy, %{user: user}) - else - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot find specified user") end end diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index f085a921..985ccf9d 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -295,7 +295,7 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do # handle deleted user thread = if thread.user_deleted, - do: thread |> Map.put(:user_id, '') |> Map.put(:username, ''), + do: thread |> Map.put(:user_id, "") |> Map.put(:username, ""), else: thread # format user output From 163ad432963297189688690c16a203edf4bba473 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 1 Nov 2024 12:47:02 -1000 Subject: [PATCH 133/231] style(format): run mix format --- config/runtime.exs | 1 + lib/epochtalk_server/bbc_parser.ex | 27 +++++++++------- .../controllers/mention.ex | 5 ++- lib/epochtalk_server_web/controllers/post.ex | 3 +- .../controllers/preference.ex | 9 ++++-- .../controllers/thread.ex | 5 ++- lib/epochtalk_server_web/controllers/user.ex | 1 - .../helpers/proxy_conversion.ex | 4 +-- .../helpers/proxy_pagination.ex | 24 ++++++++------ lib/epochtalk_server_web/json/post_json.ex | 31 ++++++++++--------- .../controllers/post_test.exs | 2 +- 11 files changed, 63 insertions(+), 49 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 45dd7bbf..187de652 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -414,6 +414,7 @@ poolboy_config = [ max_overflow: 2, strategy: :fifo ] + config :epochtalk_server, poolboy_config: poolboy_config # conditionally show debug logs in prod diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index e6aecd66..0009c5a8 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -19,11 +19,14 @@ defmodule EpochtalkServer.BBCParser do def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") - parsed = receive do - {^pid, :data, :out, data} -> - Logger.debug data - data - end + + parsed = + receive do + {^pid, :data, :out, data} -> + Logger.debug(data) + data + end + {:reply, parsed, {proc, pid}} end @@ -42,11 +45,12 @@ defmodule EpochtalkServer.BBCParser do :bbc_parser, fn pid -> try do - Logger.debug "#{__MODULE__}(ASYNC PARSE): #{inspect(pid)}" + Logger.debug("#{__MODULE__}(ASYNC PARSE): #{inspect(pid)}") GenServer.call(pid, {:parse, bbcode_data}, @timeout) catch - e, r -> IO.inspect("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") - :ok + e, r -> + IO.inspect("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") + :ok end end, @timeout @@ -57,13 +61,14 @@ defmodule EpochtalkServer.BBCParser do # returns loaded interactive php shell defp load() do - proc = %Proc{pid: pid} = Porcelain.spawn_shell("php -a",in: :receive, out: {:send, self()}) + proc = %Proc{pid: pid} = Porcelain.spawn_shell("php -a", in: :receive, out: {:send, self()}) Proc.send_input(proc, "require 'parsing.php';\n") - Logger.debug "#{__MODULE__}(LOAD): #{inspect(pid)}" + Logger.debug("#{__MODULE__}(LOAD): #{inspect(pid)}") # clear initial php interactive shell message receive do - {^pid, :data, :out, data} -> Logger.debug "#{__MODULE__}: #{inspect(data)}" + {^pid, :data, :out, data} -> Logger.debug("#{__MODULE__}: #{inspect(data)}") end + {proc, pid} end end diff --git a/lib/epochtalk_server_web/controllers/mention.ex b/lib/epochtalk_server_web/controllers/mention.ex index 6a683cdc..e3a25e19 100644 --- a/lib/epochtalk_server_web/controllers/mention.ex +++ b/lib/epochtalk_server_web/controllers/mention.ex @@ -27,8 +27,7 @@ defmodule EpochtalkServerWeb.Controllers.Mention do pagination_data: data, extended: extended }), - else: - ({:auth, nil} -> - ErrorHelpers.render_json_error(conn, 400, "Not logged in, cannot page mentions")) + else: ({:auth, nil} -> + ErrorHelpers.render_json_error(conn, 400, "Not logged in, cannot page mentions")) end end diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 5673de0a..6adaa4fd 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -557,7 +557,8 @@ defmodule EpochtalkServerWeb.Controllers.Post do page <- Validate.cast(attrs, "page", :integer, default: 1, min: 1), limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), - {:ok, posts, data} <- ProxyConversion.build_model("posts.by_user", user_id, page, limit, desc) do + {:ok, posts, data} <- + ProxyConversion.build_model("posts.by_user", user_id, page, limit, desc) do render(conn, :proxy_by_username, %{ posts: posts, count: data.total_records, diff --git a/lib/epochtalk_server_web/controllers/preference.ex b/lib/epochtalk_server_web/controllers/preference.ex index 0e7405a2..ea7e1564 100644 --- a/lib/epochtalk_server_web/controllers/preference.ex +++ b/lib/epochtalk_server_web/controllers/preference.ex @@ -14,8 +14,11 @@ defmodule EpochtalkServerWeb.Controllers.Preference do def preferences(conn, _attrs) do with {:auth, %{} = user} <- {:auth, Guardian.Plug.current_resource(conn)}, do: render(conn, :preferences, preferences: Preference.by_user_id(user.id)), - else: - ({:auth, nil} -> - ErrorHelpers.render_json_error(conn, 400, "Not logged in, cannot fetch preferences")) + else: ({:auth, nil} -> + ErrorHelpers.render_json_error( + conn, + 400, + "Not logged in, cannot fetch preferences" + )) end end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 89512b3c..21ff713f 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -38,8 +38,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do user_priority <- ACL.get_user_priority(conn), threads <- Thread.recent(user, user_priority) do render(conn, :recent, %{threads: threads}) - # else - # _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot fetch recent threads") end end @@ -819,7 +817,8 @@ defmodule EpochtalkServerWeb.Controllers.Thread do page <- Validate.cast(attrs, "page", :integer, default: 1, min: 1), limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), - {:ok, threads, data} <- ProxyConversion.build_model("threads.by_user", user_id, page, limit, desc) do + {:ok, threads, data} <- + ProxyConversion.build_model("threads.by_user", user_id, page, limit, desc) do render(conn, :proxy_by_username, %{ threads: threads, next: data.next, diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index 36c1bf7f..aa8297a6 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -292,5 +292,4 @@ defmodule EpochtalkServerWeb.Controllers.User do render(conn, :find_proxy, %{user: user}) end end - end diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index a2e3bcb3..ce96b0ca 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -502,7 +502,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do updated_at: m.modifiedTime, created_at: m.posterTime * 1000, user: %{ - id: m.id_member, + id: m.id_member } }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: desc) @@ -533,7 +533,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do board_id: t.id_board, sticky: t.isSticky, locked: t.locked, - user: %{id: t.id_member_started, deleted: false }, + user: %{id: t.id_member_started, deleted: false}, moderated: t.selfModerated, post_count: t.numReplies, thread_title: f.subject, diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index 4e819ed9..5e4ca733 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -45,27 +45,31 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do def page_simple(query, count_query, page, per_page: nil, desc: desc) when is_integer(page), do: page_simple(query, count_query, page, per_page: 15, desc: desc) - def page_simple(query, count_query, nil, per_page: per_page, desc: desc) when is_integer(per_page), - do: page_simple(query, count_query, 1, per_page: per_page, desc: desc) + def page_simple(query, count_query, nil, per_page: per_page, desc: desc) + when is_integer(per_page), + do: page_simple(query, count_query, 1, per_page: per_page, desc: desc) - def page_simple(query, count_query, nil, per_page: per_page, desc: desc) when is_binary(per_page), - do: - page_simple(query, count_query, 1, - per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1), - desc: desc - ) + def page_simple(query, count_query, nil, per_page: per_page, desc: desc) + when is_binary(per_page), + do: + page_simple(query, count_query, 1, + per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1), + desc: desc + ) def page_simple(query, count_query, page, per_page: nil, desc: desc) when is_binary(page), do: page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), - per_page: 15, desc: desc + per_page: 15, + desc: desc ) def page_simple(query, count_query, page, per_page: per_page, desc: desc) when is_binary(page) and is_binary(per_page), do: page_simple(query, count_query, Validate.cast_str(page, :integer, key: "page", min: 1), - per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1), desc: desc + per_page: Validate.cast_str(per_page, :integer, key: "limit", min: 1), + desc: desc ) def page_simple(query, count_query, page, per_page: per_page, desc: desc) do diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 16807bfc..a2b44261 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -188,17 +188,19 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do limit: limit, page: page, desc: desc - }) when is_list(posts) do + }) + when is_list(posts) do posts = posts |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) + %{ - posts: posts, - count: count, - limit: limit, - page: page, - desc: desc - } + posts: posts, + count: count, + limit: limit, + page: page, + desc: desc + } end def proxy_by_username(%{ @@ -208,13 +210,14 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do page: page, desc: desc }), - do: proxy_by_username(%{ - posts: [posts], - count: count, - limit: limit, - page: page, - desc: desc - }) + do: + proxy_by_username(%{ + posts: [posts], + count: count, + limit: limit, + page: page, + desc: desc + }) ## === Public Helper Functions === diff --git a/test/epochtalk_server_web/controllers/post_test.exs b/test/epochtalk_server_web/controllers/post_test.exs index 77ee765b..7096fc93 100644 --- a/test/epochtalk_server_web/controllers/post_test.exs +++ b/test/epochtalk_server_web/controllers/post_test.exs @@ -42,7 +42,7 @@ defmodule Test.EpochtalkServerWeb.Controllers.Post do fn -> post(conn, Routes.post_path(conn, :preview), %{ "body" => - for(_ <- 1..10_001, into: "", do: <>) + for(_ <- 1..10_001, into: "", do: <>) }) end end From 3d18378736ee5a3c50158e2dc6c0a7d8338b0d60 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 09:47:45 -1000 Subject: [PATCH 134/231] fix(user-find): bring back missing user avatar --- .../helpers/proxy_conversion.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ce96b0ca..e53abc78 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -84,9 +84,11 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do def build_user(user_id) do from(u in "smf_members", where: u.id_member == ^user_id) - |> select([u], %{ + |> join(:left, [u], a in "smf_attachments", + on: u.id_member == a.id_member and a.attachmentType == 1 + ) + |> select([u, a], %{ activity: u.activity, - avatar: u.avatar, created_at: u.dateRegistered * 1000, dob: u.birthdate, gender: u.gender, @@ -102,7 +104,14 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do name: u.realName, username: u.realName, title: u.usertitle, - website: u.websiteUrl + website: u.websiteUrl, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ) }) |> SmfRepo.one() end From 9f94b854d48e049d9a819e4a82e662cbb8d40d89 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 09:49:44 -1000 Subject: [PATCH 135/231] fix(last-post-user-id): bring back last post user id for linking to user profile in front end --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 2 +- lib/epochtalk_server_web/json/thread_json.ex | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index e53abc78..c5d19db0 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -333,7 +333,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do first_post_id: t.id_first_msg, last_post_id: t.id_last_msg, started_user_id: t.id_member_started, - updated_user_id: t.id_member_updated, + last_post_user_id: t.id_member_updated, moderated: t.selfModerated, post_count: t.numReplies, title: f.subject, diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index 985ccf9d..e93870a9 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -363,6 +363,5 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do thread |> Map.delete(:last_post_deleted) |> Map.delete(:last_post_user_deleted) - |> Map.delete(:last_post_user_id) end end From e282db0470c6126bedcd7bad5d4d9b53cb0f91ae Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 09:52:35 -1000 Subject: [PATCH 136/231] fix(proxy-pagination): fix for prev/next style pagination when there is only one record --- .../helpers/proxy_pagination.ex | 10 ++++++++-- lib/epochtalk_server_web/json/thread_json.ex | 15 +++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index 5e4ca733..7a1e7820 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -101,8 +101,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do # query one more page to calculate if next page exists result = records(query, page, nil, per_page + 1) + next = length(result) > per_page + pagination_data = %{ - next: length(result) == per_page + 1, + next: next, prev: page > 1, page: page, per_page: per_page, @@ -110,7 +112,11 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do } # remove extra element - result = result |> Enum.reverse() |> tl() |> Enum.reverse() + result = + if next, + do: result |> Enum.reverse() |> tl() |> Enum.reverse(), + else: result + {:ok, result, pagination_data} end diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index e93870a9..6aec1a2a 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -207,16 +207,15 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do desc: desc }) do %{ - posts: threads, - next: next, - prev: prev, - limit: limit, - page: page, - desc: desc - } + posts: threads, + next: next, + prev: prev, + limit: limit, + page: page, + desc: desc + } end - @doc """ Renders sticky `Thread`. From 144b3a4e8cdffd0bc4b715e4306e7b538ebcd024 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 09:55:44 -1000 Subject: [PATCH 137/231] feat(parser): user async parser for signature, code cleanup --- lib/epochtalk_server_web/json/post_json.ex | 59 ++-------------------- 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index a2b44261..6e739165 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -447,72 +447,19 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") - # %Porcelain.Result{out: parsed_body, status: _status} = - # Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - parsed_body = EpochtalkServer.BBCParser.async_parse(body) - parsed_body = Enum.join(for <>, do: <>) - # IO.inspect parsed_body - signature = if Map.get(post.user, :signature), do: String.replace(post.user.signature, "'", "\'"), else: nil parsed_signature = - if signature do - # %Porcelain.Result{out: parsed_sig, status: _status} = - # Porcelain.shell( - # "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> signature <> "');\"" - # ) - # parsed_sig - EpochtalkServer.BBCParser.async_parse(signature) - else - nil - end + if signature, + do: EpochtalkServer.BBCParser.async_parse(signature), + else: nil user = post.user |> Map.put(:signature, parsed_signature) post |> Map.put(:body_html, parsed_body) |> Map.put(:user, user) end - - alias Porcelain.Process, as: Proc - alias Porcelain.Result - - def test() do - body = File.read!("./broken_ouput.txt") - - # %Porcelain.Result{out: parsed_body, status: _status} = - # Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - # parsed_body - - proc = %Proc{out: outstream} = Porcelain.spawn("php", ["-a"], [in: :receive, out: :stream]) - Proc.send_input(proc, "require 'parsing.php';\n") ; - Proc.send_input(proc, "echo parse_bbc('[b]hello[/b]');\n") - Proc.send_input(proc, "exit\n"); - Enum.into(proc.out, "") - end - - def test2() do - body = File.read!("./broken_ouput.txt") - - # %Porcelain.Result{out: parsed_body, status: _status} = - # Porcelain.shell("php -r \"require 'parsing.php'; ECHO parse_bbc('" <> body <> "');\"") - # parsed_body - - - proc = %Proc{pid: pid} = - Porcelain.spawn_shell("php -a",in: :receive, out: {:send, self()}) - Proc.send_input(proc, "require 'parsing.php';\n") ; - Proc.send_input(proc, "echo parse_bbc('[b]hello[/b]');\n") - - receive do - {^pid, :data, :out, data} -> data - end - test = receive do - {^pid, :data, :out, data} -> data - end - IO.inspect test - IO.inspect test - end end From 4a8009789cdac1ccd5093e5d446310fbbf8098e6 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 09:56:07 -1000 Subject: [PATCH 138/231] fix(parser): add missing settings that were causing portuguese characters to break --- parsing_extra.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parsing_extra.php b/parsing_extra.php index aa17088a..2491ab44 100644 --- a/parsing_extra.php +++ b/parsing_extra.php @@ -15,6 +15,10 @@ function setReasonableValues() $context['browser']['is_ie5'] = false; $context['browser']['is_ie5.5'] = false; + // Fix for portuguese characters (test: http://localhost:8000/threads/5515847) + $context['utf8'] = true; + $context['server']['complex_preg_chars'] = true; + $txt['lang_character_set'] = 'ISO-8859-1'; $txt['smf238'] = 'Code'; $txt['smf240'] = 'Quote'; From c0cb6db77b26446122d911ebfdedac29af4061e6 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 10:08:35 -1000 Subject: [PATCH 139/231] feat(credo): upgrade credo to latest --- mix.exs | 2 +- mix.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 4586a4fe..cd773499 100644 --- a/mix.exs +++ b/mix.exs @@ -36,7 +36,7 @@ defmodule EpochtalkServer.MixProject do {:argon2_elixir, "~> 3.1.0"}, {:configparser_ex, "~> 4.0"}, {:corsica, "~> 1.3.0"}, - {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.7.9", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.2", only: [:dev], runtime: false}, {:dotenv_parser, "~> 2.0"}, {:earmark, "~> 1.4"}, diff --git a/mix.lock b/mix.lock index fb3eb478..78ba78d6 100644 --- a/mix.lock +++ b/mix.lock @@ -10,7 +10,7 @@ "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, - "credo": {:hex, :credo, "1.7.6", "b8f14011a5443f2839b04def0b252300842ce7388f3af177157c86da18dfbeea", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "146f347fb9f8cbc5f7e39e3f22f70acbef51d441baa6d10169dd604bfbc55296"}, + "credo": {:hex, :credo, "1.7.9", "07bb31907746ae2b5e569197c9e16c0d75c8578a22f01bee63f212047efb2647", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f87c11c34ba579f7c5044f02b2a807e1ed2fa5fdbb24dc7eb4ad59c1904887f3"}, "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, @@ -26,7 +26,7 @@ "ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"}, "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, "ex_utils": {:hex, :ex_utils, "0.1.7", "2c133e0bcdc49a858cf8dacf893308ebc05bc5fba501dc3d2935e65365ec0bf3", [:mix], [], "hexpm", "66d4fe75285948f2d1e69c2a5ddd651c398c813574f8d36a9eef11dc20356ef6"}, - "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, + "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, "gen_smtp": {:hex, :gen_smtp, "1.2.0", "9cfc75c72a8821588b9b9fe947ae5ab2aed95a052b81237e0928633a13276fd3", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "5ee0375680bca8f20c4d85f58c2894441443a743355430ff33a783fe03296779"}, "guardian": {:hex, :guardian, "2.3.2", "78003504b987f2b189d76ccf9496ceaa6a454bb2763627702233f31eb7212881", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "b189ff38cd46a22a8a824866a6867ca8722942347f13c33f7d23126af8821b52"}, @@ -41,7 +41,7 @@ "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.3", "67b3d9fa8691b727317e0cc96b9b3093be00ee45419ffb221cdeee88e75d1360", [:mix], [{:mochiweb, "~> 2.15 or ~> 3.1", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "87748d3c4afe949c7c6eb7150c958c2bcba43fc5b2a02686af30e636b74bccb7"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "iteraptor": {:git, "https://github.com/epochtalk/elixir-iteraptor.git", "d8d1c386c38e06bdfcf60c9ce1abf8e49161cab4", [tag: "1.13.1"]}, - "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, From 7a41e9a5a652f9c2fc3450ccfb228b3587e47948 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Mon, 4 Nov 2024 10:31:34 -1000 Subject: [PATCH 140/231] fix(credo): resolve credo issues in bbc parser --- lib/epochtalk_server/bbc_parser.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 0009c5a8..d9ec3333 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -2,7 +2,7 @@ defmodule EpochtalkServer.BBCParser do use GenServer require Logger alias Porcelain.Process, as: Proc - @timeout 10000 + @timeout 10_000 @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser @@ -49,7 +49,7 @@ defmodule EpochtalkServer.BBCParser do GenServer.call(pid, {:parse, bbcode_data}, @timeout) catch e, r -> - IO.inspect("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") + Logger.debug("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") :ok end end, From f618c325b585ffd295b69d273c6d0154947154bf Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 5 Nov 2024 10:50:57 -1000 Subject: [PATCH 141/231] feat(last-active): bring back last post date for last active in user find --- .../helpers/proxy_conversion.ex | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index c5d19db0..c7e4226b 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -83,7 +83,14 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_user(user_id) do - from(u in "smf_members", where: u.id_member == ^user_id) + last_active = + from m in "smf_messages", + where: m.id_member == ^user_id, + order_by: [desc: m.posterTime], + limit: 1, + select: %{last_active: m.posterTime * 1000} + + user = from(u in "smf_members", where: u.id_member == ^user_id) |> join(:left, [u], a in "smf_attachments", on: u.id_member == a.id_member and a.attachmentType == 1 ) @@ -94,7 +101,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do gender: u.gender, id: u.id_member, language: nil, - last_active: nil, location: u.location, merit: u.merit, id_group: u.id_group, @@ -114,6 +120,10 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do ) }) |> SmfRepo.one() + + if user.post_count > 0, + do: Map.merge(user, SmfRepo.one(last_active)), + else: Map.put(user, :last_active, user.created_at) end def build_poll(thread_id) do From 9bace91e3b09ccf7f6f25fcde7719dd33f8bac95 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 7 Nov 2024 11:08:15 -1000 Subject: [PATCH 142/231] refactor(cleanup): use BBCParser Gen Server for parsing user signature, run format --- .../helpers/proxy_conversion.ex | 68 +++++++++---------- lib/epochtalk_server_web/json/user_json.ex | 13 +--- 2 files changed, 34 insertions(+), 47 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index c7e4226b..59bea5bd 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -90,36 +90,37 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do limit: 1, select: %{last_active: m.posterTime * 1000} - user = from(u in "smf_members", where: u.id_member == ^user_id) - |> join(:left, [u], a in "smf_attachments", - on: u.id_member == a.id_member and a.attachmentType == 1 - ) - |> select([u, a], %{ - activity: u.activity, - created_at: u.dateRegistered * 1000, - dob: u.birthdate, - gender: u.gender, - id: u.id_member, - language: nil, - location: u.location, - merit: u.merit, - id_group: u.id_group, - id_post_group: u.id_post_group, - signature: u.signature, - post_count: u.posts, - name: u.realName, - username: u.realName, - title: u.usertitle, - website: u.websiteUrl, - avatar: - fragment( - "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", - u.avatar, - u.avatar, - a.filename - ) - }) - |> SmfRepo.one() + user = + from(u in "smf_members", where: u.id_member == ^user_id) + |> join(:left, [u], a in "smf_attachments", + on: u.id_member == a.id_member and a.attachmentType == 1 + ) + |> select([u, a], %{ + activity: u.activity, + created_at: u.dateRegistered * 1000, + dob: u.birthdate, + gender: u.gender, + id: u.id_member, + language: nil, + location: u.location, + merit: u.merit, + id_group: u.id_group, + id_post_group: u.id_post_group, + signature: u.signature, + post_count: u.posts, + name: u.realName, + username: u.realName, + title: u.usertitle, + website: u.websiteUrl, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ) + }) + |> SmfRepo.one() if user.post_count > 0, do: Map.merge(user, SmfRepo.one(last_active)), @@ -525,13 +526,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do } }) |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: desc) - |> case do - {:ok, [], _} -> - {:error, "Posts not found for user_id: #{id}"} - - {:ok, posts, data} -> - return_tuple(posts, data) - end end def build_threads_by_user(id, page, per_page, desc) do diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index f09d23d1..b9639831 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -18,16 +18,9 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do """ def find_proxy(%{user: user}) do parsed_signature = - if user.signature do - %Porcelain.Result{out: parsed_sig, status: _status} = - Porcelain.shell( - "php -r \"require 'parsing.php'; ECHO parse_bbc('" <> user.signature <> "');\"" - ) - - parsed_sig - else - nil - end + if user.signature, + do: EpochtalkServer.BBCParser.async_parse(user.signature), + else: nil user |> Map.put(:signature, parsed_signature) end From 8af4e0a8da7b778b1148452e02e75222c87d6552 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 7 Nov 2024 13:19:47 -1000 Subject: [PATCH 143/231] feat(gender): calculate gender for user find proxy --- lib/epochtalk_server_web/json/user_json.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index b9639831..4392ed03 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -22,7 +22,15 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do do: EpochtalkServer.BBCParser.async_parse(user.signature), else: nil - user |> Map.put(:signature, parsed_signature) + gender = case Map.get(user, :gender) do + 1 -> "Male" + 2 -> "Female" + _ -> nil + end + + user + |> Map.put(:signature, parsed_signature) + |> Map.put(:gender, gender) end @doc """ From 94157d7fca0d892da7add65d3c61e98787c3567c Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 7 Nov 2024 13:20:02 -1000 Subject: [PATCH 144/231] feat(dob): calculate dob for user find proxy --- lib/epochtalk_server_web/json/user_json.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 4392ed03..2e8b0f0e 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -28,9 +28,15 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do _ -> nil end + dob = case d = Map.get(user, :dob) do + ~D[0001-01-01] -> nil + _ -> d + end + user |> Map.put(:signature, parsed_signature) |> Map.put(:gender, gender) + |> Map.put(:dob, dob) end @doc """ From ea39654c1da8fb9ca19a99276652ab8b0a55f0da Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 11 Nov 2024 14:51:58 -0800 Subject: [PATCH 145/231] feat(thread-view): load metadata for child boards --- lib/epochtalk_server_web/controllers/thread.ex | 6 +++++- lib/epochtalk_server_web/json/board_json.ex | 16 ++++++++++++++-- lib/epochtalk_server_web/json/thread_json.ex | 8 ++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 909a7e06..23ead448 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -815,6 +815,8 @@ defmodule EpochtalkServerWeb.Controllers.Thread do :ok <- ACL.allow!(conn, "threads.byBoard"), board_mapping <- BoardMapping.all(), board_moderators <- BoardModerator.all(), + {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), + {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), {:ok, threads, data} <- ProxyConversion.build_model("threads.by_board", board_id, page, limit) do render(conn, :by_board_proxy, %{ @@ -826,7 +828,9 @@ defmodule EpochtalkServerWeb.Controllers.Thread do board_moderators: board_moderators, page: page, limit: limit, - pagination_data: data + pagination_data: data, + board_counts: board_counts, + board_last_post_info: board_last_post_info }) else _ -> diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 75ed8f30..8fa31c60 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -126,7 +126,16 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do @doc """ Board view helper method for mapping childboards and other metadata to board using board mapping and user priority """ - def format_board_data_for_find(board_moderators, board_mapping, board_id, user_priority) do + def format_board_data_for_find( + board_moderators, + board_mapping, + board_id, + user_priority, + board_counts \\ nil, + board_last_post_info \\ nil + ) do + board_counts = map_to_id(board_counts) + board_last_post_info = map_to_id(board_last_post_info) # filter out board by id [board] = Enum.filter(board_mapping, fn bm -> bm.board_id == board_id end) @@ -173,7 +182,10 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board_mapping, :children, board, - user_priority + user_priority, + # board counts and last post info for proxy version + board_counts, + board_last_post_info ) # return flattened board data with children, mod and last post data diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index feb2b94c..b5aff2e1 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -136,7 +136,9 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do board_mapping: board_mapping, board_moderators: board_moderators, page: page, - limit: limit + limit: limit, + board_counts: board_counts, + board_last_post_info: board_last_post_info }) do # format board data board = @@ -144,7 +146,9 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do board_moderators, board_mapping, board_id, - user_priority + user_priority, + board_counts, + board_last_post_info ) # format thread data From a0948d99dd9843b2c9cae33bc98a789cb43a0216 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 11 Nov 2024 14:59:31 -0800 Subject: [PATCH 146/231] fix(thread-view): re-add original format_board_data_for_find change thread_json to use proxy_format_board_data_for_find in the case when metadata is not available, we need to use the original format_board_data_for_find (such as in tests) --- lib/epochtalk_server_web/json/board_json.ex | 60 +++++++++++++++++++- lib/epochtalk_server_web/json/thread_json.ex | 2 +- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 8fa31c60..22a570da 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -124,9 +124,10 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do def movelist(%{movelist: movelist}), do: movelist @doc """ + Proxy version Board view helper method for mapping childboards and other metadata to board using board mapping and user priority """ - def format_board_data_for_find( + def proxy_format_board_data_for_find( board_moderators, board_mapping, board_id, @@ -192,6 +193,63 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do board end + @doc """ + Board view helper method for mapping childboards and other metadata to board using board mapping and user priority + """ + def format_board_data_for_find(board_moderators, board_mapping, board_id, user_priority) do + # filter out board by id + [board] = Enum.filter(board_mapping, fn bm -> bm.board_id == board_id end) + + # append board moderators to board + moderators = + board_moderators + |> Enum.filter(fn mod -> Map.get(mod, :board_id) == Map.get(board, :board_id) end) + |> Enum.map(fn mod -> %{id: mod.user_id, username: mod.user.username} end) + + board = Map.put(board, :moderators, moderators) + + # flatten needed boards data + board = + board + |> Map.merge(remove_nil(board.board)) + |> Map.merge( + remove_nil(board.stats) + |> Map.delete(:id) + ) + |> Map.merge(board.thread) + |> Map.merge(board.board.meta || board.stats) + |> Map.delete(:meta) + + # delete unneeded properties + board = + board + |> Map.delete(:board) + |> Map.delete(:stats) + |> Map.delete(:thread) + |> Map.delete(:parent) + |> Map.delete(:category) + |> Map.delete(:__meta__) + |> Map.delete(:__struct__) + + # handle deleted last post data + if !!Map.get(board, :post_deleted) or !!Map.get(board, :user_deleted), + do: board |> Map.put(:last_post_username, "deleted"), + else: board + + # iterate each child board, attempt to map nested children from board mapping + board = + process_children_from_board_mapping( + :parent_id, + board_mapping, + :children, + board, + user_priority + ) + + # return flattened board data with children, mod and last post data + board + end + defp process_children_from_board_mapping( board_mapping_key, board_mapping, diff --git a/lib/epochtalk_server_web/json/thread_json.ex b/lib/epochtalk_server_web/json/thread_json.ex index b5aff2e1..6ab0e987 100644 --- a/lib/epochtalk_server_web/json/thread_json.ex +++ b/lib/epochtalk_server_web/json/thread_json.ex @@ -142,7 +142,7 @@ defmodule EpochtalkServerWeb.Controllers.ThreadJSON do }) do # format board data board = - BoardJSON.format_board_data_for_find( + BoardJSON.proxy_format_board_data_for_find( board_moderators, board_mapping, board_id, From d9213e691be4b2117397466388176bb1c6e2b748 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 11 Nov 2024 22:04:29 -0800 Subject: [PATCH 147/231] style(board_json): mix format --- lib/epochtalk_server_web/json/board_json.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/epochtalk_server_web/json/board_json.ex b/lib/epochtalk_server_web/json/board_json.ex index 22a570da..b92f52d9 100644 --- a/lib/epochtalk_server_web/json/board_json.ex +++ b/lib/epochtalk_server_web/json/board_json.ex @@ -128,13 +128,13 @@ defmodule EpochtalkServerWeb.Controllers.BoardJSON do Board view helper method for mapping childboards and other metadata to board using board mapping and user priority """ def proxy_format_board_data_for_find( - board_moderators, - board_mapping, - board_id, - user_priority, - board_counts \\ nil, - board_last_post_info \\ nil - ) do + board_moderators, + board_mapping, + board_id, + user_priority, + board_counts \\ nil, + board_last_post_info \\ nil + ) do board_counts = map_to_id(board_counts) board_last_post_info = map_to_id(board_last_post_info) # filter out board by id From a5af54956bb7c00bb84610354c710943d2023ba8 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 12 Nov 2024 10:30:50 -1000 Subject: [PATCH 148/231] test(static): resolve dialyzer errors after upgrade of elixir/erlang --- lib/epochtalk_server_web/controllers/board.ex | 3 -- .../controllers/moderation_log.ex | 7 +--- .../controllers/notification.ex | 33 ++++--------------- lib/epochtalk_server_web/controllers/poll.ex | 14 +------- .../controllers/thread.ex | 20 +---------- lib/epochtalk_server_web/controllers/user.ex | 26 +++------------ .../helpers/proxy_pagination.ex | 13 +++++--- lib/epochtalk_server_web/router.ex | 2 +- .../controllers/user_test.exs | 6 ++-- 9 files changed, 25 insertions(+), 99 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 0106de91..44445543 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -103,9 +103,6 @@ defmodule EpochtalkServerWeb.Controllers.Board do 400, "Error, cannot convert slug: board does not exist" ) - - _ -> - ErrorHelpers.render_json_error(conn, 400, "Error, cannot convert board slug to id") end end diff --git a/lib/epochtalk_server_web/controllers/moderation_log.ex b/lib/epochtalk_server_web/controllers/moderation_log.ex index 888f8990..fb032aca 100644 --- a/lib/epochtalk_server_web/controllers/moderation_log.ex +++ b/lib/epochtalk_server_web/controllers/moderation_log.ex @@ -4,7 +4,6 @@ defmodule EpochtalkServerWeb.Controllers.ModerationLog do @moduledoc """ Controller For `ModerationLog` related API requests """ - alias EpochtalkServer.Auth.Guardian alias EpochtalkServer.Models.ModerationLog alias EpochtalkServerWeb.ErrorHelpers alias EpochtalkServerWeb.Helpers.Validate @@ -14,16 +13,12 @@ defmodule EpochtalkServerWeb.Controllers.ModerationLog do Used to page `ModerationLog` models for moderation log view` """ def page(conn, attrs) do - with {:auth, true} <- {:auth, Guardian.Plug.authenticated?(conn)}, - :ok <- ACL.allow!(conn, "moderationLogs.page"), + with :ok <- ACL.allow!(conn, "moderationLogs.page"), page <- Validate.cast(attrs, "page", :integer, min: 1), limit <- Validate.cast(attrs, "limit", :integer, min: 1), {:ok, moderation_logs, data} <- ModerationLog.page(attrs, page, per_page: limit) do render(conn, :page, %{moderation_logs: moderation_logs, pagination_data: data}) else - {:auth, false} -> - ErrorHelpers.render_json_error(conn, 400, "Not logged in, cannot page moderation log") - {:error, data} -> ErrorHelpers.render_json_error(conn, 400, data) diff --git a/lib/epochtalk_server_web/controllers/notification.ex b/lib/epochtalk_server_web/controllers/notification.ex index 8a9ffbdf..7e05df9e 100644 --- a/lib/epochtalk_server_web/controllers/notification.ex +++ b/lib/epochtalk_server_web/controllers/notification.ex @@ -14,23 +14,16 @@ defmodule EpochtalkServerWeb.Controllers.Notification do Used to retrieve `Notification` counts for a specific `User` """ def counts(conn, attrs) do - with {:auth, %{} = user} <- {:auth, Guardian.Plug.current_resource(conn)}, + with user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "notifications.counts"), max <- Validate.cast(attrs, "max", :integer, min: 1) do render(conn, :counts, data: Notification.counts_by_user_id(user.id, max: max || 99)) else - {:auth, nil} -> - ErrorHelpers.render_json_error( - conn, - 400, - "Not logged in, cannot fetch notification counts" - ) - - {:access, false} -> + _ -> ErrorHelpers.render_json_error( conn, - 400, - "Not logged in, cannot fetch notification counts" + 500, + "Something went wrong, cannot fetch notification counts" ) end end @@ -39,19 +32,12 @@ defmodule EpochtalkServerWeb.Controllers.Notification do Used to dismiss `Notification` counts for a specific `User` """ def dismiss(conn, %{"id" => id}) do - with {:auth, %{} = user} <- {:auth, Guardian.Plug.current_resource(conn)}, + with user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "notifications.dismiss"), {_count, nil} <- Notification.dismiss(id) do EpochtalkServerWeb.Endpoint.broadcast("user:#{user.id}", "refreshMentions", %{}) render(conn, :dismiss, success: true) else - {:auth, nil} -> - ErrorHelpers.render_json_error( - conn, - 400, - "Not logged in, cannot dismiss notification counts" - ) - _ -> ErrorHelpers.render_json_error( conn, @@ -62,19 +48,12 @@ defmodule EpochtalkServerWeb.Controllers.Notification do end def dismiss(conn, %{"type" => type}) do - with {:auth, %{} = user} <- {:auth, Guardian.Plug.current_resource(conn)}, + with user <- Guardian.Plug.current_resource(conn), :ok <- ACL.allow!(conn, "notifications.dismiss"), {_count, nil} <- Notification.dismiss_type_by_user_id(user.id, type) do EpochtalkServerWeb.Endpoint.broadcast("user:#{user.id}", "refreshMentions", %{}) render(conn, :dismiss, success: true) else - {:auth, nil} -> - ErrorHelpers.render_json_error( - conn, - 400, - "Not logged in, cannot dismiss notification counts" - ) - {:error, :invalid_notification_type} -> ErrorHelpers.render_json_error(conn, 400, "Cannot dismiss, invalid notification type") diff --git a/lib/epochtalk_server_web/controllers/poll.ex b/lib/epochtalk_server_web/controllers/poll.ex index 7fea5cc5..a8afa708 100644 --- a/lib/epochtalk_server_web/controllers/poll.ex +++ b/lib/epochtalk_server_web/controllers/poll.ex @@ -193,9 +193,6 @@ defmodule EpochtalkServerWeb.Controllers.Poll do "Account must be active to modify lock on poll" ) - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot lock poll") end @@ -316,9 +313,6 @@ defmodule EpochtalkServerWeb.Controllers.Poll do poll <- Poll.by_thread(thread_id) do render(conn, :poll, %{poll: poll, has_voted: false}) else - {:valid_answers_list, false} -> - ErrorHelpers.render_json_error(conn, 400, "Error, 'answer_ids' must be a list") - {:can_read, {:ok, false}} -> ErrorHelpers.render_json_error( conn, @@ -355,14 +349,8 @@ defmodule EpochtalkServerWeb.Controllers.Poll do {:board_banned, {:ok, true}} -> ErrorHelpers.render_json_error(conn, 403, "Unauthorized, you are banned from this board") - {:error, :board_does_not_exist} -> - ErrorHelpers.render_json_error(conn, 400, "Error, board does not exist") - - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - _ -> - ErrorHelpers.render_json_error(conn, 400, "Error, cannot cast vote") + ErrorHelpers.render_json_error(conn, 400, "Error, cannot delete vote") end end diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 21ff713f..b5cc091a 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -215,14 +215,8 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:can_read, {:error, :board_does_not_exist}} -> ErrorHelpers.render_json_error(conn, 400, "Read error, board does not exist") - {:board_banned, {:ok, true}} -> - ErrorHelpers.render_json_error(conn, 403, "Unauthorized, you are banned from this board") - {:has_threads, false} -> ErrorHelpers.render_json_error(conn, 404, "Error, requested threads not found in board") - - _ -> - ErrorHelpers.render_json_error(conn, 400, "Error, cannot get threads by board") end end @@ -341,9 +335,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do "Account must be active to unwatch thread" ) - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot unwatch thread") end @@ -402,9 +393,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do "Account must be active to modify lock on thread" ) - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot lock thread") end @@ -463,9 +451,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do "Account must be active to modify sticky on thread" ) - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot sticky thread") end @@ -639,9 +624,6 @@ defmodule EpochtalkServerWeb.Controllers.Thread do 400, "Error, cannot convert slug, thread does not exist" ) - - _ -> - ErrorHelpers.render_json_error(conn, 400, "Error, cannot convert thread slug to id") end end @@ -664,7 +646,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do |> send_resp(200, []) |> halt() else - {:error, :board_does_not_exist} -> + {:can_read, {:error, :board_does_not_exist}} -> ErrorHelpers.render_json_error( conn, 400, diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index aa8297a6..184d9609 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -142,12 +142,6 @@ defmodule EpochtalkServerWeb.Controllers.User do 500, "There was an error banning malicious user, upon confirming account" ) - - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - - _ -> - ErrorHelpers.render_json_error(conn, 500, "There was an issue registering") end end @@ -182,17 +176,11 @@ defmodule EpochtalkServerWeb.Controllers.User do show_hidden: show_hidden }) else - {:error, :user_not_found} -> - ErrorHelpers.render_json_error(conn, 400, "Account not found") - - {:error, data} -> - ErrorHelpers.render_json_error(conn, 400, data) - {:view_deleted, false} -> ErrorHelpers.render_json_error(conn, 400, "Account not found") - _ -> - ErrorHelpers.render_json_error(conn, 500, "There was an issue finding user") + {:error, _} -> + ErrorHelpers.render_json_error(conn, 400, "Account not found") end end @@ -211,16 +199,13 @@ defmodule EpochtalkServerWeb.Controllers.User do Logs out the logged in `User` """ def logout(conn, _attrs) do - with {:auth, true} <- {:auth, Guardian.Plug.authenticated?(conn)}, - user <- Guardian.Plug.current_resource(conn), + with user <- Guardian.Plug.current_resource(conn), token <- Guardian.Plug.current_token(conn), {:ok, conn} <- Session.delete(conn) do EpochtalkServerWeb.Endpoint.broadcast("user:#{user.id}", "logout", %{token: token}) render(conn, :data, data: %{success: true}) else - {:auth, false} -> ErrorHelpers.render_json_error(conn, 400, "Not logged in") - {:error, error} -> ErrorHelpers.render_json_error(conn, 500, error) - _ -> ErrorHelpers.render_json_error(conn, 500, "There was an issue signing out") + {:error, data} -> ErrorHelpers.render_json_error(conn, 500, data) end end @@ -265,9 +250,6 @@ defmodule EpochtalkServerWeb.Controllers.User do {:error, :unban_error} -> ErrorHelpers.render_json_error(conn, 500, "There was an issue unbanning user, upon login") - - _ -> - ErrorHelpers.render_json_error(conn, 500, "There was an issue while attempting to login") end end diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index 7a1e7820..a2d239ea 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -16,13 +16,14 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do iex> alias EpochtalkServerWeb.Helpers.Pagination iex> Mention ...> |> order_by(asc: :id) - ...> |> Pagination.page_simple(1, per_page: 25) + ...> |> Pagination.page_simple(1, per_page: 25, desc: true) {:ok, [], %{next: false, page: 1, per_page: 25, prev: false, total_pages: 1, - total_records: 0}} + total_records: 0, + desc: true}} iex> Invitation ...> |> order_by(desc: :email) ...> |> Pagination.page_simple(1, per_page: 10) @@ -31,14 +32,16 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do per_page: 10, prev: false, total_pages: 1, - total_records: 0}} + total_records: 0, + desc: true}} """ @spec page_simple( query :: Ecto.Queryable.t(), count_query :: Ecto.Queryable.t(), page :: integer | String.t() | nil, - per_page: integer | String.t() | nil - ) :: {:ok, list :: [term()] | [], pagination_data :: map()} + per_page: integer | String.t() | nil, + desc: boolean + ) :: {:ok, list :: [term()] | [], pagination_data :: map()} | {:error, data :: any()} def page_simple(query, count_query, nil, per_page: nil, desc: desc), do: page_simple(query, count_query, 1, per_page: 15, desc: desc) diff --git a/lib/epochtalk_server_web/router.ex b/lib/epochtalk_server_web/router.ex index dcdb48d2..98225547 100644 --- a/lib/epochtalk_server_web/router.ex +++ b/lib/epochtalk_server_web/router.ex @@ -65,6 +65,7 @@ defmodule EpochtalkServerWeb.Router do get "/admin/modlog", ModerationLog, :page get "/boards/movelist", Board, :movelist post "/images/s3/upload", ImageReference, :s3_request_upload + delete "/logout", User, :logout end scope "/api", EpochtalkServerWeb.Controllers do @@ -86,7 +87,6 @@ defmodule EpochtalkServerWeb.Router do post "/register", User, :register post "/login", User, :login post "/confirm", User, :confirm - delete "/logout", User, :logout end scope "/", EpochtalkServerWeb.Controllers do diff --git a/test/epochtalk_server_web/controllers/user_test.exs b/test/epochtalk_server_web/controllers/user_test.exs index 2319200d..92e7ebfd 100644 --- a/test/epochtalk_server_web/controllers/user_test.exs +++ b/test/epochtalk_server_web/controllers/user_test.exs @@ -314,10 +314,10 @@ defmodule Test.EpochtalkServerWeb.Controllers.User do response = conn |> delete(Routes.user_path(conn, :logout)) - |> json_response(400) + |> json_response(401) - assert response["error"] == "Bad Request" - assert response["message"] == "Not logged in" + assert response["error"] == "Unauthorized" + assert response["message"] == "No resource found" end end From d8dbf14ed2777a5c7f97f1925b543f57d6f137e4 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 12 Nov 2024 10:31:23 -1000 Subject: [PATCH 149/231] feat(user-find): convert gender and dob into human readable format --- lib/epochtalk_server_web/json/user_json.ex | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 2e8b0f0e..a5a36294 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -22,16 +22,18 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do do: EpochtalkServer.BBCParser.async_parse(user.signature), else: nil - gender = case Map.get(user, :gender) do - 1 -> "Male" - 2 -> "Female" - _ -> nil - end - - dob = case d = Map.get(user, :dob) do - ~D[0001-01-01] -> nil - _ -> d - end + gender = + case Map.get(user, :gender) do + 1 -> "Male" + 2 -> "Female" + _ -> nil + end + + dob = + case d = Map.get(user, :dob) do + ~D[0001-01-01] -> nil + _ -> d + end user |> Map.put(:signature, parsed_signature) From e09340fd8fff872b11fd634692791fc35374998f Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Tue, 12 Nov 2024 10:57:15 -1000 Subject: [PATCH 150/231] test(warnings): resolve test warnings --- mix.exs | 2 +- test/epochtalk_server_web/controllers/user_test.exs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index cd773499..93a592d9 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EpochtalkServer.MixProject do {:hackney, "~> 1.9"}, {:hammer, "~> 6.2"}, {:hammer_backend_redis, "~> 6.1"}, - {:html_entities, "~> 0.5.2", only: [:dev]}, + {:html_entities, "~> 0.5.2", only: [:dev, :test]}, {:html_sanitize_ex, "~> 1.4"}, {:iteraptor, git: "https://github.com/epochtalk/elixir-iteraptor.git", tag: "1.13.1"}, {:jason, "~> 1.4.0"}, diff --git a/test/epochtalk_server_web/controllers/user_test.exs b/test/epochtalk_server_web/controllers/user_test.exs index 92e7ebfd..52d9733e 100644 --- a/test/epochtalk_server_web/controllers/user_test.exs +++ b/test/epochtalk_server_web/controllers/user_test.exs @@ -56,16 +56,16 @@ defmodule Test.EpochtalkServerWeb.Controllers.User do end end - @tag :banned describe "unban/1" do + @tag :banned test "unbans banned user", %{users: %{user: user}} do {:ok, unbanned_user_changeset} = Ban.unban(user) assert unbanned_user_changeset.ban_info == nil end end - @tag :malicious describe "handle_malicious_user/2" do + @tag :malicious test "populates ban_info and malicious_score if user is malicious", %{ users: %{user: user}, malicious_user_changeset: malicious_user_changeset From ea4e4deedd8077db040ad01300cd581946fe55f5 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 14 Nov 2024 05:50:46 -0800 Subject: [PATCH 151/231] refactor(proxy_conversion): multiply by 1000 -> @ms_per_sec --- .../helpers/proxy_conversion.ex | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ae4e291b..6f8bfb34 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -3,6 +3,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do alias EpochtalkServer.SmfRepo alias EpochtalkServerWeb.Helpers.ProxyPagination + @ms_per_sec 1000 + @moduledoc """ Helper for pulling and formatting data from SmfRepo """ @@ -75,7 +77,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do id: t.id_poll, change_vote: p.changeVote, display_mode: "always", - expiration: p.expireTime * 1000, + expiration: p.expireTime * @ms_per_sec, has_voted: false, locked: p.votingLocked == 1, max_answers: p.maxVotes, @@ -133,8 +135,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do first_post_id: t.id_first_msg, last_post_id: t.id_last_msg, title: f.subject, - updated_at: l.posterTime * 1000, - last_post_created_at: l.posterTime * 1000, + updated_at: l.posterTime * @ms_per_sec, + last_post_created_at: l.posterTime * @ms_per_sec, last_post_user_id: l.id_member, last_post_username: l.posterName, view_count: t.numViews, @@ -207,7 +209,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do |> join(:left, [b, m], t in "smf_topics", on: m.id_topic == t.id_topic) |> select([b, m, t], %{ id: b.id_board, - last_post_created_at: m.posterTime * 1000, + last_post_created_at: m.posterTime * @ms_per_sec, last_post_position: t.numReplies, last_post_username: m.posterName, last_thread_created_at: t.id_member_started, @@ -215,7 +217,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do last_thread_post_count: t.numReplies, last_thread_slug: t.id_topic, last_thread_title: m.subject, - last_thread_updated_at: m.posterTime * 1000 + last_thread_updated_at: m.posterTime * @ms_per_sec }) |> SmfRepo.all() |> case do @@ -288,9 +290,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do title: f.subject, user_id: f.id_member, username: f.posterName, - created_at: f.posterTime * 1000, + created_at: f.posterTime * @ms_per_sec, user_deleted: false, - last_post_created_at: l.posterTime * 1000, + last_post_created_at: l.posterTime * @ms_per_sec, last_post_deleted: false, last_post_user_id: l.id_member, last_post_username: l.posterName, @@ -334,9 +336,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do title: f.subject, user_id: f.id_member, username: f.posterName, - created_at: f.posterTime * 1000, + created_at: f.posterTime * @ms_per_sec, user_deleted: false, - last_post_created_at: l.posterTime * 1000, + last_post_created_at: l.posterTime * @ms_per_sec, last_post_deleted: false, last_post_user_id: l.id_member, last_post_username: l.posterName, @@ -405,7 +407,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do body: m.body, updated_at: m.modifiedTime, username: m.posterName, - created_at: m.posterTime * 1000, + created_at: m.posterTime * @ms_per_sec, modified_time: m.modifiedTime, avatar: fragment( From aeb5cf3367ba2fb6b6dd28ff88eabfdae0bb4488 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 14 Nov 2024 05:51:50 -0800 Subject: [PATCH 152/231] refactor(proxy_conversion): implement @limit_exceeded_error will split this functionality over different functions later, consolidating error message now --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 6f8bfb34..bcf34d4a 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -3,6 +3,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do alias EpochtalkServer.SmfRepo alias EpochtalkServerWeb.Helpers.ProxyPagination + @limit_exceeded_error {:error, "Limit too large, please try again"} @ms_per_sec 1000 @moduledoc """ @@ -14,7 +15,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_model(_, ids, _, _) when length(ids) > 25 do - {:error, "Limit too large, please try again"} + @limit_exceeded_error end def build_model(model_type, id, page, per_page) when is_integer(id) do From b9cefea4208e062a71844981cbf838368980ff44 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 14 Nov 2024 10:55:38 -1000 Subject: [PATCH 153/231] feat(last-active): implement last active, respecting flag showOnline if user doesnt want to be shown as active within the past 72 hours --- .../helpers/proxy_conversion.ex | 31 ++++++++++++------- lib/epochtalk_server_web/json/user_json.ex | 14 +++++++++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 59bea5bd..794aba1f 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -83,19 +83,18 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_user(user_id) do - last_active = - from m in "smf_messages", - where: m.id_member == ^user_id, - order_by: [desc: m.posterTime], - limit: 1, - select: %{last_active: m.posterTime * 1000} - user = from(u in "smf_members", where: u.id_member == ^user_id) |> join(:left, [u], a in "smf_attachments", on: u.id_member == a.id_member and a.attachmentType == 1 ) - |> select([u, a], %{ + |> join(:left, [u], m in "smf_membergroups", + on: u.id_group != 0 and u.id_group == m.id_group + ) + |> join(:left, [u], g in "smf_membergroups", + on: u.id_post_group == g.id_group + ) + |> select([u, a, m, g], %{ activity: u.activity, created_at: u.dateRegistered * 1000, dob: u.birthdate, @@ -112,6 +111,12 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do username: u.realName, title: u.usertitle, website: u.websiteUrl, + last_login: u.lastLogin * 1000, + show_online: u.showOnline, + group_name: m.groupName, + group_name_2: g.groupName, + group_color: m.onlineColor, + group_color_2: g.onlineColor, avatar: fragment( "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", @@ -122,9 +127,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do }) |> SmfRepo.one() - if user.post_count > 0, - do: Map.merge(user, SmfRepo.one(last_active)), - else: Map.put(user, :last_active, user.created_at) + user + |> Map.put(:position, user.group_name || user.group_name_2) + |> Map.put(:position_color, user.group_color || user.group_color_2) + |> Map.delete(:group_name) + |> Map.delete(:group_name_2) + |> Map.delete(:group_color) + |> Map.delete(:group_color_2) end def build_poll(thread_id) do diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index a5a36294..5c673f9f 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -35,10 +35,24 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do _ -> d end + {:ok, last_login} = DateTime.from_unix(1731427963000, :millisecond) + last_login_past_72_hours = DateTime.diff(DateTime.utc_now, last_login, :hour) > 72 + + last_active = if user.show_online == 1 or last_login_past_72_hours, + do: user.last_login, + else: nil + + user = if user.title == "", + do: user |> Map.delete(:title), + else: user + user |> Map.put(:signature, parsed_signature) |> Map.put(:gender, gender) |> Map.put(:dob, dob) + |> Map.put(:last_active, last_active) + |> Map.delete(:last_login) + |> Map.delete(:show_online) end @doc """ From dfd0a30c7333a27d3689420513357835826422b4 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 14 Nov 2024 11:11:38 -1000 Subject: [PATCH 154/231] fix(last-login): use users last login instead of test value, resolve credo error --- lib/epochtalk_server_web/json/user_json.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 5c673f9f..9f0e0b6d 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -35,17 +35,13 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do _ -> d end - {:ok, last_login} = DateTime.from_unix(1731427963000, :millisecond) + {:ok, last_login} = DateTime.from_unix(user.last_login, :millisecond) last_login_past_72_hours = DateTime.diff(DateTime.utc_now, last_login, :hour) > 72 last_active = if user.show_online == 1 or last_login_past_72_hours, do: user.last_login, else: nil - user = if user.title == "", - do: user |> Map.delete(:title), - else: user - user |> Map.put(:signature, parsed_signature) |> Map.put(:gender, gender) From 0068cf48cee8792b70a2e4ac61ac33cf439243c2 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 14 Nov 2024 11:26:14 -1000 Subject: [PATCH 155/231] refactor(user-find): move manipulation of user object into user_json file --- .../helpers/proxy_conversion.ex | 89 ++++++++----------- lib/epochtalk_server_web/json/user_json.ex | 15 +++- 2 files changed, 49 insertions(+), 55 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 794aba1f..171bfdaa 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -83,57 +83,44 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end def build_user(user_id) do - user = - from(u in "smf_members", where: u.id_member == ^user_id) - |> join(:left, [u], a in "smf_attachments", - on: u.id_member == a.id_member and a.attachmentType == 1 - ) - |> join(:left, [u], m in "smf_membergroups", - on: u.id_group != 0 and u.id_group == m.id_group - ) - |> join(:left, [u], g in "smf_membergroups", - on: u.id_post_group == g.id_group - ) - |> select([u, a, m, g], %{ - activity: u.activity, - created_at: u.dateRegistered * 1000, - dob: u.birthdate, - gender: u.gender, - id: u.id_member, - language: nil, - location: u.location, - merit: u.merit, - id_group: u.id_group, - id_post_group: u.id_post_group, - signature: u.signature, - post_count: u.posts, - name: u.realName, - username: u.realName, - title: u.usertitle, - website: u.websiteUrl, - last_login: u.lastLogin * 1000, - show_online: u.showOnline, - group_name: m.groupName, - group_name_2: g.groupName, - group_color: m.onlineColor, - group_color_2: g.onlineColor, - avatar: - fragment( - "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", - u.avatar, - u.avatar, - a.filename - ) - }) - |> SmfRepo.one() - - user - |> Map.put(:position, user.group_name || user.group_name_2) - |> Map.put(:position_color, user.group_color || user.group_color_2) - |> Map.delete(:group_name) - |> Map.delete(:group_name_2) - |> Map.delete(:group_color) - |> Map.delete(:group_color_2) + from(u in "smf_members", where: u.id_member == ^user_id) + |> join(:left, [u], a in "smf_attachments", + on: u.id_member == a.id_member and a.attachmentType == 1 + ) + |> join(:left, [u], m in "smf_membergroups", on: u.id_group != 0 and u.id_group == m.id_group) + |> join(:left, [u], g in "smf_membergroups", on: u.id_post_group == g.id_group) + |> select([u, a, m, g], %{ + activity: u.activity, + created_at: u.dateRegistered * 1000, + dob: u.birthdate, + gender: u.gender, + id: u.id_member, + language: nil, + location: u.location, + merit: u.merit, + id_group: u.id_group, + id_post_group: u.id_post_group, + signature: u.signature, + post_count: u.posts, + name: u.realName, + username: u.realName, + title: u.usertitle, + website: u.websiteUrl, + last_login: u.lastLogin * 1000, + show_online: u.showOnline, + group_name: m.groupName, + group_name_2: g.groupName, + group_color: m.onlineColor, + group_color_2: g.onlineColor, + avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ) + }) + |> SmfRepo.one() end def build_poll(thread_id) do diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 9f0e0b6d..315eb498 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -36,19 +36,26 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do end {:ok, last_login} = DateTime.from_unix(user.last_login, :millisecond) - last_login_past_72_hours = DateTime.diff(DateTime.utc_now, last_login, :hour) > 72 + last_login_past_72_hours = DateTime.diff(DateTime.utc_now(), last_login, :hour) > 72 - last_active = if user.show_online == 1 or last_login_past_72_hours, - do: user.last_login, - else: nil + last_active = + if user.show_online == 1 or last_login_past_72_hours, + do: user.last_login, + else: nil user |> Map.put(:signature, parsed_signature) |> Map.put(:gender, gender) |> Map.put(:dob, dob) |> Map.put(:last_active, last_active) + |> Map.put(:position, user.group_name || user.group_name_2) + |> Map.put(:position_color, user.group_color || user.group_color_2) |> Map.delete(:last_login) |> Map.delete(:show_online) + |> Map.delete(:group_name) + |> Map.delete(:group_name_2) + |> Map.delete(:group_color) + |> Map.delete(:group_color_2) end @doc """ From 1bdd8b6207bfd0ebb8abb96aa90423d829bacc77 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 14 Nov 2024 11:47:55 -1000 Subject: [PATCH 156/231] fix(credo): resolve credo error with user_json --- lib/epochtalk_server_web/json/user_json.ex | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 315eb498..93456f53 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -35,21 +35,18 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do _ -> d end - {:ok, last_login} = DateTime.from_unix(user.last_login, :millisecond) - last_login_past_72_hours = DateTime.diff(DateTime.utc_now(), last_login, :hour) > 72 + last_active = calculate_last_active(user) - last_active = - if user.show_online == 1 or last_login_past_72_hours, - do: user.last_login, - else: nil + position = user.group_name || user.group_name_2 + position_color = user.group_color || user.group_color_2 user |> Map.put(:signature, parsed_signature) |> Map.put(:gender, gender) |> Map.put(:dob, dob) |> Map.put(:last_active, last_active) - |> Map.put(:position, user.group_name || user.group_name_2) - |> Map.put(:position_color, user.group_color || user.group_color_2) + |> Map.put(:position, position) + |> Map.put(:position_color, position_color) |> Map.delete(:last_login) |> Map.delete(:show_online) |> Map.delete(:group_name) @@ -187,4 +184,11 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do reply = if malicious_score, do: Map.put(reply, :malicious_score, malicious_score), else: reply reply end + + defp calculate_last_active(user) when is_map(user) do + {:ok, last_login} = DateTime.from_unix(user.last_login, :millisecond) + last_login_past_72_hours = DateTime.diff(DateTime.utc_now(), last_login, :hour) > 72 + + if user.show_online == 1 or last_login_past_72_hours, do: user.last_login + end end From 99ab83642e1264185de8bfdefe3e0d94c7d5793e Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 14 Nov 2024 11:58:55 -1000 Subject: [PATCH 157/231] feat(docker): upgrade elixir version in docker file --- Dockerfile | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d7b70934..adc5f7bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM elixir:1.14.0 +FROM elixir:1.17.3 # install php RUN curl -sSL https://packages.sury.org/php/README.txt | bash -x RUN apt update diff --git a/mix.exs b/mix.exs index 93a592d9..de965665 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule EpochtalkServer.MixProject do [ app: :epochtalk_server, version: "0.1.0", - elixir: "~> 1.12", + elixir: "~> 1.17", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), start_permanent: Mix.env() == :prod, From 755640e4c255c6fe021eb4019a631d66d611cc54 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Thu, 14 Nov 2024 20:49:58 -0800 Subject: [PATCH 158/231] feat(board-moderators): Added query to get moderators for all boards in smf repo --- lib/epochtalk_server_web/controllers/board.ex | 2 +- .../controllers/thread.ex | 2 +- .../helpers/proxy_conversion.ex | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 0106de91..ce4008ab 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -153,7 +153,7 @@ defmodule EpochtalkServerWeb.Controllers.Board do stripped <- Validate.cast(attrs, "stripped", :boolean, default: false), user_priority <- ACL.get_user_priority(conn), board_mapping <- BoardMapping.all(stripped: stripped), - board_moderators <- BoardModerator.all(), + {:ok, board_moderators} <- ProxyConversion.build_model("boards.moderators"), {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), categories <- Category.all() do diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 23ead448..8f252b83 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -814,7 +814,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "threads.byBoard"), board_mapping <- BoardMapping.all(), - board_moderators <- BoardModerator.all(), + {:ok, board_moderators} <- ProxyConversion.build_model("boards.moderators"), {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), {:ok, threads, data} <- diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ae4e291b..d138a9f7 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -58,6 +58,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "boards.last_post_info" -> build_board_last_post_info() + "boards.moderators" -> + build_board_moderators() + "threads.recent" -> build_recent_threads() @@ -227,6 +230,32 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do end end + def build_board_moderators() do + %{id_board_blacklist: id_board_blacklist} = + Application.get_env(:epochtalk_server, :proxy_config) + + from(b in "smf_boards", + where: b.id_board not in ^id_board_blacklist + ) + |> join(:left, [b], mod in "smf_moderators", on: b.id_board == mod.id_board) + |> join(:left, [b, mod], m in "smf_members", on: mod.id_member == m.id_member) + |> select([b, mod, m], %{ + board_id: b.id_board, + user_id: m.id_member, + user: %{ + username: m.realname + } + }) + |> SmfRepo.all() + |> case do + [] -> + {:error, "Board Moderators not found"} + + moderators -> + return_tuple(moderators) + end + end + def build_board(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) From 1741f4e2775a8a6912f242c874c94962b3254f25 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 15 Nov 2024 09:48:02 -0800 Subject: [PATCH 159/231] feat(helpers/proxy_conversion): load board last post user avatar --- .../helpers/proxy_conversion.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index ae4e291b..56b13382 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -205,11 +205,22 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do ) |> join(:left, [b], m in "smf_messages", on: b.id_last_msg == m.id_msg) |> join(:left, [b, m], t in "smf_topics", on: m.id_topic == t.id_topic) - |> select([b, m, t], %{ + |> join(:left, [b, m, t], u in "smf_members", on: m.id_member == u.id_member) + |> join(:left, [b, m, t, u], a in "smf_attachments", + on: m.id_member == a.id_member and a.attachmentType == 1 + ) + |> select([b, m, t, u, a], %{ id: b.id_board, last_post_created_at: m.posterTime * 1000, last_post_position: t.numReplies, last_post_username: m.posterName, + last_post_avatar: + fragment( + "if(? <>'',concat('https://bitcointalk.org/avatars/',?),ifnull(concat('https://bitcointalk.org/useravatars/',?),''))", + u.avatar, + u.avatar, + a.filename + ), last_thread_created_at: t.id_member_started, last_thread_id: t.id_topic, last_thread_post_count: t.numReplies, From 40044b4ebce5f8205fdbcbf5fe9797531fb01a4c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 15 Nov 2024 10:21:31 -0800 Subject: [PATCH 160/231] refactor(config): poolboy_config -> bbc_parser_poolboy_config specifically, the poolboy_config for bbc_parser --- config/runtime.exs | 4 ++-- lib/epochtalk_server/application.ex | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 187de652..61fded12 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -407,7 +407,7 @@ end ##### PROXY REPO CONFIGURATIONS ##### -poolboy_config = [ +bbc_parser_poolboy_config = [ name: {:local, :bbc_parser}, worker_module: EpochtalkServer.BBCParser, size: 5, @@ -415,7 +415,7 @@ poolboy_config = [ strategy: :fifo ] -config :epochtalk_server, poolboy_config: poolboy_config +config :epochtalk_server, bbc_parser_poolboy_config: bbc_parser_poolboy_config # conditionally show debug logs in prod if config_env() == :prod do diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 13507d66..84541ff1 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -22,7 +22,7 @@ defmodule EpochtalkServer.Application do # Start the Smf repository EpochtalkServer.SmfRepo, # Start the BBC Parser - :poolboy.child_spec(:bbc_parser, poolboy_config()), + :poolboy.child_spec(:bbc_parser, bbc_parser_poolboy_config()), # Start Role Cache EpochtalkServer.Cache.Role, # Warm frontend_config variable (referenced by api controllers) @@ -71,5 +71,5 @@ defmodule EpochtalkServer.Application do # fetch redix config defp redix_config(), do: Application.get_env(:epochtalk_server, :redix) - defp poolboy_config, do: Application.get_env(:epochtalk_server, :poolboy_config) + defp bbc_parser_poolboy_config, do: Application.get_env(:epochtalk_server, :bbc_parser_poolboy_config) end From dc411238284b33365889c4930ab8c2971aea1957 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 15 Nov 2024 10:22:25 -0800 Subject: [PATCH 161/231] refactor(bbc_parser): reduce bbc parser php call timeout to 1 second 10 seconds is much too long, we don't want to take up resources for that amount of time. 1 second is fairly long as well --- lib/epochtalk_server/bbc_parser.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index d9ec3333..279ab6d4 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -2,7 +2,7 @@ defmodule EpochtalkServer.BBCParser do use GenServer require Logger alias Porcelain.Process, as: Proc - @timeout 10_000 + @timeout 1_000 @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser From d0f1a03c569fbf2ea238ea8e6d951a43b2e84f83 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 15 Nov 2024 10:50:11 -0800 Subject: [PATCH 162/231] style(application): mix format --- lib/epochtalk_server/application.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/epochtalk_server/application.ex b/lib/epochtalk_server/application.ex index 84541ff1..d9de576b 100644 --- a/lib/epochtalk_server/application.ex +++ b/lib/epochtalk_server/application.ex @@ -71,5 +71,6 @@ defmodule EpochtalkServer.Application do # fetch redix config defp redix_config(), do: Application.get_env(:epochtalk_server, :redix) - defp bbc_parser_poolboy_config, do: Application.get_env(:epochtalk_server, :bbc_parser_poolboy_config) + defp bbc_parser_poolboy_config, + do: Application.get_env(:epochtalk_server, :bbc_parser_poolboy_config) end From d976b9db77be42c23f0097410f38f2db12877d41 Mon Sep 17 00:00:00 2001 From: Chris Rodrigues Date: Fri, 15 Nov 2024 13:41:01 -0800 Subject: [PATCH 163/231] fix(moderators): Changed to inner join to return only matched values and none with a nil id_member; --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 1e9bd2e5..477d6d9c 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -306,8 +306,8 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do from(b in "smf_boards", where: b.id_board not in ^id_board_blacklist ) - |> join(:left, [b], mod in "smf_moderators", on: b.id_board == mod.id_board) - |> join(:left, [b, mod], m in "smf_members", on: mod.id_member == m.id_member) + |> join(:inner, [b], mod in "smf_moderators", on: b.id_board == mod.id_board) + |> join(:inner, [b, mod], m in "smf_members", on: mod.id_member == m.id_member) |> select([b, mod, m], %{ board_id: b.id_board, user_id: m.id_member, From 2f01aa0d9111464c38fc0717a2ff89edb4def1d9 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Nov 2024 03:36:32 -0800 Subject: [PATCH 164/231] refactor(bbc_parser): async_parse -> parse this function is not async --- lib/epochtalk_server/bbc_parser.ex | 2 +- lib/epochtalk_server_web/json/post_json.ex | 4 ++-- lib/epochtalk_server_web/json/user_json.ex | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 279ab6d4..f7ea89f1 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -40,7 +40,7 @@ defmodule EpochtalkServer.BBCParser do @doc """ Uses poolboy to call parser """ - def async_parse(bbcode_data) do + def parse(bbcode_data) do :poolboy.transaction( :bbc_parser, fn pid -> diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 6e739165..ac42582c 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -447,7 +447,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") - parsed_body = EpochtalkServer.BBCParser.async_parse(body) + parsed_body = EpochtalkServer.BBCParser.parse(body) signature = if Map.get(post.user, :signature), @@ -456,7 +456,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do parsed_signature = if signature, - do: EpochtalkServer.BBCParser.async_parse(signature), + do: EpochtalkServer.BBCParser.parse(signature), else: nil user = post.user |> Map.put(:signature, parsed_signature) diff --git a/lib/epochtalk_server_web/json/user_json.ex b/lib/epochtalk_server_web/json/user_json.ex index 93456f53..3194528c 100644 --- a/lib/epochtalk_server_web/json/user_json.ex +++ b/lib/epochtalk_server_web/json/user_json.ex @@ -19,7 +19,7 @@ defmodule EpochtalkServerWeb.Controllers.UserJSON do def find_proxy(%{user: user}) do parsed_signature = if user.signature, - do: EpochtalkServer.BBCParser.async_parse(user.signature), + do: EpochtalkServer.BBCParser.parse(user.signature), else: nil gender = From 2f0d80cc00eaae5437e029e1ef3a8e688079bbc8 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Nov 2024 03:38:35 -0800 Subject: [PATCH 165/231] refactor(bbc_parser): return unparsed bbcode on poolboy error return original data if parsing failed, instead of :ok (confusing to look at in UI) improve logging * user Logger.error * log error message, type, pid * specify module/function better --- lib/epochtalk_server/bbc_parser.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index f7ea89f1..98437054 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -45,12 +45,13 @@ defmodule EpochtalkServer.BBCParser do :bbc_parser, fn pid -> try do - Logger.debug("#{__MODULE__}(ASYNC PARSE): #{inspect(pid)}") + Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") GenServer.call(pid, {:parse, bbcode_data}, @timeout) catch e, r -> - Logger.debug("poolboy transaction caught error: #{inspect(e)}, #{inspect(r)}") - :ok + # something went wrong, log the error + Logger.error("#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}") + bbcode_data end end, @timeout From e811005eba9bd887a1084d40464f7a829b30c820 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Nov 2024 03:42:16 -0800 Subject: [PATCH 166/231] fix(bbc_parser): handle hanging in genserver parse call set a timeout for porcelain php parser call differentiate call and receive timeouts return {:ok, parsed} when data is parsed in time return {:timeout, unparsed} when data does not return in time handle return types after genserver call log as error when timeout happens this will help us debug and find offending bbcode in logs --- lib/epochtalk_server/bbc_parser.ex | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 98437054..bd2898ca 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -2,7 +2,12 @@ defmodule EpochtalkServer.BBCParser do use GenServer require Logger alias Porcelain.Process, as: Proc - @timeout 1_000 + + # poolboy genserver call timeout (ms) + # should be greater than internal porcelain php call + @call_timeout 500 + # porcelain php parser call timeout (ms) + @receive_timeout 400 @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser @@ -22,9 +27,10 @@ defmodule EpochtalkServer.BBCParser do parsed = receive do - {^pid, :data, :out, data} -> - Logger.debug(data) - data + {^pid, :data, :out, data} -> {:ok, data} + after + # time out after not receiving any data + @receive_timeout -> {:timeout, bbcode_data} end {:reply, parsed, {proc, pid}} @@ -46,7 +52,15 @@ defmodule EpochtalkServer.BBCParser do fn pid -> try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse, bbcode_data}, @timeout) + GenServer.call(pid, {:parse, bbcode_data}, @call_timeout) + |> case do + # on success, return parsed data + {:ok, parsed} -> parsed + # on parse timeout, log and return unparsed data + {:timeout, unparsed} -> + Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(unparsed)}") + "

    ((bbcode parse timeout))


    " <> unparsed + end catch e, r -> # something went wrong, log the error @@ -54,7 +68,7 @@ defmodule EpochtalkServer.BBCParser do bbcode_data end end, - @timeout + @call_timeout ) end From d911e383c259ddfd9b211ce9295981c74c193889 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Nov 2024 03:46:44 -0800 Subject: [PATCH 167/231] fix(bbc_parser): use {:ok, data} format for empty string case --- lib/epochtalk_server/bbc_parser.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index bd2898ca..ef150633 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -20,7 +20,7 @@ defmodule EpochtalkServer.BBCParser do @impl true def handle_call({:parse, ""}, _from, {proc, pid}), - do: {:reply, "", {proc, pid}} + do: {:reply, {:ok, ""}, {proc, pid}} def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") From a0e701da6eac580c19cdf550cbf630b5ca132514 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Nov 2024 03:47:35 -0800 Subject: [PATCH 168/231] refactor(config/runtime): increase bbcode parser pool size to 50 --- config/runtime.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index 61fded12..bc1cb528 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -410,7 +410,7 @@ end bbc_parser_poolboy_config = [ name: {:local, :bbc_parser}, worker_module: EpochtalkServer.BBCParser, - size: 5, + size: 50, max_overflow: 2, strategy: :fifo ] From 6e48d6da9be7a7bd5d349305414bbafa9e3f22a4 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 18 Nov 2024 03:48:53 -0800 Subject: [PATCH 169/231] style(bbc_parser): mix format --- lib/epochtalk_server/bbc_parser.ex | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index ef150633..17142f9e 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -52,19 +52,27 @@ defmodule EpochtalkServer.BBCParser do fn pid -> try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") + GenServer.call(pid, {:parse, bbcode_data}, @call_timeout) |> case do # on success, return parsed data - {:ok, parsed} -> parsed + {:ok, parsed} -> + parsed + # on parse timeout, log and return unparsed data {:timeout, unparsed} -> Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(unparsed)}") - "

    ((bbcode parse timeout))


    " <> unparsed + + "

    ((bbcode parse timeout))


    " <> + unparsed end catch e, r -> # something went wrong, log the error - Logger.error("#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}") + Logger.error( + "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" + ) + bbcode_data end end, From 19445aba47c4bae83a3dfd7dc15cda7f383b148f Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 22 Nov 2024 10:27:52 -1000 Subject: [PATCH 170/231] fix(parser): resolve issue with with posts ending in backslashes causing parser to fail --- lib/epochtalk_server_web/json/post_json.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index ac42582c..99c9da4f 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -447,6 +447,11 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do defp format_proxy_post_data_for_by_thread(post) do body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") + # add space to end if the last character is a backslash (fix for parser) + body_len = String.length(body) + last_char = String.slice(body, body_len - 1..body_len) + body = if last_char == "\\", do: body <> " ", else: body + parsed_body = EpochtalkServer.BBCParser.parse(body) signature = From bf28bb03be7fd879969bb7e0d36f57e135299348 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 22 Nov 2024 11:35:30 -1000 Subject: [PATCH 171/231] fix(format): run formatter --- lib/epochtalk_server_web/json/post_json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 99c9da4f..99611f13 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -449,7 +449,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do # add space to end if the last character is a backslash (fix for parser) body_len = String.length(body) - last_char = String.slice(body, body_len - 1..body_len) + last_char = String.slice(body, (body_len - 1)..body_len) body = if last_char == "\\", do: body <> " ", else: body parsed_body = EpochtalkServer.BBCParser.parse(body) From 6c40de6512a01a271575b8a013eaf7923bf4ccbb Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 26 Nov 2024 11:53:25 -0800 Subject: [PATCH 172/231] refactor(proxy_conversion): consolidate model builds into one function move options into opts --- .../helpers/proxy_conversion.ex | 75 ++++++++----------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 57696a75..d8f0cbaa 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -3,6 +3,9 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do alias EpochtalkServer.SmfRepo alias EpochtalkServerWeb.Helpers.ProxyPagination + @default_page 1 + @default_per_page 25 + @max_ids 25 @limit_exceeded_error {:error, "Limit too large, please try again"} @ms_per_sec 1000 @@ -10,42 +13,38 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do Helper for pulling and formatting data from SmfRepo """ - def build_model(model_type, ids, _, _) when is_nil(model_type) or is_nil(ids) do + def build_model(model_type, ids, _opts) when is_nil(model_type) or is_nil(ids) do {:ok, %{}, %{}} end - def build_model(_, ids, _, _) when length(ids) > 25 do + def build_model(_, ids, _opts) when length(ids) > @max_ids do @limit_exceeded_error end - def build_model(model_type, id, page, per_page) when is_integer(id) do + def build_model(model_type, id, opts) when is_integer(id) do + default_opts = %{ + page: @default_page, + per_page: @default_per_page, + desc: false + } + %{ + page: page, + per_page: per_page, + desc: desc + } = Enum.into(opts, default_opts) case model_type do - "threads.by_board" -> - build_threads_by_board(id, page, per_page) - - "posts.by_thread" -> - build_posts_by_thread(id, page, per_page) - - _ -> - build_model(nil, nil, nil, nil) - end - end + "boards.counts" -> + build_board_counts() - def build_model(model_type, id, page, per_page, desc) when is_integer(id) do - case model_type do - "threads.by_user" -> - build_threads_by_user(id, page, per_page, desc) + "boards.last_post_info" -> + build_board_last_post_info() - "posts.by_user" -> - build_posts_by_user(id, page, per_page, desc) + "boards.moderators" -> + build_board_moderators() - _ -> - build_model(nil, nil, nil, nil) - end - end + "threads.recent" -> + build_recent_threads() - def build_model(model_type, id) do - case model_type do "category" -> build_category(id) @@ -64,27 +63,19 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "user.find" -> build_user(id) - _ -> - build_model(nil, nil, nil, nil) - end - end - - def build_model(model_type) do - case model_type do - "boards.counts" -> - build_board_counts() - - "boards.last_post_info" -> - build_board_last_post_info() + "threads.by_board" -> + build_threads_by_board(id, page, per_page) - "boards.moderators" -> - build_board_moderators() + "posts.by_thread" -> + build_posts_by_thread(id, page, per_page) - "threads.recent" -> - build_recent_threads() + "threads.by_user" -> + build_threads_by_user(id, page, per_page, desc) + "posts.by_user" -> + build_posts_by_user(id, page, per_page, desc) _ -> - build_model(nil, nil, nil, nil) + build_model(nil, nil, nil) end end From 7447735b96ffbfa4a12b4bfdb12bd20910cd16fe Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 26 Nov 2024 11:58:59 -0800 Subject: [PATCH 173/231] refactor(controllers/post&thread): use new options form for build_model --- lib/epochtalk_server_web/controllers/post.ex | 4 ++-- lib/epochtalk_server_web/controllers/thread.ex | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 6adaa4fd..a990f4dc 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -558,7 +558,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_user", user_id, page, limit, desc) do + ProxyConversion.build_model("posts.by_user", user_id, %{page: page, limit: limit, desc: desc}) do render(conn, :proxy_by_username, %{ posts: posts, count: data.total_records, @@ -584,7 +584,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do thread <- ProxyConversion.build_model("thread", thread_id), poll <- ProxyConversion.build_model("poll.by_thread", thread_id), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_thread", thread_id, page, limit) do + ProxyConversion.build_model("posts.by_thread", thread_id, %{page: page, limit: limit}) do render(conn, :by_thread_proxy, %{ posts: posts, poll: poll, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index c4762a1c..7a71333f 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -800,7 +800,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, threads, data} <- - ProxyConversion.build_model("threads.by_user", user_id, page, limit, desc) do + ProxyConversion.build_model("threads.by_user", user_id, %{page: page, limit: limit, desc: desc}) do render(conn, :proxy_by_username, %{ threads: threads, next: data.next, @@ -827,7 +827,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), {:ok, threads, data} <- - ProxyConversion.build_model("threads.by_board", board_id, page, limit) do + ProxyConversion.build_model("threads.by_board", board_id, %{page: page, limit: limit}) do render(conn, :by_board_proxy, %{ threads: threads, user: user, From 54cc0b95c61045aab8dce83b8d2f8fa9be539b6f Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 26 Nov 2024 11:59:40 -0800 Subject: [PATCH 174/231] style(): mix format --- lib/epochtalk_server_web/controllers/post.ex | 6 +++++- lib/epochtalk_server_web/controllers/thread.ex | 6 +++++- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index a990f4dc..eb29fd2a 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -558,7 +558,11 @@ defmodule EpochtalkServerWeb.Controllers.Post do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_user", user_id, %{page: page, limit: limit, desc: desc}) do + ProxyConversion.build_model("posts.by_user", user_id, %{ + page: page, + limit: limit, + desc: desc + }) do render(conn, :proxy_by_username, %{ posts: posts, count: data.total_records, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 7a71333f..4add75b3 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -800,7 +800,11 @@ defmodule EpochtalkServerWeb.Controllers.Thread do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, threads, data} <- - ProxyConversion.build_model("threads.by_user", user_id, %{page: page, limit: limit, desc: desc}) do + ProxyConversion.build_model("threads.by_user", user_id, %{ + page: page, + limit: limit, + desc: desc + }) do render(conn, :proxy_by_username, %{ threads: threads, next: data.next, diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index d8f0cbaa..fe10d6ec 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -27,11 +27,13 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do per_page: @default_per_page, desc: false } + %{ page: page, per_page: per_page, desc: desc } = Enum.into(opts, default_opts) + case model_type do "boards.counts" -> build_board_counts() @@ -74,6 +76,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do "posts.by_user" -> build_posts_by_user(id, page, per_page, desc) + _ -> build_model(nil, nil, nil) end From e63f1f5102aa7bb677af81c0ea9b20c52eee62dd Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 26 Nov 2024 12:00:50 -0800 Subject: [PATCH 175/231] refactor(helpers/proxy_conversion): move build_model default case into function --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index fe10d6ec..31ea55f1 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -13,10 +13,6 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do Helper for pulling and formatting data from SmfRepo """ - def build_model(model_type, ids, _opts) when is_nil(model_type) or is_nil(ids) do - {:ok, %{}, %{}} - end - def build_model(_, ids, _opts) when length(ids) > @max_ids do @limit_exceeded_error end @@ -78,7 +74,7 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do build_posts_by_user(id, page, per_page, desc) _ -> - build_model(nil, nil, nil) + {:ok, %{}, %{}} end end From 4e0aa86b0eede53faba3c85f92569f817a80f203 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 26 Nov 2024 12:24:47 -0800 Subject: [PATCH 176/231] refactor(proxy_conversion): combine build_model function heads --- lib/epochtalk_server_web/helpers/proxy_conversion.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server_web/helpers/proxy_conversion.ex index 31ea55f1..e737927f 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server_web/helpers/proxy_conversion.ex @@ -5,19 +5,15 @@ defmodule EpochtalkServerWeb.Helpers.ProxyConversion do @default_page 1 @default_per_page 25 - @max_ids 25 - @limit_exceeded_error {:error, "Limit too large, please try again"} @ms_per_sec 1000 @moduledoc """ Helper for pulling and formatting data from SmfRepo """ - def build_model(_, ids, _opts) when length(ids) > @max_ids do - @limit_exceeded_error - end - - def build_model(model_type, id, opts) when is_integer(id) do + def build_model(model_type), do: build_model(model_type, nil, nil) + def build_model(model_type, id), do: build_model(model_type, id, nil) + def build_model(model_type, id, opts) do default_opts = %{ page: @default_page, per_page: @default_per_page, From 86bd752f5cf44545ad40089858cf3400fb894488 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 14:44:29 -0800 Subject: [PATCH 177/231] refactor(smf_query): migrate proxy_conversion to smf_query --- .../smf_query.ex} | 2 +- lib/epochtalk_server_web/controllers/board.ex | 8 ++++---- lib/epochtalk_server_web/controllers/post.ex | 10 +++++----- lib/epochtalk_server_web/controllers/thread.ex | 14 +++++++------- lib/epochtalk_server_web/controllers/user.ex | 4 ++-- lib/epochtalk_server_web/helpers/breadcrumbs.ex | 4 ++-- 6 files changed, 21 insertions(+), 21 deletions(-) rename lib/{epochtalk_server_web/helpers/proxy_conversion.ex => epochtalk_server/smf_query.ex} (99%) diff --git a/lib/epochtalk_server_web/helpers/proxy_conversion.ex b/lib/epochtalk_server/smf_query.ex similarity index 99% rename from lib/epochtalk_server_web/helpers/proxy_conversion.ex rename to lib/epochtalk_server/smf_query.ex index e737927f..6bda9e93 100644 --- a/lib/epochtalk_server_web/helpers/proxy_conversion.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -1,4 +1,4 @@ -defmodule EpochtalkServerWeb.Helpers.ProxyConversion do +defmodule EpochtalkServer.SmfQuery do import Ecto.Query alias EpochtalkServer.SmfRepo alias EpochtalkServerWeb.Helpers.ProxyPagination diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 445d2bb5..031ca132 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -11,7 +11,7 @@ defmodule EpochtalkServerWeb.Controllers.Board do alias EpochtalkServerWeb.ErrorHelpers alias EpochtalkServerWeb.Helpers.Validate alias EpochtalkServerWeb.Helpers.ACL - alias EpochtalkServerWeb.Helpers.ProxyConversion + alias EpochtalkServer.SmfQuery plug :check_proxy when action in [:slug_to_id, :by_category] @@ -150,9 +150,9 @@ defmodule EpochtalkServerWeb.Controllers.Board do stripped <- Validate.cast(attrs, "stripped", :boolean, default: false), user_priority <- ACL.get_user_priority(conn), board_mapping <- BoardMapping.all(stripped: stripped), - {:ok, board_moderators} <- ProxyConversion.build_model("boards.moderators"), - {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), - {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), + {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), + {:ok, board_counts} <- SmfQuery.build_model("boards.counts"), + {:ok, board_last_post_info} <- SmfQuery.build_model("boards.last_post_info"), categories <- Category.all() do render(conn, :proxy_by_category, %{ categories: categories, diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index eb29fd2a..1d5fae58 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -10,7 +10,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do alias EpochtalkServerWeb.Helpers.ACL alias EpochtalkServerWeb.Helpers.Sanitize alias EpochtalkServerWeb.Helpers.Parse - alias EpochtalkServerWeb.Helpers.ProxyConversion + alias EpochtalkServer.SmfQuery alias EpochtalkServer.Models.Profile alias EpochtalkServer.Models.Post alias EpochtalkServer.Models.Poll @@ -558,7 +558,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_user", user_id, %{ + SmfQuery.build_model("posts.by_user", user_id, %{ page: page, limit: limit, desc: desc @@ -585,10 +585,10 @@ defmodule EpochtalkServerWeb.Controllers.Post do :ok <- ACL.allow!(conn, "posts.byThread"), board_mapping <- BoardMapping.all(), board_moderators <- BoardModerator.all(), - thread <- ProxyConversion.build_model("thread", thread_id), - poll <- ProxyConversion.build_model("poll.by_thread", thread_id), + thread <- SmfQuery.build_model("thread", thread_id), + poll <- SmfQuery.build_model("poll.by_thread", thread_id), {:ok, posts, data} <- - ProxyConversion.build_model("posts.by_thread", thread_id, %{page: page, limit: limit}) do + SmfQuery.build_model("posts.by_thread", thread_id, %{page: page, limit: limit}) do render(conn, :by_thread_proxy, %{ posts: posts, poll: poll, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 4add75b3..66998e42 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -26,7 +26,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do alias EpochtalkServer.Models.UserActivity alias EpochtalkServer.Models.ThreadSubscription alias EpochtalkServer.Models.Mention - alias EpochtalkServerWeb.Helpers.ProxyConversion + alias EpochtalkServer.SmfQuery plug :check_proxy when action in [:by_board, :by_username, :slug_to_id, :viewed, :recent] @@ -800,7 +800,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, threads, data} <- - ProxyConversion.build_model("threads.by_user", user_id, %{ + SmfQuery.build_model("threads.by_user", user_id, %{ page: page, limit: limit, desc: desc @@ -827,11 +827,11 @@ defmodule EpochtalkServerWeb.Controllers.Thread do user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "threads.byBoard"), board_mapping <- BoardMapping.all(), - {:ok, board_moderators} <- ProxyConversion.build_model("boards.moderators"), - {:ok, board_counts} <- ProxyConversion.build_model("boards.counts"), - {:ok, board_last_post_info} <- ProxyConversion.build_model("boards.last_post_info"), + {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), + {:ok, board_counts} <- SmfQuery.build_model("boards.counts"), + {:ok, board_last_post_info} <- SmfQuery.build_model("boards.last_post_info"), {:ok, threads, data} <- - ProxyConversion.build_model("threads.by_board", board_id, %{page: page, limit: limit}) do + SmfQuery.build_model("threads.by_board", board_id, %{page: page, limit: limit}) do render(conn, :by_board_proxy, %{ threads: threads, user: user, @@ -852,7 +852,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do end defp proxy_recent(conn, _attrs) do - with threads <- ProxyConversion.build_model("threads.recent") do + with threads <- SmfQuery.build_model("threads.recent") do render(conn, :recent, %{threads: threads}) end end diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index 184d9609..9e86dda8 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -17,7 +17,7 @@ defmodule EpochtalkServerWeb.Controllers.User do alias EpochtalkServerWeb.CustomErrors.InvalidPayload alias EpochtalkServerWeb.Helpers.ACL alias EpochtalkServerWeb.Helpers.Validate - alias EpochtalkServerWeb.Helpers.ProxyConversion + alias EpochtalkServer.SmfQuery plug :check_proxy when action in [:find] @@ -270,7 +270,7 @@ defmodule EpochtalkServerWeb.Controllers.User do end defp proxy_find(conn, attrs) do - with user <- ProxyConversion.build_model("user.find", attrs["id"]) do + with user <- SmfQuery.build_model("user.find", attrs["id"]) do render(conn, :find_proxy, %{user: user}) end end diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 23e07bc5..66532a56 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -2,7 +2,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do @moduledoc """ Helper for creating breadcrumbs """ - alias EpochtalkServerWeb.Helpers.ProxyConversion + alias EpochtalkServer.SmfQuery alias EpochtalkServer.Models.Category alias EpochtalkServer.Models.Board alias EpochtalkServer.Models.Thread @@ -76,7 +76,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do thread = cond do is_integer(id) && id < threads_seq -> - ProxyConversion.build_model("thread", id) + SmfQuery.build_model("thread", id) is_binary(id) -> Thread.find(id) From b295b7323f8c20933edbdd6e77988d295f8538fb Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 14:48:12 -0800 Subject: [PATCH 178/231] fix(smf_query): pass empty build_model opts as %{} --- lib/epochtalk_server/smf_query.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 6bda9e93..11bd1d1d 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -11,8 +11,8 @@ defmodule EpochtalkServer.SmfQuery do Helper for pulling and formatting data from SmfRepo """ - def build_model(model_type), do: build_model(model_type, nil, nil) - def build_model(model_type, id), do: build_model(model_type, id, nil) + def build_model(model_type), do: build_model(model_type, nil, %{}) + def build_model(model_type, id), do: build_model(model_type, id, %{}) def build_model(model_type, id, opts) do default_opts = %{ page: @default_page, From 3366aa1759cdeb4df893b855825022932028d10f Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 14:54:36 -0800 Subject: [PATCH 179/231] refactor(smf_query): separate extract_opts functionality into private function --- lib/epochtalk_server/smf_query.ex | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 11bd1d1d..f34cf0c5 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -11,20 +11,23 @@ defmodule EpochtalkServer.SmfQuery do Helper for pulling and formatting data from SmfRepo """ - def build_model(model_type), do: build_model(model_type, nil, %{}) - def build_model(model_type, id), do: build_model(model_type, id, %{}) - def build_model(model_type, id, opts) do + defp extract_opts(opts) do default_opts = %{ page: @default_page, per_page: @default_per_page, desc: false } + Enum.into(opts, default_opts) + end + def build_model(model_type), do: build_model(model_type, nil, %{}) + def build_model(model_type, id), do: build_model(model_type, id, %{}) + def build_model(model_type, id, opts) do %{ page: page, per_page: per_page, desc: desc - } = Enum.into(opts, default_opts) + } = extract_opts(opts) case model_type do "boards.counts" -> From cca5b932cd5e9fa566182c7b0e17662cb7348b71 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 14:58:51 -0800 Subject: [PATCH 180/231] refactor(smf_query): remove board_counts() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/board.ex | 2 +- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index f34cf0c5..8eb490d2 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -30,9 +30,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "boards.counts" -> - build_board_counts() - "boards.last_post_info" -> build_board_last_post_info() @@ -226,7 +223,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_board_counts() do + def board_counts() do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index 031ca132..b3d66573 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -151,7 +151,7 @@ defmodule EpochtalkServerWeb.Controllers.Board do user_priority <- ACL.get_user_priority(conn), board_mapping <- BoardMapping.all(stripped: stripped), {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), - {:ok, board_counts} <- SmfQuery.build_model("boards.counts"), + {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.build_model("boards.last_post_info"), categories <- Category.all() do render(conn, :proxy_by_category, %{ diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 66998e42..7aa350e2 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -828,7 +828,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do :ok <- ACL.allow!(conn, "threads.byBoard"), board_mapping <- BoardMapping.all(), {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), - {:ok, board_counts} <- SmfQuery.build_model("boards.counts"), + {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.build_model("boards.last_post_info"), {:ok, threads, data} <- SmfQuery.build_model("threads.by_board", board_id, %{page: page, limit: limit}) do From ba1e4260d7ee86c07835f5771bda72873f2f6a65 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:02:11 -0800 Subject: [PATCH 181/231] refactor(smf_query): remove board_last_post_info() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/board.ex | 2 +- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 8eb490d2..1cf6681e 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -30,9 +30,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "boards.last_post_info" -> - build_board_last_post_info() - "boards.moderators" -> build_board_moderators() @@ -245,7 +242,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_board_last_post_info() do + def board_last_post_info() do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index b3d66573..d76548f6 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -152,7 +152,7 @@ defmodule EpochtalkServerWeb.Controllers.Board do board_mapping <- BoardMapping.all(stripped: stripped), {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), {:ok, board_counts} <- SmfQuery.board_counts(), - {:ok, board_last_post_info} <- SmfQuery.build_model("boards.last_post_info"), + {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), categories <- Category.all() do render(conn, :proxy_by_category, %{ categories: categories, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 7aa350e2..d533d06f 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -829,7 +829,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do board_mapping <- BoardMapping.all(), {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), {:ok, board_counts} <- SmfQuery.board_counts(), - {:ok, board_last_post_info} <- SmfQuery.build_model("boards.last_post_info"), + {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), {:ok, threads, data} <- SmfQuery.build_model("threads.by_board", board_id, %{page: page, limit: limit}) do render(conn, :by_board_proxy, %{ From d037520097d3ffba0517178c1d8fb4c9f33c4716 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:06:27 -0800 Subject: [PATCH 182/231] refactor(smf_query): remove board_moderators() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/board.ex | 2 +- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 1cf6681e..7ccf7ff9 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -30,9 +30,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "boards.moderators" -> - build_board_moderators() - "threads.recent" -> build_recent_threads() @@ -285,7 +282,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_board_moderators() do + def board_moderators() do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/board.ex b/lib/epochtalk_server_web/controllers/board.ex index d76548f6..7782b3d5 100644 --- a/lib/epochtalk_server_web/controllers/board.ex +++ b/lib/epochtalk_server_web/controllers/board.ex @@ -150,7 +150,7 @@ defmodule EpochtalkServerWeb.Controllers.Board do stripped <- Validate.cast(attrs, "stripped", :boolean, default: false), user_priority <- ACL.get_user_priority(conn), board_mapping <- BoardMapping.all(stripped: stripped), - {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), + {:ok, board_moderators} <- SmfQuery.board_moderators(), {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), categories <- Category.all() do diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index d533d06f..17fd892c 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -827,7 +827,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "threads.byBoard"), board_mapping <- BoardMapping.all(), - {:ok, board_moderators} <- SmfQuery.build_model("boards.moderators"), + {:ok, board_moderators} <- SmfQuery.board_moderators(), {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), {:ok, threads, data} <- From f33fd62ebc682820e886aa1ca5a548e93b9f42bb Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:08:52 -0800 Subject: [PATCH 183/231] refactor(smf_query): remove recent_threads() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 7ccf7ff9..301cf765 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -30,9 +30,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "threads.recent" -> - build_recent_threads() - "category" -> build_category(id) @@ -151,7 +148,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_recent_threads() do + def recent_threads() do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 17fd892c..2712cd68 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -852,7 +852,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do end defp proxy_recent(conn, _attrs) do - with threads <- SmfQuery.build_model("threads.recent") do + with threads <- SmfQuery.recent_threads() do render(conn, :recent, %{threads: threads}) end end From 217fba099f0c032fe05090effe6bf83b6958cad5 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:10:45 -0800 Subject: [PATCH 184/231] refactor(smf_query): remove unused build_model header --- lib/epochtalk_server/smf_query.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 301cf765..a77c65c1 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -20,7 +20,6 @@ defmodule EpochtalkServer.SmfQuery do Enum.into(opts, default_opts) end - def build_model(model_type), do: build_model(model_type, nil, %{}) def build_model(model_type, id), do: build_model(model_type, id, %{}) def build_model(model_type, id, opts) do %{ From 0429eba2fd9330023f08fb21467220009db6b14a Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:13:02 -0800 Subject: [PATCH 185/231] refactor(smf_query): remove category() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index a77c65c1..ad5aca58 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -29,9 +29,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "category" -> - build_category(id) - "board" -> build_board(id) @@ -194,7 +191,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_category(id) do + def category(id) do from(c in "smf_categories", where: c.id_cat == ^id, select: %{ From 40ebb14099a4da2169f14eb1f8873383702bf57c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:14:16 -0800 Subject: [PATCH 186/231] refactor(smf_query): remove board() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index ad5aca58..7fa69def 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -29,9 +29,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "board" -> - build_board(id) - "thread" -> build_thread(id) @@ -301,7 +298,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_board(id) do + def board(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) From 592d0afc05cc2c9f9909eb79a2b75a8538003d1c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:16:50 -0800 Subject: [PATCH 187/231] refactor(smf_query): remove thread() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/post.ex | 2 +- lib/epochtalk_server_web/helpers/breadcrumbs.ex | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 7fa69def..6824d81b 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -29,9 +29,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "thread" -> - build_thread(id) - "post" -> build_post(id) @@ -379,7 +376,7 @@ defmodule EpochtalkServer.SmfQuery do |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: true) end - def build_thread(id) do + def thread(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 1d5fae58..46e74696 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -585,7 +585,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do :ok <- ACL.allow!(conn, "posts.byThread"), board_mapping <- BoardMapping.all(), board_moderators <- BoardModerator.all(), - thread <- SmfQuery.build_model("thread", thread_id), + thread <- SmfQuery.thread(thread_id), poll <- SmfQuery.build_model("poll.by_thread", thread_id), {:ok, posts, data} <- SmfQuery.build_model("posts.by_thread", thread_id, %{page: page, limit: limit}) do diff --git a/lib/epochtalk_server_web/helpers/breadcrumbs.ex b/lib/epochtalk_server_web/helpers/breadcrumbs.ex index 66532a56..a183099c 100644 --- a/lib/epochtalk_server_web/helpers/breadcrumbs.ex +++ b/lib/epochtalk_server_web/helpers/breadcrumbs.ex @@ -76,7 +76,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do thread = cond do is_integer(id) && id < threads_seq -> - SmfQuery.build_model("thread", id) + SmfQuery.thread(id) is_binary(id) -> Thread.find(id) From 9df19c6e53c1e253920ec768eeeb9836629be243 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:17:41 -0800 Subject: [PATCH 188/231] refactor(smf_query): remove post() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 6824d81b..21319528 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -29,9 +29,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "post" -> - build_post(id) - "poll.by_thread" -> build_poll(id) @@ -421,7 +418,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_post(id) do + def post(id) do %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) From 13cef99500fa4e5fd9e54793bdb48865c12c9cc8 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:21:14 -0800 Subject: [PATCH 189/231] refactor(smf_query): remove poll_by_thread() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/post.ex | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 21319528..67b65526 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -29,9 +29,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "poll.by_thread" -> - build_poll(id) - "user.find" -> build_user(id) @@ -93,7 +90,7 @@ defmodule EpochtalkServer.SmfQuery do |> SmfRepo.one() end - def build_poll(thread_id) do + def poll_by_thread(thread_id) do from(t in "smf_topics", where: t.id_topic == ^thread_id ) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 46e74696..2b5c6c72 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -586,7 +586,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do board_mapping <- BoardMapping.all(), board_moderators <- BoardModerator.all(), thread <- SmfQuery.thread(thread_id), - poll <- SmfQuery.build_model("poll.by_thread", thread_id), + poll <- SmfQuery.poll_by_thread(thread_id), {:ok, posts, data} <- SmfQuery.build_model("posts.by_thread", thread_id, %{page: page, limit: limit}) do render(conn, :by_thread_proxy, %{ From e43ae567d719c85f2607fd694c5602eaa2478cfd Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:23:28 -0800 Subject: [PATCH 190/231] refactor(smf_query): remove find_user() from build_model --- lib/epochtalk_server/smf_query.ex | 5 +---- lib/epochtalk_server_web/controllers/user.ex | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 67b65526..8e8e9d10 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -29,9 +29,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "user.find" -> - build_user(id) - "threads.by_board" -> build_threads_by_board(id, page, per_page) @@ -49,7 +46,7 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_user(user_id) do + def find_user(user_id) do from(u in "smf_members", where: u.id_member == ^user_id) |> join(:left, [u], a in "smf_attachments", on: u.id_member == a.id_member and a.attachmentType == 1 diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index 9e86dda8..18efa46e 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -270,7 +270,7 @@ defmodule EpochtalkServerWeb.Controllers.User do end defp proxy_find(conn, attrs) do - with user <- SmfQuery.build_model("user.find", attrs["id"]) do + with user <- SmfQuery.find_user(attrs["id"]) do render(conn, :find_proxy, %{user: user}) end end From a35ff1de55331769744d41657a5848ff5bc3d6e1 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:23:50 -0800 Subject: [PATCH 191/231] refactor(smf_query): remove unused build_model header --- lib/epochtalk_server/smf_query.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 8e8e9d10..ea80ce50 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -20,7 +20,6 @@ defmodule EpochtalkServer.SmfQuery do Enum.into(opts, default_opts) end - def build_model(model_type, id), do: build_model(model_type, id, %{}) def build_model(model_type, id, opts) do %{ page: page, From 554d71a577627f8db13b8ca624f1aae6a8e376bf Mon Sep 17 00:00:00 2001 From: unenglishable Date: Mon, 2 Dec 2024 15:31:11 -0800 Subject: [PATCH 192/231] refactor(smf_query): remove threads_by_board() from build_model --- lib/epochtalk_server/smf_query.ex | 10 ++++++---- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index ea80ce50..e9b56947 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -28,9 +28,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "threads.by_board" -> - build_threads_by_board(id, page, per_page) - "posts.by_thread" -> build_posts_by_thread(id, page, per_page) @@ -311,7 +308,12 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_threads_by_board(id, page, per_page) do + def threads_by_board(id, opts) do + %{ + page: page, + per_page: per_page + } = extract_opts(opts) + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 2712cd68..cde1447d 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -831,7 +831,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), {:ok, threads, data} <- - SmfQuery.build_model("threads.by_board", board_id, %{page: page, limit: limit}) do + SmfQuery.threads_by_board(board_id, %{page: page, limit: limit}) do render(conn, :by_board_proxy, %{ threads: threads, user: user, From b29cd4bd24d873876aa19c8ff9b915909f82c29f Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 3 Dec 2024 10:07:45 -1000 Subject: [PATCH 193/231] refactor(smf_query): remove posts_by_thread() from build_model --- lib/epochtalk_server/smf_query.ex | 10 ++++++---- lib/epochtalk_server_web/controllers/post.ex | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index e9b56947..8cf8e81d 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -28,9 +28,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "posts.by_thread" -> - build_posts_by_thread(id, page, per_page) - "threads.by_user" -> build_threads_by_user(id, page, per_page, desc) @@ -439,7 +436,12 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_posts_by_thread(id, page, per_page) do + def posts_by_thread(id, opts) do + %{ + page: page, + per_page: per_page + } = extract_opts(opts) + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 2b5c6c72..c83c7a96 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -588,7 +588,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do thread <- SmfQuery.thread(thread_id), poll <- SmfQuery.poll_by_thread(thread_id), {:ok, posts, data} <- - SmfQuery.build_model("posts.by_thread", thread_id, %{page: page, limit: limit}) do + SmfQuery.posts_by_thread(thread_id, %{page: page, limit: limit}) do render(conn, :by_thread_proxy, %{ posts: posts, poll: poll, From db7bd19b40401b785f86ceb0ea2972e5830ae4a8 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 3 Dec 2024 21:24:45 -1000 Subject: [PATCH 194/231] refactor(smf_query): remove threads_by_user() from build_model --- lib/epochtalk_server/smf_query.ex | 11 +++++++---- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 8cf8e81d..346afbc7 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -28,9 +28,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "threads.by_user" -> - build_threads_by_user(id, page, per_page, desc) - "posts.by_user" -> build_posts_by_user(id, page, per_page, desc) @@ -528,7 +525,13 @@ defmodule EpochtalkServer.SmfQuery do |> ProxyPagination.page_simple(count_query, page, per_page: per_page, desc: desc) end - def build_threads_by_user(id, page, per_page, desc) do + def threads_by_user(id, opts) do + %{ + page: page, + per_page: per_page, + desc: desc + } = extract_opts(opts) + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index cde1447d..357011bc 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -800,7 +800,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, threads, data} <- - SmfQuery.build_model("threads.by_user", user_id, %{ + SmfQuery.threads_by_user(user_id, %{ page: page, limit: limit, desc: desc From 56acb74d484b35cfd4f6a485f8ece454fcab2bcc Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 3 Dec 2024 21:27:12 -1000 Subject: [PATCH 195/231] refactor(smf_query): remove posts_by_user() from build_model --- lib/epochtalk_server/smf_query.ex | 11 +++++++---- lib/epochtalk_server_web/controllers/post.ex | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 346afbc7..782b0114 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -28,9 +28,6 @@ defmodule EpochtalkServer.SmfQuery do } = extract_opts(opts) case model_type do - "posts.by_user" -> - build_posts_by_user(id, page, per_page, desc) - _ -> {:ok, %{}, %{}} end @@ -492,7 +489,13 @@ defmodule EpochtalkServer.SmfQuery do end end - def build_posts_by_user(id, page, per_page, desc) do + def posts_by_user(id, opts) do + %{ + page: page, + per_page: per_page, + desc: desc + } = extract_opts(opts) + %{id_board_blacklist: id_board_blacklist} = Application.get_env(:epochtalk_server, :proxy_config) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index c83c7a96..7acd0732 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -558,7 +558,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), desc <- Validate.cast(attrs, "desc", :boolean, default: true), {:ok, posts, data} <- - SmfQuery.build_model("posts.by_user", user_id, %{ + SmfQuery.posts_by_user(user_id, %{ page: page, limit: limit, desc: desc From 721796222dbc4a164648d12b337429a61a0dd44c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 3 Dec 2024 21:32:59 -1000 Subject: [PATCH 196/231] refactor(controllers): limit -> per_page correct key name, since it matters now (args are passed as opts instead of args) --- lib/epochtalk_server_web/controllers/post.ex | 4 ++-- lib/epochtalk_server_web/controllers/thread.ex | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 7acd0732..5808d4c7 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -560,7 +560,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do {:ok, posts, data} <- SmfQuery.posts_by_user(user_id, %{ page: page, - limit: limit, + per_page: per_page, desc: desc }) do render(conn, :proxy_by_username, %{ @@ -588,7 +588,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do thread <- SmfQuery.thread(thread_id), poll <- SmfQuery.poll_by_thread(thread_id), {:ok, posts, data} <- - SmfQuery.posts_by_thread(thread_id, %{page: page, limit: limit}) do + SmfQuery.posts_by_thread(thread_id, %{page: page, per_page: per_page}) do render(conn, :by_thread_proxy, %{ posts: posts, poll: poll, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 357011bc..28494a4f 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -802,7 +802,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:ok, threads, data} <- SmfQuery.threads_by_user(user_id, %{ page: page, - limit: limit, + per_page: per_page, desc: desc }) do render(conn, :proxy_by_username, %{ @@ -831,7 +831,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), {:ok, threads, data} <- - SmfQuery.threads_by_board(board_id, %{page: page, limit: limit}) do + SmfQuery.threads_by_board(board_id, %{page: page, per_page: per_page}) do render(conn, :by_board_proxy, %{ threads: threads, user: user, From fdc1ce21d2df6410e2c4d838fa9597f200b83a1c Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 3 Dec 2024 21:34:21 -1000 Subject: [PATCH 197/231] refactor(smf_query): remove unused build_model header --- lib/epochtalk_server/smf_query.ex | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index 782b0114..cf9cb0e5 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -20,18 +20,6 @@ defmodule EpochtalkServer.SmfQuery do Enum.into(opts, default_opts) end - def build_model(model_type, id, opts) do - %{ - page: page, - per_page: per_page, - desc: desc - } = extract_opts(opts) - - case model_type do - _ -> - {:ok, %{}, %{}} - end - end def find_user(user_id) do from(u in "smf_members", where: u.id_member == ^user_id) From 819e77fc18624ceeadd8e8b7c02b98cba194ae0e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 3 Dec 2024 21:36:18 -1000 Subject: [PATCH 198/231] refactor(controllers): pass limit into per_page --- lib/epochtalk_server_web/controllers/post.ex | 4 ++-- lib/epochtalk_server_web/controllers/thread.ex | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index 5808d4c7..f08b6c04 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -560,7 +560,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do {:ok, posts, data} <- SmfQuery.posts_by_user(user_id, %{ page: page, - per_page: per_page, + per_page: limit, desc: desc }) do render(conn, :proxy_by_username, %{ @@ -588,7 +588,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do thread <- SmfQuery.thread(thread_id), poll <- SmfQuery.poll_by_thread(thread_id), {:ok, posts, data} <- - SmfQuery.posts_by_thread(thread_id, %{page: page, per_page: per_page}) do + SmfQuery.posts_by_thread(thread_id, %{page: page, per_page: limit}) do render(conn, :by_thread_proxy, %{ posts: posts, poll: poll, diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 28494a4f..396f2479 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -802,7 +802,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:ok, threads, data} <- SmfQuery.threads_by_user(user_id, %{ page: page, - per_page: per_page, + per_page: limit, desc: desc }) do render(conn, :proxy_by_username, %{ @@ -831,7 +831,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do {:ok, board_counts} <- SmfQuery.board_counts(), {:ok, board_last_post_info} <- SmfQuery.board_last_post_info(), {:ok, threads, data} <- - SmfQuery.threads_by_board(board_id, %{page: page, per_page: per_page}) do + SmfQuery.threads_by_board(board_id, %{page: page, per_page: limit}) do render(conn, :by_board_proxy, %{ threads: threads, user: user, From e73fffc39a7ccb5340abee6da1a3ebf154156c88 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 5 Dec 2024 19:43:04 -1000 Subject: [PATCH 199/231] fix(controllers/user): remove {:auth, false} case from logout fixes dialyzer unreachable case --- lib/epochtalk_server_web/controllers/user.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/epochtalk_server_web/controllers/user.ex b/lib/epochtalk_server_web/controllers/user.ex index 6a42b614..a496895e 100644 --- a/lib/epochtalk_server_web/controllers/user.ex +++ b/lib/epochtalk_server_web/controllers/user.ex @@ -205,7 +205,6 @@ defmodule EpochtalkServerWeb.Controllers.User do EpochtalkServerWeb.Endpoint.broadcast("user:#{user.id}", "logout", %{token: token}) render(conn, :data, data: %{success: true}) else - {:auth, false} -> ErrorHelpers.render_json_error(conn, 400, "Not logged in") {:error, error} -> ErrorHelpers.render_json_error(conn, 500, error) end end From b5e7ec7de2247cbf12405320355daee235a6d270 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 10 Dec 2024 11:03:23 -1000 Subject: [PATCH 200/231] refactor(config/runtime): update bbc parser configs default workers lower for dev/test allow configuration for prod --- config/runtime.exs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index bc1cb528..fa526617 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -407,11 +407,25 @@ end ##### PROXY REPO CONFIGURATIONS ##### +# configure poolboy for bbcode parser +bbc_config = + case config_env() do + :prod -> + %{ + size: get_env_cast_integer_with_default.("BBC_PARSER_WORKERS", "50"), + max_overflow: get_env_cast_integer_with_default.("BBC_PARSER_OVERFLOW", "20") + } + _ -> + %{ + size: 5, + max_overflow: 2 + } + end bbc_parser_poolboy_config = [ name: {:local, :bbc_parser}, worker_module: EpochtalkServer.BBCParser, - size: 50, - max_overflow: 2, + size: bbc_config.size, + max_overflow: bbc_config.max_overflow, strategy: :fifo ] From e7a6eedc036c8723059eeca458f0c551b4bdfa11 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 10 Dec 2024 11:19:50 -1000 Subject: [PATCH 201/231] style(config/runtime): mix format --- config/runtime.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/runtime.exs b/config/runtime.exs index fa526617..13418ebb 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -415,12 +415,14 @@ bbc_config = size: get_env_cast_integer_with_default.("BBC_PARSER_WORKERS", "50"), max_overflow: get_env_cast_integer_with_default.("BBC_PARSER_OVERFLOW", "20") } + _ -> %{ size: 5, max_overflow: 2 } end + bbc_parser_poolboy_config = [ name: {:local, :bbc_parser}, worker_module: EpochtalkServer.BBCParser, From bef1c3d8490dc759c14ef9ee1ba404785eb7e079 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:18:49 -1000 Subject: [PATCH 202/231] perf(bbc_parser): don't block with receive when starting bbcode parsers handle_info will receive asynchronously and print when parsers have started --- lib/epochtalk_server/bbc_parser.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 17142f9e..13d3cda7 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -18,6 +18,12 @@ defmodule EpochtalkServer.BBCParser do @impl true def init(:ok), do: {:ok, load()} + @impl true + def handle_info({_pid, :data, :out, data}, state) do + Logger.debug("#{__MODULE__}(info): #{inspect(data)}") + {:noreply, state} + end + @impl true def handle_call({:parse, ""}, _from, {proc, pid}), do: {:reply, {:ok, ""}, {proc, pid}} @@ -88,9 +94,6 @@ defmodule EpochtalkServer.BBCParser do Proc.send_input(proc, "require 'parsing.php';\n") Logger.debug("#{__MODULE__}(LOAD): #{inspect(pid)}") # clear initial php interactive shell message - receive do - {^pid, :data, :out, data} -> Logger.debug("#{__MODULE__}: #{inspect(data)}") - end {proc, pid} end From e4e269f37b89d366ead2bf08ffc6e0979e76866e Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:24:02 -1000 Subject: [PATCH 203/231] refactor(bbc_parser): split out parse with proc functionality can be reused later by batch processing functions --- lib/epochtalk_server/bbc_parser.ex | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 13d3cda7..6e1c7b17 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -29,17 +29,29 @@ defmodule EpochtalkServer.BBCParser do do: {:reply, {:ok, ""}, {proc, pid}} def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do - Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") + Logger.debug("#{__MODULE__}(start parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + parsed = parse_with_proc(bbcode_data, {proc, pid}) + Logger.debug("#{__MODULE__}(finish parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + {:reply, parsed, {proc, pid}} + end - parsed = - receive do - {^pid, :data, :out, data} -> {:ok, data} - after - # time out after not receiving any data - @receive_timeout -> {:timeout, bbcode_data} - end + defp parse_with_proc(nil, {_proc, _pid}), do: {:ok, nil} + defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} + defp parse_with_proc(bbcode_data, {proc, pid}) do + Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") - {:reply, parsed, {proc, pid}} + receive do + {^pid, :data, :out, data} -> + {:ok, data} + after + # time out after not receiving any data + @receive_timeout -> + Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(bbcode_data)}") + bbcode_data = + "

    ((bbcode parse timeout))


    " <> + bbcode_data + {:timeout, bbcode_data} + end end ## === parser api functions ==== From 6ef431a87813e2cb5bdd2044e6fa7d572b165bd7 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:24:47 -1000 Subject: [PATCH 204/231] feat(bbc_parser): use parse_with_proc for parsing list and list tuple --- lib/epochtalk_server/bbc_parser.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 6e1c7b17..86a8ea00 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -35,6 +35,17 @@ defmodule EpochtalkServer.BBCParser do {:reply, parsed, {proc, pid}} end + defp parse_list_tuple_with_proc({left, right}, {proc, pid}) do + left = parse_list_with_proc(left, {proc, pid}) + right = parse_list_with_proc(right, {proc, pid}) + {left, right} + end + + defp parse_list_with_proc(bbcode_data_list, {proc, pid}) do + bbcode_data_list + |> Enum.map(&(parse_with_proc(&1, {proc, pid}))) + end + defp parse_with_proc(nil, {_proc, _pid}), do: {:ok, nil} defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} defp parse_with_proc(bbcode_data, {proc, pid}) do From f27dbfc163f0113cad06c433e8d1e77b5c18eff4 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:26:16 -1000 Subject: [PATCH 205/231] refactor(bbc_parser): remove timeout error handling from parse convenience function moved to parse_with_proc --- lib/epochtalk_server/bbc_parser.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 86a8ea00..3bf08700 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -90,10 +90,7 @@ defmodule EpochtalkServer.BBCParser do # on parse timeout, log and return unparsed data {:timeout, unparsed} -> - Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(unparsed)}") - - "

    ((bbcode parse timeout))


    " <> - unparsed + unparsed end catch e, r -> From 81b1bf7251a57f7ca198830db43c6d90ec43e201 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:29:13 -1000 Subject: [PATCH 206/231] feat(post_json): implement format_proxy_posts_for_by_thread extracts bodies and signatures as lists from post will be processed later as a batch --- lib/epochtalk_server_web/json/post_json.ex | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 99611f13..47372606 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -444,6 +444,31 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do |> Map.delete(:role_name) end + defp format_proxy_posts_for_by_thread(posts) do + # extract body/signature lists from posts + {body_list, signature_list} = + posts + |> Enum.reduce({[], []}, fn post, {body_list, signature_list} -> + body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") + + # add space to end if the last character is a backslash (fix for parser) + body_len = String.length(body) + last_char = String.slice(body, (body_len - 1)..body_len) + body = if last_char == "\\", do: body <> " ", else: body + + signature = + if Map.get(post.user, :signature), + do: String.replace(post.user.signature, "'", "\'"), + else: nil + + # return body/signature lists in reverse order + {[body | body_list], [signature | signature_list]} + end) + + # reverse body/signature lists + {body_list, signature_list} = {Enum.reverse(body_list), Enum.reverse(signature_list)} + end + defp format_proxy_post_data_for_by_thread(post) do body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") From 76adcf60a4bd6585390632728c20be1d472c1fa9 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:30:41 -1000 Subject: [PATCH 207/231] feat(bbc_parser): implement call handler for parsing list tuple --- lib/epochtalk_server/bbc_parser.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 3bf08700..2d02ef31 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -35,6 +35,13 @@ defmodule EpochtalkServer.BBCParser do {:reply, parsed, {proc, pid}} end + def handle_call({:parse_list_tuple, {left_list, right_list}}, _from, {proc, pid}) do + Logger.debug("#{__MODULE__}(start parse list tuple): #{NaiveDateTime.utc_now()}") + parsed = parse_list_tuple_with_proc({left_list, right_list}, {proc, pid}) + Logger.debug("#{__MODULE__}(finish parse list tuple): #{NaiveDateTime.utc_now()}") + {:reply, {:ok, parsed}, {proc, pid}} + end + defp parse_list_tuple_with_proc({left, right}, {proc, pid}) do left = parse_list_with_proc(left, {proc, pid}) right = parse_list_with_proc(right, {proc, pid}) From 81469e5ece036e1cb06e9567ef5a66a860dea6d8 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:32:25 -1000 Subject: [PATCH 208/231] feat(bbc_parser): implement list tuple parser batch handler uses a single process to parse all posts/signatures for a thread page --- lib/epochtalk_server/bbc_parser.ex | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 2d02ef31..6f32a1e5 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -82,6 +82,27 @@ defmodule EpochtalkServer.BBCParser do @doc """ Uses poolboy to call parser """ + def parse_list_tuple({left_bbcode_data, right_bbcode_data}) do + :poolboy.transaction( + :bbc_parser, + fn pid -> + try do + Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") + + GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}) + catch + e, r -> + # something went wrong, log the error + Logger.error( + "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" + ) + + {:error, {left_bbcode_data, right_bbcode_data}} + end + end, + @call_timeout + ) + end def parse(bbcode_data) do :poolboy.transaction( :bbc_parser, From c3836c9dcc83ecf668451894a9d755c60b90d0e0 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:34:21 -1000 Subject: [PATCH 209/231] feat(post_json): use BBCParser.parse_list_tuple on body and signature lists batch process bbcode parsing --- lib/epochtalk_server_web/json/post_json.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 47372606..39580609 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -467,6 +467,17 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do # reverse body/signature lists {body_list, signature_list} = {Enum.reverse(body_list), Enum.reverse(signature_list)} + + # parse body/signature lists + {parsed_body_list, parsed_signature_list} = + {body_list, signature_list} + |> EpochtalkServer.BBCParser.parse_list_tuple() + |> case do + {:ok, parsed_tuple} -> parsed_tuple + {:error, unparsed_tuple} -> + Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") + unparsed_tuple + end end defp format_proxy_post_data_for_by_thread(post) do From 07124211ddd1738bf5fea2d7cc5e1a5e42ebae17 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:36:09 -1000 Subject: [PATCH 210/231] feat(post_json): zip parsed body/signature data back into posts --- lib/epochtalk_server_web/json/post_json.ex | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 39580609..a5eb545b 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -478,6 +478,29 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") unparsed_tuple end + + # zip posts with body/signature lists + Enum.zip_with( + [posts, parsed_body_list, parsed_signature_list], + fn [post, parsed_body, parsed_signature] -> + parsed_body = case parsed_body do + {:ok, parsed_body} -> + parsed_body + {:timeout, unparsed_body} -> + unparsed_body + end + parsed_signature = case parsed_signature do + {:ok, parsed_signature} -> + parsed_signature + {:timeout, unparsed_signature} -> + unparsed_signature + end + user = post.user |> Map.put(:signature, parsed_signature) + post + |> Map.put(:body_html, parsed_body) + |> Map.put(:user, user) + end + ) end defp format_proxy_post_data_for_by_thread(post) do From 43c01028e307ab4a9989a36f6ea56edbc1a7d075 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:36:51 -1000 Subject: [PATCH 211/231] feat(post_json): add logging to batch processing/zip --- lib/epochtalk_server_web/json/post_json.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index a5eb545b..f82fe4d9 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -2,6 +2,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do alias EpochtalkServerWeb.Controllers.BoardJSON alias EpochtalkServerWeb.Controllers.ThreadJSON alias EpochtalkServerWeb.Helpers.ACL + require Logger @moduledoc """ Renders and formats `Post` data, in JSON format for frontend @@ -485,14 +486,18 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do fn [post, parsed_body, parsed_signature] -> parsed_body = case parsed_body do {:ok, parsed_body} -> + Logger.debug("#{__MODULE__}(body): post_id #{inspect(post.id)}") parsed_body {:timeout, unparsed_body} -> + Logger.error("#{__MODULE__}(body timeout): post_id #{inspect(post.id)}") unparsed_body end parsed_signature = case parsed_signature do {:ok, parsed_signature} -> + Logger.debug("#{__MODULE__}(signature): user_id #{inspect(post.user.id)}") parsed_signature {:timeout, unparsed_signature} -> + Logger.error("#{__MODULE__}(signature timeout): user_id #{inspect(post.user.id)}") unparsed_signature end user = post.user |> Map.put(:signature, parsed_signature) From e90317d1535771a7a52d65223b1625322c1f1866 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:37:29 -1000 Subject: [PATCH 212/231] perf(post_json): use batch processor for formatting proxy posts --- lib/epochtalk_server_web/json/post_json.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index f82fe4d9..761011b1 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -141,7 +141,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do # format post data posts = posts - |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) + |> format_proxy_posts_for_by_thread() # build by_thread results %{ @@ -193,7 +193,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do when is_list(posts) do posts = posts - |> Enum.map(&format_proxy_post_data_for_by_thread(&1)) + |> format_proxy_posts_for_by_thread() %{ posts: posts, From 4a5e52fe96b663ad4b0aa4e551b81e746e5e2a11 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:39:01 -1000 Subject: [PATCH 213/231] refactor(post_json): remove unused proxy format function --- lib/epochtalk_server_web/json/post_json.ex | 24 ---------------------- 1 file changed, 24 deletions(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 761011b1..9859212f 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -507,28 +507,4 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do end ) end - - defp format_proxy_post_data_for_by_thread(post) do - body = String.replace(Map.get(post, :body) || Map.get(post, :body_html), "'", "\'") - - # add space to end if the last character is a backslash (fix for parser) - body_len = String.length(body) - last_char = String.slice(body, (body_len - 1)..body_len) - body = if last_char == "\\", do: body <> " ", else: body - - parsed_body = EpochtalkServer.BBCParser.parse(body) - - signature = - if Map.get(post.user, :signature), - do: String.replace(post.user.signature, "'", "\'"), - else: nil - - parsed_signature = - if signature, - do: EpochtalkServer.BBCParser.parse(signature), - else: nil - - user = post.user |> Map.put(:signature, parsed_signature) - post |> Map.put(:body_html, parsed_body) |> Map.put(:user, user) - end end From a4991fa258eefec95f9a65c038bfd62d45477c72 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:40:12 -1000 Subject: [PATCH 214/231] style(post_json,bbc_parser): mix format --- lib/epochtalk_server/bbc_parser.ex | 17 ++++++-- lib/epochtalk_server_web/json/post_json.ex | 47 +++++++++++++--------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 6f32a1e5..c42e5bf7 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -29,9 +29,16 @@ defmodule EpochtalkServer.BBCParser do do: {:reply, {:ok, ""}, {proc, pid}} def handle_call({:parse, bbcode_data}, _from, {proc, pid}) when is_binary(bbcode_data) do - Logger.debug("#{__MODULE__}(start parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + Logger.debug( + "#{__MODULE__}(start parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}" + ) + parsed = parse_with_proc(bbcode_data, {proc, pid}) - Logger.debug("#{__MODULE__}(finish parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}") + + Logger.debug( + "#{__MODULE__}(finish parse): #{String.first(bbcode_data)} #{NaiveDateTime.utc_now()}" + ) + {:reply, parsed, {proc, pid}} end @@ -50,11 +57,12 @@ defmodule EpochtalkServer.BBCParser do defp parse_list_with_proc(bbcode_data_list, {proc, pid}) do bbcode_data_list - |> Enum.map(&(parse_with_proc(&1, {proc, pid}))) + |> Enum.map(&parse_with_proc(&1, {proc, pid})) end defp parse_with_proc(nil, {_proc, _pid}), do: {:ok, nil} defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} + defp parse_with_proc(bbcode_data, {proc, pid}) do Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") @@ -65,9 +73,11 @@ defmodule EpochtalkServer.BBCParser do # time out after not receiving any data @receive_timeout -> Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(bbcode_data)}") + bbcode_data = "

    ((bbcode parse timeout))


    " <> bbcode_data + {:timeout, bbcode_data} end end @@ -103,6 +113,7 @@ defmodule EpochtalkServer.BBCParser do @call_timeout ) end + def parse(bbcode_data) do :poolboy.transaction( :bbc_parser, diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 9859212f..6c66f6bf 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -474,7 +474,9 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do {body_list, signature_list} |> EpochtalkServer.BBCParser.parse_list_tuple() |> case do - {:ok, parsed_tuple} -> parsed_tuple + {:ok, parsed_tuple} -> + parsed_tuple + {:error, unparsed_tuple} -> Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") unparsed_tuple @@ -484,26 +486,33 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do Enum.zip_with( [posts, parsed_body_list, parsed_signature_list], fn [post, parsed_body, parsed_signature] -> - parsed_body = case parsed_body do - {:ok, parsed_body} -> - Logger.debug("#{__MODULE__}(body): post_id #{inspect(post.id)}") - parsed_body - {:timeout, unparsed_body} -> - Logger.error("#{__MODULE__}(body timeout): post_id #{inspect(post.id)}") - unparsed_body - end - parsed_signature = case parsed_signature do - {:ok, parsed_signature} -> - Logger.debug("#{__MODULE__}(signature): user_id #{inspect(post.user.id)}") - parsed_signature - {:timeout, unparsed_signature} -> - Logger.error("#{__MODULE__}(signature timeout): user_id #{inspect(post.user.id)}") - unparsed_signature - end + parsed_body = + case parsed_body do + {:ok, parsed_body} -> + Logger.debug("#{__MODULE__}(body): post_id #{inspect(post.id)}") + parsed_body + + {:timeout, unparsed_body} -> + Logger.error("#{__MODULE__}(body timeout): post_id #{inspect(post.id)}") + unparsed_body + end + + parsed_signature = + case parsed_signature do + {:ok, parsed_signature} -> + Logger.debug("#{__MODULE__}(signature): user_id #{inspect(post.user.id)}") + parsed_signature + + {:timeout, unparsed_signature} -> + Logger.error("#{__MODULE__}(signature timeout): user_id #{inspect(post.user.id)}") + unparsed_signature + end + user = post.user |> Map.put(:signature, parsed_signature) + post - |> Map.put(:body_html, parsed_body) - |> Map.put(:user, user) + |> Map.put(:body_html, parsed_body) + |> Map.put(:user, user) end ) end From 885396a7c6c2ff3267c818ba6dbc7c3f95322484 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 12:59:13 -1000 Subject: [PATCH 215/231] refactor(bbc_parser): update timeouts split genserver and poolboy timeouts reduce receive timeout to ensure quick loading --- lib/epochtalk_server/bbc_parser.ex | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index c42e5bf7..8b300fbd 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -3,11 +3,15 @@ defmodule EpochtalkServer.BBCParser do require Logger alias Porcelain.Process, as: Proc - # poolboy genserver call timeout (ms) - # should be greater than internal porcelain php call - @call_timeout 500 + # genserver call timeouts (ms) + @genserver_parse_timeout 5000 + @genserver_parse_tuple_timeout 5000 + + # poolboy timeout (ms) + @poolboy_transaction_timeout 200 + # porcelain php parser call timeout (ms) - @receive_timeout 400 + @receive_timeout 20 @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser @@ -99,7 +103,7 @@ defmodule EpochtalkServer.BBCParser do try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}) + GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}, @genserver_parse_tuple_timeout) catch e, r -> # something went wrong, log the error @@ -110,7 +114,7 @@ defmodule EpochtalkServer.BBCParser do {:error, {left_bbcode_data, right_bbcode_data}} end end, - @call_timeout + @poolboy_transaction_timeout ) end @@ -121,7 +125,7 @@ defmodule EpochtalkServer.BBCParser do try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse, bbcode_data}, @call_timeout) + GenServer.call(pid, {:parse, bbcode_data}, @genserver_parse_timeout) |> case do # on success, return parsed data {:ok, parsed} -> @@ -141,7 +145,7 @@ defmodule EpochtalkServer.BBCParser do bbcode_data end end, - @call_timeout + @poolboy_transaction_timeout ) end From 80a30e5d3a75b1d41908eef484f05163569a6173 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 13:20:16 -1000 Subject: [PATCH 216/231] fix(bbc_parser): add :timeout atom to genserver tuple timeout return --- lib/epochtalk_server/bbc_parser.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 8b300fbd..1e21d219 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -111,6 +111,8 @@ defmodule EpochtalkServer.BBCParser do "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" ) + left_bbcode_data = left_bbcode_data |> Enum.map(&({:timeout, &1})) + right_bbcode_data = right_bbcode_data |> Enum.map(&({:timeout, &1})) {:error, {left_bbcode_data, right_bbcode_data}} end end, From 3b8b579f8df5a9cb380bdd53eab21700ead32db4 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 13:21:10 -1000 Subject: [PATCH 217/231] perf(bbc_parser): reduce genserver tuple call timeout don't allow call to go over half a second --- lib/epochtalk_server/bbc_parser.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 1e21d219..ec2e01e0 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -5,7 +5,7 @@ defmodule EpochtalkServer.BBCParser do # genserver call timeouts (ms) @genserver_parse_timeout 5000 - @genserver_parse_tuple_timeout 5000 + @genserver_parse_tuple_timeout 500 # poolboy timeout (ms) @poolboy_transaction_timeout 200 From c921a2683e290b9eb3643499707665b6388d1d47 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 13:21:42 -1000 Subject: [PATCH 218/231] refactor(post_json): log unparsed tuple instead of post on error --- lib/epochtalk_server_web/json/post_json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index 6c66f6bf..c56edbea 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -478,7 +478,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do parsed_tuple {:error, unparsed_tuple} -> - Logger.error("#{__MODULE__}(tuple parse): #{inspect(posts)}") + Logger.error("#{__MODULE__}(tuple parse): #{inspect(unparsed_tuple)}") unparsed_tuple end From 288e2150f4f54a32d5b96e83d0462673b65ec0e1 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 12 Dec 2024 16:06:50 -1000 Subject: [PATCH 219/231] feat(script): implement script to sequentially hit by thread route and log when there is an error --- parse_by_thread.exs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 parse_by_thread.exs diff --git a/parse_by_thread.exs b/parse_by_thread.exs new file mode 100644 index 00000000..7aa1d315 --- /dev/null +++ b/parse_by_thread.exs @@ -0,0 +1,28 @@ +Mix.install([{:httpoison, "~> 2.0"}, {:logger_file_backend, "~> 0.0.10"},]) + +defmodule ParseByThread do + require Logger + + Logger.add_backend {LoggerFileBackend, :error} + Logger.configure_backend {LoggerFileBackend, :error}, + path: "./parse_by_thread_error.log", + level: :error + + @start_id 5_485_059 + @end_id 1 + + def run do + for id <- @start_id..@end_id//-1 do + case HTTPoison.get("http://localhost:4000/api/posts?thread_id=#{id}") do + {:ok, %HTTPoison.Response{status_code: 200, body: _body}} -> + Logger.info("Successfully parsed thread with id (#{id})") + {:ok, %HTTPoison.Response{status_code: status_code}} -> + Logger.error("Thread with id (#{id}) received response with status code #{status_code}") + {:error, %HTTPoison.Error{reason: reason}} -> + Logger.error("Thread with id (#{id}) HTTP request failed: #{reason}") + end + end + end +end + +ParseByThread.run() From cf4bd4991a7ee451a77d92345addd0590d90fee9 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Thu, 12 Dec 2024 23:03:20 -1000 Subject: [PATCH 220/231] style(bbc_parser): mix format --- lib/epochtalk_server/bbc_parser.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index ec2e01e0..ab607196 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -103,7 +103,11 @@ defmodule EpochtalkServer.BBCParser do try do Logger.debug("#{__MODULE__}(parse): #{inspect(pid)}") - GenServer.call(pid, {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}, @genserver_parse_tuple_timeout) + GenServer.call( + pid, + {:parse_list_tuple, {left_bbcode_data, right_bbcode_data}}, + @genserver_parse_tuple_timeout + ) catch e, r -> # something went wrong, log the error @@ -111,8 +115,8 @@ defmodule EpochtalkServer.BBCParser do "#{__MODULE__}(parse poolboy): #{inspect(pid)}, #{inspect(e)}, #{inspect(r)}" ) - left_bbcode_data = left_bbcode_data |> Enum.map(&({:timeout, &1})) - right_bbcode_data = right_bbcode_data |> Enum.map(&({:timeout, &1})) + left_bbcode_data = left_bbcode_data |> Enum.map(&{:timeout, &1}) + right_bbcode_data = right_bbcode_data |> Enum.map(&{:timeout, &1}) {:error, {left_bbcode_data, right_bbcode_data}} end end, From 8f51391e8fe13923ed6d5993d7c9f49e60a50019 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Fri, 13 Dec 2024 11:15:00 -1000 Subject: [PATCH 221/231] fix(format): resolve error caused by format --- parse_by_thread.exs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parse_by_thread.exs b/parse_by_thread.exs index 7aa1d315..2170bee7 100644 --- a/parse_by_thread.exs +++ b/parse_by_thread.exs @@ -1,12 +1,14 @@ -Mix.install([{:httpoison, "~> 2.0"}, {:logger_file_backend, "~> 0.0.10"},]) +Mix.install([{:httpoison, "~> 2.0"}, {:logger_file_backend, "~> 0.0.10"}]) defmodule ParseByThread do require Logger - Logger.add_backend {LoggerFileBackend, :error} - Logger.configure_backend {LoggerFileBackend, :error}, + Logger.add_backend({LoggerFileBackend, :error}) + + Logger.configure_backend({LoggerFileBackend, :error}, path: "./parse_by_thread_error.log", level: :error + ) @start_id 5_485_059 @end_id 1 @@ -16,8 +18,10 @@ defmodule ParseByThread do case HTTPoison.get("http://localhost:4000/api/posts?thread_id=#{id}") do {:ok, %HTTPoison.Response{status_code: 200, body: _body}} -> Logger.info("Successfully parsed thread with id (#{id})") + {:ok, %HTTPoison.Response{status_code: status_code}} -> Logger.error("Thread with id (#{id}) received response with status code #{status_code}") + {:error, %HTTPoison.Error{reason: reason}} -> Logger.error("Thread with id (#{id}) HTTP request failed: #{reason}") end From cfefcd6ed17bfa205e0d04a64d5e510e8e56f0f3 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 17 Dec 2024 10:35:27 -1000 Subject: [PATCH 222/231] refactor(post_json): extract zip posts function fixes credo complexity issue --- lib/epochtalk_server_web/json/post_json.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index c56edbea..e5545e3f 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -482,6 +482,9 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do unparsed_tuple end + zip_posts(posts, parsed_body_list, parsed_signature_list) + end + defp zip_posts(posts, parsed_body_list, parsed_signature_list) do # zip posts with body/signature lists Enum.zip_with( [posts, parsed_body_list, parsed_signature_list], From 857f2755c571423896c72e97c351b54caf1675ae Mon Sep 17 00:00:00 2001 From: unenglishable Date: Tue, 17 Dec 2024 10:36:24 -1000 Subject: [PATCH 223/231] style(post_json): mix format --- lib/epochtalk_server_web/json/post_json.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/epochtalk_server_web/json/post_json.ex b/lib/epochtalk_server_web/json/post_json.ex index e5545e3f..1a845943 100644 --- a/lib/epochtalk_server_web/json/post_json.ex +++ b/lib/epochtalk_server_web/json/post_json.ex @@ -484,6 +484,7 @@ defmodule EpochtalkServerWeb.Controllers.PostJSON do zip_posts(posts, parsed_body_list, parsed_signature_list) end + defp zip_posts(posts, parsed_body_list, parsed_signature_list) do # zip posts with body/signature lists Enum.zip_with( From 2bd18c4daf23c9a7680756e5c138fec0565aa6c2 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 19 Dec 2024 15:44:46 -1000 Subject: [PATCH 224/231] feat(page-threads): make script that hits thread hit each page of a thread --- lib/epochtalk_server/smf_query.ex | 3 ++ lib/epochtalk_server_web/controllers/post.ex | 5 +++- .../helpers/proxy_pagination.ex | 28 +++++++++++-------- parse_by_thread.exs | 25 +++++++++++++---- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/lib/epochtalk_server/smf_query.ex b/lib/epochtalk_server/smf_query.ex index cf9cb0e5..cbfcd1e2 100644 --- a/lib/epochtalk_server/smf_query.ex +++ b/lib/epochtalk_server/smf_query.ex @@ -472,6 +472,9 @@ defmodule EpochtalkServer.SmfQuery do {:ok, [], _} -> {:error, "Posts not found for thread_id: #{id}"} + {:error, :page_does_not_exist} -> + {:error, :page_does_not_exist} + {:ok, posts, data} -> return_tuple(posts, data) end diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index f08b6c04..abf500fd 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -579,7 +579,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do defp proxy_by_thread(conn, attrs) do with thread_id <- Validate.cast(attrs, "thread_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), - limit <- Validate.cast(attrs, "limit", :integer, default: 5), + limit <- Validate.cast(attrs, "limit", :integer, default: 25), user <- Guardian.Plug.current_resource(conn), user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "posts.byThread"), @@ -602,6 +602,9 @@ defmodule EpochtalkServerWeb.Controllers.Post do pagination_data: data }) else + {:error, :page_does_not_exist} -> + ErrorHelpers.render_json_error(conn, 400, "Error, page does not exist") + _ -> ErrorHelpers.render_json_error(conn, 400, "Error, cannot get posts by thread") end diff --git a/lib/epochtalk_server_web/helpers/proxy_pagination.ex b/lib/epochtalk_server_web/helpers/proxy_pagination.ex index a2d239ea..73106bed 100644 --- a/lib/epochtalk_server_web/helpers/proxy_pagination.ex +++ b/lib/epochtalk_server_web/helpers/proxy_pagination.ex @@ -85,19 +85,23 @@ defmodule EpochtalkServerWeb.Helpers.ProxyPagination do total_pages = total_pages(total_records, per_page) - result = records(query, page, total_pages, per_page) - - pagination_data = %{ - next: page < total_pages, - prev: page > 1, - page: page, - per_page: per_page, - total_records: total_records, - total_pages: total_pages, - desc: desc - } + if page > total_pages do + {:error, :page_does_not_exist} + else + result = records(query, page, total_pages, per_page) + + pagination_data = %{ + next: page < total_pages, + prev: page > 1, + page: page, + per_page: per_page, + total_records: total_records, + total_pages: total_pages, + desc: desc + } - {:ok, result, pagination_data} + {:ok, result, pagination_data} + end end def page_next_prev(query, page, per_page: per_page, desc: desc) do diff --git a/parse_by_thread.exs b/parse_by_thread.exs index 2170bee7..c15713eb 100644 --- a/parse_by_thread.exs +++ b/parse_by_thread.exs @@ -12,20 +12,35 @@ defmodule ParseByThread do @start_id 5_485_059 @end_id 1 + @max_page 999_999 - def run do - for id <- @start_id..@end_id//-1 do - case HTTPoison.get("http://localhost:4000/api/posts?thread_id=#{id}") do + def run, + do: for(id <- @start_id..@end_id//-1, do: process_thread(id)) + + def process_thread(id) do + Enum.reduce_while(1..@max_page, nil, fn page, _acc -> + case HTTPoison.get("http://localhost:4000/api/posts?thread_id=#{id}&page=#{page}") do {:ok, %HTTPoison.Response{status_code: 200, body: _body}} -> - Logger.info("Successfully parsed thread with id (#{id})") + {:cont, nil} + + {:ok, + %HTTPoison.Response{ + status_code: 400, + body: + "{\"error\":\"Bad Request\",\"message\":\"Error, page does not exist\",\"status\":400}" + }} -> + Logger.info("Successfully parsed #{page - 1} page(s) of thread with id (#{id})") + {:halt, nil} {:ok, %HTTPoison.Response{status_code: status_code}} -> Logger.error("Thread with id (#{id}) received response with status code #{status_code}") + {:halt, nil} {:error, %HTTPoison.Error{reason: reason}} -> Logger.error("Thread with id (#{id}) HTTP request failed: #{reason}") + {:halt, nil} end - end + end) end end From 1313520355a704e1a2a96fb539d685e8ab391fd6 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 20 Dec 2024 10:05:10 -1000 Subject: [PATCH 225/231] refactor(config/runtime): section off poolboy configs for bbc_parser --- config/runtime.exs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 13418ebb..886fe517 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -407,27 +407,32 @@ end ##### PROXY REPO CONFIGURATIONS ##### -# configure poolboy for bbcode parser -bbc_config = +# configurations for bbcode parser +bbc_parser_config = case config_env() do :prod -> %{ - size: get_env_cast_integer_with_default.("BBC_PARSER_WORKERS", "50"), - max_overflow: get_env_cast_integer_with_default.("BBC_PARSER_OVERFLOW", "20") + poolboy: %{ + size: get_env_cast_integer_with_default.("BBC_PARSER_WORKERS", "50"), + max_overflow: get_env_cast_integer_with_default.("BBC_PARSER_OVERFLOW", "20") + } } _ -> %{ - size: 5, - max_overflow: 2 + poolboy: %{ + size: 5, + max_overflow: 2 + } } end +# configure poolboy for bbcode parser bbc_parser_poolboy_config = [ name: {:local, :bbc_parser}, worker_module: EpochtalkServer.BBCParser, - size: bbc_config.size, - max_overflow: bbc_config.max_overflow, + size: bbc_parser_config.poolboy.size, + max_overflow: bbc_parser_config.poolboy.max_overflow, strategy: :fifo ] From 603d14b2994be59a8b45eaa65ff7f3e56ec6d125 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 20 Dec 2024 10:19:08 -1000 Subject: [PATCH 226/231] feat(config/runtime): add configurations for porcelain timeout --- config/runtime.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/runtime.exs b/config/runtime.exs index 886fe517..14f0efe6 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -415,6 +415,9 @@ bbc_parser_config = poolboy: %{ size: get_env_cast_integer_with_default.("BBC_PARSER_WORKERS", "50"), max_overflow: get_env_cast_integer_with_default.("BBC_PARSER_OVERFLOW", "20") + }, + parser: %{ + porcelain_receive_timeout: get_env_cast_integer_with_default.("BBC_PARSER_PORCELAIN_TIMEOUT", "100") } } @@ -423,6 +426,9 @@ bbc_parser_config = poolboy: %{ size: 5, max_overflow: 2 + }, + parser: %{ + porcelain_receive_timeout: 100 } } end @@ -438,6 +444,9 @@ bbc_parser_poolboy_config = [ config :epochtalk_server, bbc_parser_poolboy_config: bbc_parser_poolboy_config +# configure bbcode parser +config :epochtalk_server, bbc_parser_config: bbc_parser_config.parser + # conditionally show debug logs in prod if config_env() == :prod do logger_level = From c544e3f4114cd7cfa145e35d3ac871dc3dfdc031 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 20 Dec 2024 10:19:49 -1000 Subject: [PATCH 227/231] perf(config/runtime): set genserver/poolboy timeouts to default (5000ms) increasing these configs won't affect parser execution time; only affect the upper limit on pool connection and call waiting (i think) --- lib/epochtalk_server/bbc_parser.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index ab607196..34229927 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -5,10 +5,10 @@ defmodule EpochtalkServer.BBCParser do # genserver call timeouts (ms) @genserver_parse_timeout 5000 - @genserver_parse_tuple_timeout 500 + @genserver_parse_tuple_timeout 5000 # poolboy timeout (ms) - @poolboy_transaction_timeout 200 + @poolboy_transaction_timeout 5000 # porcelain php parser call timeout (ms) @receive_timeout 20 From 03ca7cb33737ba7d184df1f538ca1a618c12c5d7 Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 20 Dec 2024 10:22:12 -1000 Subject: [PATCH 228/231] feat(config/runtime): load receive timeout from application config allows configuration through env variable in production --- lib/epochtalk_server/bbc_parser.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/epochtalk_server/bbc_parser.ex b/lib/epochtalk_server/bbc_parser.ex index 34229927..8bdfe22e 100644 --- a/lib/epochtalk_server/bbc_parser.ex +++ b/lib/epochtalk_server/bbc_parser.ex @@ -10,9 +10,6 @@ defmodule EpochtalkServer.BBCParser do # poolboy timeout (ms) @poolboy_transaction_timeout 5000 - # porcelain php parser call timeout (ms) - @receive_timeout 20 - @moduledoc """ `BBCParser` genserver, runs interactive php shell to call bbcode parser """ @@ -68,6 +65,10 @@ defmodule EpochtalkServer.BBCParser do defp parse_with_proc("", {_proc, _pid}), do: {:ok, ""} defp parse_with_proc(bbcode_data, {proc, pid}) do + config = Application.get_env(:epochtalk_server, :bbc_parser_config) + # porcelain php parser call timeout (ms) + receive_timeout = config.porcelain_receive_timeout + Proc.send_input(proc, "echo parse_bbc('#{bbcode_data}');\n") receive do @@ -75,7 +76,7 @@ defmodule EpochtalkServer.BBCParser do {:ok, data} after # time out after not receiving any data - @receive_timeout -> + receive_timeout -> Logger.error("#{__MODULE__}(parse timeout): #{inspect(pid)}, #{inspect(bbcode_data)}") bbcode_data = From 4a52bfcc80aa2e05f38b147dbc410065ff977c8d Mon Sep 17 00:00:00 2001 From: unenglishable Date: Fri, 20 Dec 2024 10:23:04 -1000 Subject: [PATCH 229/231] style(config/runtime): mix format --- config/runtime.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index 14f0efe6..7ebb2cd4 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -417,7 +417,8 @@ bbc_parser_config = max_overflow: get_env_cast_integer_with_default.("BBC_PARSER_OVERFLOW", "20") }, parser: %{ - porcelain_receive_timeout: get_env_cast_integer_with_default.("BBC_PARSER_PORCELAIN_TIMEOUT", "100") + porcelain_receive_timeout: + get_env_cast_integer_with_default.("BBC_PARSER_PORCELAIN_TIMEOUT", "100") } } From 9bfd61c41b3efc1506d9b092f0196ffe1c374322 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 9 Jan 2025 11:21:06 -1000 Subject: [PATCH 230/231] fix(post-limit): set max of 100 for posts by thread --- lib/epochtalk_server_web/controllers/post.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/controllers/post.ex b/lib/epochtalk_server_web/controllers/post.ex index abf500fd..40941929 100644 --- a/lib/epochtalk_server_web/controllers/post.ex +++ b/lib/epochtalk_server_web/controllers/post.ex @@ -579,7 +579,7 @@ defmodule EpochtalkServerWeb.Controllers.Post do defp proxy_by_thread(conn, attrs) do with thread_id <- Validate.cast(attrs, "thread_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), - limit <- Validate.cast(attrs, "limit", :integer, default: 25), + limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), user <- Guardian.Plug.current_resource(conn), user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "posts.byThread"), From 53302117647e9408abeb57d530488f0a4ab02cf7 Mon Sep 17 00:00:00 2001 From: Anthony Kinsey Date: Thu, 9 Jan 2025 11:21:25 -1000 Subject: [PATCH 231/231] fix(thread-limit): add max of 100 for thread by board --- lib/epochtalk_server_web/controllers/thread.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/epochtalk_server_web/controllers/thread.ex b/lib/epochtalk_server_web/controllers/thread.ex index 82aa5945..87cc8d27 100644 --- a/lib/epochtalk_server_web/controllers/thread.ex +++ b/lib/epochtalk_server_web/controllers/thread.ex @@ -823,7 +823,7 @@ defmodule EpochtalkServerWeb.Controllers.Thread do defp proxy_by_board(conn, attrs) do with board_id <- Validate.cast(attrs, "board_id", :integer, required: true), page <- Validate.cast(attrs, "page", :integer, default: 1), - limit <- Validate.cast(attrs, "limit", :integer, default: 5), + limit <- Validate.cast(attrs, "limit", :integer, default: 25, min: 1, max: 100), user <- Guardian.Plug.current_resource(conn), user_priority <- ACL.get_user_priority(conn), :ok <- ACL.allow!(conn, "threads.byBoard"),