Skip to content

Commit

Permalink
style(): mix format
Browse files Browse the repository at this point in the history
  • Loading branch information
unenglishable committed Oct 17, 2024
1 parent 7241ee4 commit 7b73724
Show file tree
Hide file tree
Showing 6 changed files with 160 additions and 101 deletions.
2 changes: 1 addition & 1 deletion config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ if config_env() == :prod do
id_board_blacklist =
id_board_blacklist
|> String.split()
|> Enum.map(&(String.to_integer(&1)))
|> Enum.map(&String.to_integer(&1))

# Configure proxy
config :epochtalk_server,
Expand Down
110 changes: 66 additions & 44 deletions lib/epochtalk_server/smf_loader.ex
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ defmodule EpochtalkServer.SMFLoader do

# converts smf_boards tsv file to epochtalk boards tsv file
def convert_smf_boards_tsv_file(path) do
{boards, board_mappings} = load_from_tsv_file(path)
|> map_boards_stream()
{boards, board_mappings} =
load_from_tsv_file(path)
|> map_boards_stream()

# write boards to file for import
boards
Expand All @@ -14,6 +15,7 @@ defmodule EpochtalkServer.SMFLoader do
# return board mappings for later insertion
board_mappings
end

def load_categories_mappings_from_tsv_file(path) do
# load categories (in postgres format)
# and return category mappings for later insertion
Expand All @@ -28,42 +30,46 @@ defmodule EpochtalkServer.SMFLoader do
}
end)
end

def insert_board_mappings(board_mappings) do
# board mappings
board_mappings
|> BoardMapping.update()
end

# loads smf data from a tsv file
def load_from_tsv_file(path) do
with true <- if(File.exists?(path), do: true, else: "ファイルがない"),
{:ok, file} <- File.read(path),
file_lines <- file |> String.trim() |> String.split("\n"),
[header_line | file_lines] <- file_lines,
headers <-
# clean line
header_line
|> String.trim()
|> String.split("\t"),
smf_boards <-
file_lines
|> Enum.map(&(
&1
# clean line
|> String.trim()
|> String.split("\t")
))
|> Enum.map(fn line ->
# map each line to headers
Enum.zip(headers, line)
|> Enum.into(%{})
end) do
smf_boards
{:ok, file} <- File.read(path),
file_lines <- file |> String.trim() |> String.split("\n"),
[header_line | file_lines] <- file_lines,
# clean line
headers <-
header_line
|> String.trim()
|> String.split("\t"),
smf_boards <-
file_lines
|> Enum.map(
&(&1
# clean line
|> String.trim()
|> String.split("\t"))
)
|> Enum.map(fn line ->
# map each line to headers
Enum.zip(headers, line)
|> Enum.into(%{})
end) do
smf_boards
else
problem -> IO.puts("問題がある: #{inspect(problem)}")
end
end

def map_boards_stream(boards_stream) do
now = NaiveDateTime.utc_now() |> NaiveDateTime.to_string()

{board_mapping_pairs, _slug_duplicate_index} =
boards_stream
|> Enum.map_reduce(%{}, fn smf_board, slugs ->
Expand All @@ -72,19 +78,24 @@ defmodule EpochtalkServer.SMFLoader do
|> HtmlEntities.decode()
|> String.replace(~r{[ /]}, "-")
|> String.slice(0..99)

# handle duplicate slugs
slug_duplicate_index = Map.get(slugs, slug)

{slugs, slug} =
if slug_duplicate_index == nil do
# keep track of used slugs, return original slug
{Map.put(slugs, slug, 0), slug}
else
# replace last characters with index
new_slug = slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10))))
new_slug =
slug |> String.slice(0..(99 - (1 + Integer.floor_div(slug_duplicate_index, 10))))

new_slug = new_slug <> to_string(slug_duplicate_index)
# increment used slugs, return new slug
{Map.put(slugs, slug, slug_duplicate_index + 1), new_slug}
end

# build board
board =
%{}
Expand All @@ -98,7 +109,10 @@ defmodule EpochtalkServer.SMFLoader do
|> Map.put(:created_at, now)
|> Map.put(:imported_at, now)
|> Map.put(:updated_at, now)
|> Map.put(:meta, "\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\"")
|> Map.put(
:meta,
"\"{\"\"disable_self_mod\"\": false, \"\"disable_post_edit\"\": null, \"\"disable_signature\"\": false}\""
)
|> Map.put(:right_to_left, "f")
|> Map.put(:slug, slug)

Expand All @@ -114,6 +128,7 @@ defmodule EpochtalkServer.SMFLoader do
category_id: smf_board["ID_CAT"] |> String.to_integer(),
view_order: smf_board["boardOrder"] |> String.to_integer()
}

_ ->
%{
type: "board",
Expand All @@ -123,10 +138,13 @@ defmodule EpochtalkServer.SMFLoader do
view_order: smf_board["boardOrder"] |> String.to_integer()
}
end

{{board, board_mapping}, slugs}
end)

Enum.unzip(board_mapping_pairs)
end

def tabulate_boards_map(boards_map) do
data =
boards_map
Expand All @@ -149,27 +167,31 @@ defmodule EpochtalkServer.SMFLoader do
|> Enum.join("\t")
end)

header = [
"id",
"name",
"description",
"post_count",
"thread_count",
"viewable_by",
"postable_by",
"created_at",
"imported_at",
"updated_at",
"meta",
"right_to_left",
"slug"
] |> Enum.join("\t")
[ header | data ]
header =
[
"id",
"name",
"description",
"post_count",
"thread_count",
"viewable_by",
"postable_by",
"created_at",
"imported_at",
"updated_at",
"meta",
"right_to_left",
"slug"
]
|> Enum.join("\t")

[header | data]
end

def write_to_tsv_file(data, path) do
with false <- if(File.exists?(path), do: "ファイルがもうある", else: false),
file <- File.stream!(path) do
data |> Enum.into(file, fn line -> line <> "\n" end)
file <- File.stream!(path) do
data |> Enum.into(file, fn line -> line <> "\n" end)
else
problem -> IO.puts("問題がある: #{inspect(problem)}")
end
Expand Down
1 change: 1 addition & 0 deletions lib/epochtalk_server_web/helpers/breadcrumbs.ex
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ defmodule EpochtalkServerWeb.Helpers.Breadcrumbs do
"thread" ->
{:ok, board} = Board.find_by_id(object.board_id)
%{id: board.slug, type: "board"}

_ ->
nil
end
Expand Down
Loading

0 comments on commit 7b73724

Please sign in to comment.