diff --git a/README.md b/README.md index 4f9dc96..3773a66 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,12 @@ containing the `20xxx.ets` file that ships with this library. For instance with this config: `config :tzdata, :data_dir, "/etc/elixir_tzdata_data"` an `.ets` file such as `/etc/elixir_tzdata_data/release_ets/2017b.ets` should be present. +If you are running in an environment where there is no writeable directory (such as a read-only Docker filesystem), you can set the `read_only_fs?` configuration to true which will not write these update files. The automatic updater will still run if configured (see below). + +``` elixir +config :tzdata, :read_only_fs?, true +``` + ## Automatic data updates By default Tzdata will poll for timezone database updates every day. diff --git a/config/config.exs b/config/config.exs index bd813c5..62aa4a5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -4,4 +4,5 @@ use Mix.Config config :logger, utc_log: true config :tzdata, :autoupdate, :enabled +config :tzdata, :read_only_fs?, false # config :tzdata, :data_dir, "/etc/elixir_tzdata_storage" diff --git a/lib/tzdata/basic_data_map.ex b/lib/tzdata/basic_data_map.ex index 2909b57..0e25740 100644 --- a/lib/tzdata/basic_data_map.ex +++ b/lib/tzdata/basic_data_map.ex @@ -1,21 +1,32 @@ defmodule Tzdata.BasicDataMap do @moduledoc false + alias Tzdata.DataLoader alias Tzdata.Parser alias Tzdata.ParserOrganizer, as: Organizer + @file_names ~w(africa antarctica asia australasia backward etcetera europe northamerica southamerica)s def from_files_in_dir(dir_name) do - Enum.map(@file_names, fn file_name -> {String.to_atom(file_name), Parser.read_file(file_name, dir_name)} end) - |> make_map + from_file_names(dir_name, @file_names) end def from_single_file_in_dir(dir_name, file_name) do - [{String.to_atom(file_name), Parser.read_file(file_name, dir_name)}] + from_file_names(dir_name, [file_name]) + end + + defp from_file_names(dir_name, file_names) do + file_names + |> DataLoader.file_lines(dir_name) + |> Enum.map(fn {file_name, lines} -> + {String.to_atom(file_name), Parser.process_file(lines)} + end) |> make_map end def make_map(all_files_read) do - all_files_flattened = all_files_read |> Enum.map(fn {_name, read_file} -> read_file end) |> List.flatten + all_files_flattened = + all_files_read |> Enum.map(fn {_name, read_file} -> read_file end) |> List.flatten() + rules = Organizer.rules(all_files_flattened) zones = Organizer.zones(all_files_flattened) links = Organizer.links(all_files_flattened) @@ -23,18 +34,20 @@ defmodule Tzdata.BasicDataMap do link_list = Organizer.link_list(all_files_flattened) zone_and_link_list = Organizer.zone_and_link_list(all_files_flattened) - by_group = all_files_read - |> Enum.map(fn {name, file_read} -> {name, Organizer.zone_and_link_list(file_read)} end) - |> Enum.into(Map.new) + by_group = + all_files_read + |> Enum.map(fn {name, file_read} -> {name, Organizer.zone_and_link_list(file_read)} end) + |> Enum.into(Map.new()) + {:ok, - %{rules: rules, - zones: zones, - links: links, - zone_list: zone_list, - link_list: link_list, - zone_and_link_list: zone_and_link_list, - by_group: by_group, - } - } + %{ + rules: rules, + zones: zones, + links: links, + zone_list: zone_list, + link_list: link_list, + zone_and_link_list: zone_and_link_list, + by_group: by_group + }} end end diff --git a/lib/tzdata/data_builder.ex b/lib/tzdata/data_builder.ex index ab62d66..a4d4ad4 100644 --- a/lib/tzdata/data_builder.ex +++ b/lib/tzdata/data_builder.ex @@ -13,7 +13,7 @@ defmodule Tzdata.DataBuilder do if release_version == current_version do # remove temporary tzdata dir - File.rm_rf(tzdata_dir) + DataLoader.cleanup(tzdata_dir) Logger.info( "Downloaded tzdata release from IANA is the same version as the version currently in use (#{ @@ -45,27 +45,35 @@ defmodule Tzdata.DataBuilder do map.zone_list |> Enum.each(fn zone_name -> - insert_periods_for_zone(table, map, zone_name) - end) - - # remove temporary tzdata dir - File.rm_rf(tzdata_dir) - ets_tmp_file_name = "#{release_dir()}/#{release_version}.tmp" - ets_file_name = ets_file_name_for_release_version(release_version) - File.mkdir_p(release_dir()) - # Create file using a .tmp line ending to avoid it being - # recognized as a complete file before writing to it is complete. - :ets.tab2file(table, :erlang.binary_to_list(ets_tmp_file_name)) - :ets.delete(table) - # Then rename it, which should be an atomic operation. - :file.rename(ets_tmp_file_name, ets_file_name) + insert_periods_for_zone(table, map, zone_name) + end) + + # cleanup temporary tzdata dir + DataLoader.cleanup(tzdata_dir) + + case Application.fetch_env(:tzdata, :read_only_fs?) do + {:ok, true} -> + ets_tmp_file_name = "#{release_dir()}/#{release_version}.tmp" + ets_file_name = ets_file_name_for_release_version(release_version) + File.mkdir_p(release_dir()) + # Create file using a .tmp line ending to avoid it being + # recognized as a complete file before writing to it is complete. + :ets.tab2file(table, :erlang.binary_to_list(ets_tmp_file_name)) + :ets.delete(table) + # Then rename it, which should be an atomic operation. + :file.rename(ets_tmp_file_name, ets_file_name) + + _ -> + :ok + end + {:ok, content_length, release_version} end defp leap_sec_data(tzdata_dir), do: LeapSecParser.read_file(tzdata_dir) def ets_file_name_for_release_version(release_version) do - "#{release_dir()}/#{release_version}.v#{Tzdata.EtsHolder.file_version}.ets" + "#{release_dir()}/#{release_version}.v#{Tzdata.EtsHolder.file_version()}.ets" end def ets_table_name_for_release_version(release_version) do @@ -79,10 +87,11 @@ defmodule Tzdata.DataBuilder do tuple_periods = periods |> Enum.map(fn period -> - period_to_tuple(key, period) - end) + period_to_tuple(key, period) + end) - tuple_periods |> Enum.each(fn tuple_period -> + tuple_periods + |> Enum.each(fn tuple_period -> :ets.insert(table, tuple_period) end) end diff --git a/lib/tzdata/data_loader.ex b/lib/tzdata/data_loader.ex index 192dd54..0d2cc71 100644 --- a/lib/tzdata/data_loader.ex +++ b/lib/tzdata/data_loader.ex @@ -13,27 +13,74 @@ defmodule Tzdata.DataLoader do {:ok, last_modified} = last_modified_from_headers(headers) new_dir_name = - "#{data_dir()}/tmp_downloads/#{content_length}_#{:rand.uniform(100_000_000)}/" + case Application.fetch_env(:tzdata, :read_only_fs?) do + {:ok, true} -> + {:binary, body} + + {:ok, false} -> + new_dir_name = + "#{data_dir()}/tmp_downloads/#{content_length}_#{:rand.uniform(100_000_000)}/" + + File.mkdir_p!(new_dir_name) + target_filename = "#{new_dir_name}latest.tar.gz" + File.write!(target_filename, body) + extract(target_filename, new_dir_name) + new_dir_name + end - File.mkdir_p!(new_dir_name) - target_filename = "#{new_dir_name}latest.tar.gz" - File.write!(target_filename, body) - extract(target_filename, new_dir_name) release_version = release_version_for_dir(new_dir_name) Logger.debug("Tzdata data downloaded. Release version #{release_version}.") {:ok, content_length, release_version, new_dir_name, last_modified} end - defp extract(filename, target_dir) do + def stream_file(filename, target_dir) do + [{^filename, lines}] = file_lines([filename], target_dir) + lines + end + + def file_lines(filenames, target_dir) + + def file_lines(filenames, target_dir) when is_binary(target_dir) do + for filename <- filenames do + {filename, File.stream!(Path.join(target_dir, filename))} + end + end + + def file_lines(filenames, {:binary, _} = body) do + filenames = Enum.map(filenames, &:binary.bin_to_list/1) + {:ok, extracted} = :erl_tar.extract(body, [:compressed, :memory, {:files, filenames}]) + + for {charlist, contents} <- extracted do + # split into lines, keeping the delimiters + lines = + ~r/[^\n]*\n/ + |> Regex.scan(contents) + |> List.flatten() + + {:binary.list_to_bin(charlist), lines} + end + end + + defp extract(filename, target_dir) when is_binary(target_dir) do :erl_tar.extract(filename, [:compressed, {:cwd, target_dir}]) # remove tar.gz file after extraction File.rm!(filename) end + def cleanup(dir_name) + + def cleanup(dir_name) when is_binary(dir_name) do + File.rm_rf(dir_name) + end + + def cleanup({:binary, _}) do + :ok + end + def release_version_for_dir(dir_name) do [only_line_in_file] = - "#{dir_name}/version" - |> File.stream!() + "version" + |> stream_file(dir_name) |> Enum.to_list() only_line_in_file |> String.replace(~r/\s/, "") diff --git a/lib/tzdata/leap_sec_parser.ex b/lib/tzdata/leap_sec_parser.ex index ba6fca4..f6edafb 100644 --- a/lib/tzdata/leap_sec_parser.ex +++ b/lib/tzdata/leap_sec_parser.ex @@ -8,7 +8,8 @@ defmodule Tzdata.LeapSecParser do @file_name "leap-seconds.list" def read_file(dir_prepend \\ "source_data", file_name \\ @file_name) do - File.stream!("#{dir_prepend}#{file_name}") + file_name + |> Tzdata.DataLoader.stream_file(dir_prepend) |> process_file end diff --git a/test/test_helper.exs b/test/test_helper.exs index 869559e..066bec6 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1,7 @@ ExUnit.start() + +# Allow testing the read-only filesystem code with +# `mix test --include read_only_fs` +if :read_only_fs in ExUnit.configuration()[:include] do + Application.put_env(:tzdata, :read_only_fs?, true) +end