Merge branch 'master' into find-invalid-irve-static-resources

apps/transport/lib/db/dataset.ex

@@ -235,11 +235,21 @@ defmodule DB.Dataset do
 
   @spec filter_by_climate_resilience_bill(Ecto.Query.t(), map()) :: Ecto.Query.t()
   defp filter_by_climate_resilience_bill(%Ecto.Query{} = query, %{"loi-climat-resilience" => "true"}) do
-    where(query, [dataset: d], fragment("'loi-climat-resilience' = any(?)", d.custom_tags))
+    filter_by_custom_tag(query, %{"custom_tag" => "loi-climat-resilience"})
   end
 
   defp filter_by_climate_resilience_bill(%Ecto.Query{} = query, _), do: query
 
+  @spec filter_by_custom_tag(Ecto.Query.t(), binary() | map()) :: Ecto.Query.t()
+  defp filter_by_custom_tag(%Ecto.Query{} = query, custom_tag) when is_binary(custom_tag) do
+    where(query, [dataset: d], fragment("? = any(?)", ^custom_tag, d.custom_tags))
+  end
+
+  defp filter_by_custom_tag(%Ecto.Query{} = query, %{"custom_tag" => custom_tag}),
+    do: filter_by_custom_tag(query, custom_tag)
+
+  defp filter_by_custom_tag(%Ecto.Query{} = query, _), do: query
+
   @spec filter_by_feature(Ecto.Query.t(), map()) :: Ecto.Query.t()
   defp filter_by_feature(query, %{"features" => [feature]})
        when feature in ["service_alerts", "trip_updates", "vehicle_positions"] do
@@ -377,6 +387,7 @@ defmodule DB.Dataset do
       |> filter_by_commune(params)
       |> filter_by_licence(params)
       |> filter_by_climate_resilience_bill(params)
+      |> filter_by_custom_tag(params)
       |> filter_by_fulltext(params)
       |> select([dataset: d], d.id)
 
@@ -634,6 +645,11 @@ defmodule DB.Dataset do
     |> Repo.aggregate(:count, :id)
   end
 
+  @spec count_by_custom_tag(binary()) :: non_neg_integer()
+  def count_by_custom_tag(custom_tag) do
+    base_query() |> filter_by_custom_tag(custom_tag) |> Repo.aggregate(:count, :id)
+  end
+
   @spec get_by_slug(binary) :: {:ok, __MODULE__.t()} | {:error, binary()}
   def get_by_slug(slug) do
     preload_without_validations()

apps/transport/lib/jobs/dataset_quality_score.ex

@@ -280,11 +280,16 @@ defmodule Transport.Jobs.DatasetAvailabilityScore do
 
   @spec current_dataset_availability(integer()) :: %{score: float | nil, details: map()}
   def current_dataset_availability(dataset_id) do
-    resources = dataset_id |> dataset_resources() |> Enum.reject(&DB.Resource.is_documentation?/1)
-    current_dataset_infos = resources |> Enum.map(&resource_availability(&1))
-    scores = current_dataset_infos |> Enum.map(fn %{availability: availability} -> availability end)
+    relevant_resources = dataset_id |> dataset_resources() |> Enum.reject(&DB.Resource.is_documentation?/1)
 
-    %{score: average(scores), details: %{resources: current_dataset_infos}}
+    if Enum.empty?(relevant_resources) do
+      %{score: 0.0, details: %{resources: []}}
+    else
+      current_dataset_infos = Enum.map(relevant_resources, &resource_availability(&1))
+      scores = current_dataset_infos |> Enum.map(fn %{availability: availability} -> availability end)
+
+      %{score: average(scores), details: %{resources: current_dataset_infos}}
+    end
   end
 
   @spec resource_availability(DB.Resource.t()) :: %{

apps/transport/lib/transport_web/controllers/page_controller.ex

@@ -48,6 +48,7 @@ defmodule TransportWeb.PageController do
       count_aoms_with_dataset: count_aoms_with_dataset(),
       count_regions_completed: count_regions_completed(),
       count_public_transport_has_realtime: Dataset.count_public_transport_has_realtime(),
+      count_paris2024: Dataset.count_by_custom_tag("paris2024"),
       percent_population: percent_population(),
       reusers: CSVDocuments.reusers(),
       facilitators: CSVDocuments.facilitators()
@@ -276,7 +277,13 @@ defmodule TransportWeb.PageController do
       type_tile(conn, "charging-stations"),
       type_tile(conn, "private-parking"),
       type_tile(conn, "locations"),
-      type_tile(conn, "informations")
+      type_tile(conn, "informations"),
+      %Tile{
+        link: dataset_path(conn, :index, %{"custom_tag" => "paris2024"}),
+        icon: icon_type_path("paris2024"),
+        title: "JOP Paris 2024",
+        count: Keyword.fetch!(counts, :count_paris2024)
+      }
     ]
   end
 

apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex

@@ -18,7 +18,7 @@ stats = @metadata["stats"] %>
         <li>
           <div>
             <div class="networks-list">
-              <%= dgettext("validations", "calendar span by network") %> :
+              <%= dgettext("validations", "calendar span by network:") %>
               <.networks_start_end_dates
                 networks_start_end_dates={networks_start_end_dates}
                 locale={get_session(@conn, :locale)}
@@ -40,11 +40,17 @@ stats = @metadata["stats"] %>
         <%= dngettext("validations", "transport mode", "transport modes", length(@modes)) %> :
         <strong><%= Enum.join(@modes, ", ") %></strong>
       </li>
-      <li :if={@metadata["stop_points_count"] != nil}>
-        <%= dgettext("validations", "number of stop points") %> : <strong><%= @metadata["stop_points_count"] %></strong>
+      <li :if={stats["routes_count"] != nil}>
+        <%= dgettext("validations", "number of routes:") %>
+        <strong><%= format_nil_or_number(stats["routes_count"], locale) %></strong>
       </li>
-      <li :if={@metadata["stop_areas_count"] != nil}>
-        <%= dgettext("validations", "number of stop areas") %> : <strong><%= @metadata["stop_areas_count"] %></strong>
+      <li :if={stats["stop_points_count"] != nil}>
+        <%= dgettext("validations", "number of stop points:") %>
+        <strong><%= format_nil_or_number(stats["stop_points_count"], locale) %></strong>
+      </li>
+      <li :if={stats["stop_areas_count"] != nil}>
+        <%= dgettext("validations", "number of stop areas:") %>
+        <strong><%= format_nil_or_number(stats["stop_areas_count"], locale) %></strong>
       </li>
     </ul>
   </div>

apps/transport/lib/transport_web/views/dataset_view.ex

@@ -203,7 +203,9 @@ defmodule TransportWeb.DatasetView do
       "real-time-public-transit" => "bus-stop.svg",
       "long-distance-coach" => "bus.svg",
       "train" => "train.svg",
-      "boat" => "boat.svg"
+      "boat" => "boat.svg",
+      # Custom tags
+      "paris2024" => "olympics.svg"
     }
 
     if Map.has_key?(icons, type), do: "/images/icons/#{Map.get(icons, type)}"

apps/transport/priv/gettext/en/LC_MESSAGES/validations.po

@@ -70,14 +70,6 @@ msgstr ""
 msgid "Validation report"
 msgstr ""
 
-#, elixir-autogen, elixir-format
-msgid "number of stop areas"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "number of stop points"
-msgstr ""
-
 #, elixir-autogen, elixir-format
 msgid "to"
 msgstr ""
@@ -289,10 +281,6 @@ msgstr ""
 msgid "Validation carried out using the <a href=\"%{gtfs_link}\">current GTFS file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">PAN GTFS validator</a>."
 msgstr ""
 
-#, elixir-autogen, elixir-format
-msgid "calendar span by network"
-msgstr ""
-
 #, elixir-autogen, elixir-format
 msgid "from"
 msgstr ""
@@ -416,3 +404,19 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "This file contains the GTFS-Flex extension. We are not able to validate it for now."
 msgstr ""
+
+#, elixir-autogen, elixir-format, fuzzy
+msgid "calendar span by network:"
+msgstr ""
+
+#, elixir-autogen, elixir-format, fuzzy
+msgid "number of routes:"
+msgstr ""
+
+#, elixir-autogen, elixir-format, fuzzy
+msgid "number of stop areas:"
+msgstr ""
+
+#, elixir-autogen, elixir-format, fuzzy
+msgid "number of stop points:"
+msgstr ""

apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po

@@ -70,14 +70,6 @@ msgstr "Cette ressource fait partie du jeu de données"
 msgid "Validation report"
 msgstr "Rapport de validation"
 
-#, elixir-autogen, elixir-format
-msgid "number of stop areas"
-msgstr "nombre de zones d'arrêts (stop areas)"
-
-#, elixir-autogen, elixir-format
-msgid "number of stop points"
-msgstr "nombre de stops (stop points)"
-
 #, elixir-autogen, elixir-format
 msgid "to"
 msgstr "au"
@@ -289,10 +281,6 @@ msgstr "Envoi en cours"
 msgid "Validation carried out using the <a href=\"%{gtfs_link}\">current GTFS file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">PAN GTFS validator</a>."
 msgstr "Validation effectuée en utilisant <a href=\"%{gtfs_link}\">le fichier GTFS en vigueur</a> le %{date} avec <a href=\"%{validator_url}\" target=\"_blank\">le validateur GTFS du <abbr title=\"Point d'Accès National\">PAN</abbr></a>."
 
-#, elixir-autogen, elixir-format
-msgid "calendar span by network"
-msgstr "couverture calendaire par réseau"
-
 #, elixir-autogen, elixir-format
 msgid "from"
 msgstr "du"
@@ -416,3 +404,19 @@ msgstr "Veuillez noter que notre validateur GTFS n'est pas en mesure de valider
 #, elixir-autogen, elixir-format
 msgid "This file contains the GTFS-Flex extension. We are not able to validate it for now."
 msgstr "Ce fichier contient l'extension GTFS-Flex. Nous ne sommes pas en mesure de le valider pour le moment."
+
+#, elixir-autogen, elixir-format
+msgid "calendar span by network:"
+msgstr "couverture calendaire par réseau :"
+
+#, elixir-autogen, elixir-format
+msgid "number of routes:"
+msgstr "nombre de lignes :"
+
+#, elixir-autogen, elixir-format
+msgid "number of stop areas:"
+msgstr "nombre de zones d'arrêts :"
+
+#, elixir-autogen, elixir-format
+msgid "number of stop points:"
+msgstr "nombre d'arrêts :"

apps/transport/priv/gettext/validations.pot

@@ -70,14 +70,6 @@ msgstr ""
 msgid "Validation report"
 msgstr ""
 
-#, elixir-autogen, elixir-format
-msgid "number of stop areas"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "number of stop points"
-msgstr ""
-
 #, elixir-autogen, elixir-format
 msgid "to"
 msgstr ""
@@ -288,10 +280,6 @@ msgstr ""
 msgid "Validation carried out using the <a href=\"%{gtfs_link}\">current GTFS file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">PAN GTFS validator</a>."
 msgstr ""
 
-#, elixir-autogen, elixir-format
-msgid "calendar span by network"
-msgstr ""
-
 #, elixir-autogen, elixir-format
 msgid "from"
 msgstr ""
@@ -415,3 +403,19 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "This file contains the GTFS-Flex extension. We are not able to validate it for now."
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "calendar span by network:"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "number of routes:"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "number of stop areas:"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "number of stop points:"
+msgstr ""

apps/transport/priv/search_custom_messages.yml

@@ -299,3 +299,17 @@
         They contain traffic data for pedestrians, bicycles, and micro-mobility.
 
         The standard used is described [in this documentation](https://doc.transport.data.gouv.fr/producteurs/comptage-des-mobilites).
+
+- category: paris2024
+  search_params:
+    - key: custom_tag
+      value: paris2024
+  msg:
+    fr: |
+        Les jeux de données référencés dans cette catégorie contiennent des données publiées sur data.gouv.fr contenant des informations liées à la mobilité à l'occasion des jeux Olympiques et Paralympiques 2024.
+
+        Les données ne sont pas forcément aux standards attendus, mais nous avons fait le choix de les répertorier étant donné l'importance de l'évènement.
+    en: |
+        The datasets listed in this category contain data published on data.gouv.fr related to mobility during the 2024 Olympic and Paralympic Games.
+
+        While the datasets listed here may not fully meet all quality standards, we have chosen to include them due to their potential importance

apps/transport/test/db/db_dataset_test.exs

@@ -584,6 +584,12 @@ defmodule DB.DatasetDBTest do
     assert DB.Dataset.count_coach() == 1
   end
 
+  test "count_by_custom_tag" do
+    assert 0 == DB.Dataset.count_by_custom_tag("foo")
+    insert(:dataset, type: "public-transit", is_active: true, custom_tags: ["bar", "foo"])
+    assert 1 == DB.Dataset.count_by_custom_tag("foo")
+  end
+
   test "correct organization type" do
     insert(:dataset, datagouv_id: datagouv_id = Ecto.UUID.generate())
 

apps/transport/test/transport/jobs/dataset_quality_score_test.exs

@@ -263,6 +263,14 @@ defmodule Transport.Test.Transport.Jobs.DatasetQualityScoreTest do
                score: 0.75
              } == current_dataset_availability(dataset.id)
     end
+
+    test "a dataset with 0 resources" do
+      dataset = insert(:dataset, is_active: true)
+
+      assert [] == dataset |> DB.Repo.preload(:resources) |> Map.fetch!(:resources)
+
+      assert %{score: 0.0, details: %{resources: []}} == current_dataset_availability(dataset.id)
+    end
   end
 
   describe "current dataset freshness" do

apps/transport/test/transport_web/controllers/dataset_search_test.exs

@@ -184,6 +184,18 @@ defmodule TransportWeb.DatasetSearchControllerTest do
              %{"type" => "public-transit", "loi-climat-resilience" => "true"} |> Dataset.list_datasets() |> Repo.all()
   end
 
+  test "searching with a custom tag" do
+    %DB.Dataset{id: dataset_id} =
+      insert(:dataset, type: "public-transit", is_active: true, custom_tags: ["bar", "foo"])
+
+    assert 3 == %{"type" => "public-transit"} |> Dataset.list_datasets() |> Repo.all() |> Enum.count()
+
+    assert [%DB.Dataset{id: ^dataset_id}] =
+             %{"type" => "public-transit", "custom_tag" => "foo"} |> Dataset.list_datasets() |> Repo.all()
+
+    assert 1 == DB.Dataset.count_by_custom_tag("foo")
+  end
+
   test "searching for datasets in an AOM" do
     aom = insert(:aom)
     aom2 = insert(:aom)

apps/transport/test/transport_web/controllers/resource_controller_test.exs

@@ -384,6 +384,9 @@ defmodule TransportWeb.ResourceControllerTest do
               }
             },
             "stats" => %{
+              "stop_points_count" => 1_322,
+              "stop_areas_count" => 30,
+              "routes_count" => 123,
               "routes_with_short_name_count" => 5
             }
           },
@@ -395,6 +398,9 @@ defmodule TransportWeb.ResourceControllerTest do
     content = conn |> get(resource_path(conn, :details, resource_id)) |> html_response(200)
     assert content =~ "Rapport de validation"
     assert content =~ "ferry"
+    assert content =~ ~r"nombre de lignes :(\s*)<strong>123</strong>"
+    assert content =~ ~r"nombre d&#39;arrêts :(\s*)<strong>1 322</strong>"
+    assert content =~ ~r"nombre de zones d&#39;arrêts :(\s*)<strong>30</strong>"
     assert content =~ "couverture calendaire par réseau"
     assert content =~ "3CM"
     assert content =~ "30/09/2022"