Use "observed" in copy and adopt nudge in insample cards (#2958)

Summary:

We're using observed to be consistent with the scatter plot https://fburl.com/code/ammcq511.

Bumps:
- +1 for being modeled
- +2 for being an objective
- +1 for being a constraint
- - up to 9 for being an old trial, by index
  - on the experiment, we want newer trials first.

Reviewed By: ItsMrLin

Differential Revision: D64605408

ax/analysis/plotly/arm_effects/insample_effects.py

@@ -32,7 +32,7 @@
 
 class InSampleEffectsPlot(PlotlyAnalysis):
     """
-    Plotly Insample Effecs plot for a single metric on a single trial, with one point
+    Plotly Insample Effects plot for a single metric on a single trial, with one point
     per unique arm across all trials. The plot may either use modeled effects, or
     raw / observed data.
 
@@ -65,7 +65,7 @@ def __init__(
             metric_name: The name of the metric to plot.
             trial_index: The of the trial to plot arms for.
             use_modeled_effects: Whether to use modeled effects or show
-                raw effects.
+                observed effects.
         """
 
         self.metric_name = metric_name
@@ -111,17 +111,25 @@ def compute(
             df=df, metric_name=self.metric_name, outcome_constraints=outcome_constraints
         )
 
-        if (
-            experiment.optimization_config is None
-            or self.metric_name not in experiment.optimization_config.metrics
-        ):
-            level = AnalysisCardLevel.LOW
-        elif self.metric_name in experiment.optimization_config.objective.metric_names:
-            level = AnalysisCardLevel.HIGH
-        else:
-            level = AnalysisCardLevel.MID
-
-        plot_type = "Modeled" if self.use_modeled_effects else "Raw"
+        nudge = 0
+        level = AnalysisCardLevel.MID
+        if experiment.optimization_config is not None:
+            if (
+                self.metric_name
+                in experiment.optimization_config.objective.metric_names
+            ):
+                nudge = 2
+            elif self.metric_name in experiment.optimization_config.metrics:
+                nudge = 1
+
+        level = AnalysisCardLevel.MID
+        if self.use_modeled_effects:
+            nudge += 1
+
+        max_trial_index = max(experiment.trial_indices_expecting_data, default=0)
+        nudge -= min(max_trial_index - self.trial_index, 9)
+
+        plot_type = "Modeled" if self.use_modeled_effects else "Observed"
         subtitle = (
             "View a trial and its arms' "
             f"{'predicted' if self.use_modeled_effects else 'observed'} "
@@ -133,7 +141,7 @@ def compute(
                 f"on trial {self.trial_index}"
             ),
             subtitle=subtitle,
-            level=level,
+            level=level + nudge,
             df=df,
             fig=fig,
         )
@@ -203,7 +211,7 @@ def _get_model(
 
         return model
     else:
-        # This model just predicts raw data
+        # This model just predicts observed data
         return Models.THOMPSON(
             data=trial_data,
             search_space=experiment.search_space,

ax/analysis/plotly/tests/test_insample_effects.py

@@ -174,8 +174,8 @@ def test_compute_modeled_can_use_ebts_for_gs_with_non_predictive_model(
         self.assertEqual(
             card.subtitle, "View a trial and its arms' predicted metric values"
         )
-        # high because it's on objective
-        self.assertEqual(card.level, AnalysisCardLevel.HIGH)
+        # +2 because it's on objective, +1 because it's modeled
+        self.assertEqual(card.level, AnalysisCardLevel.MID + 3)
 
     def test_compute_modeled_can_use_ebts_for_no_gs(self) -> None:
         # GIVEN an experiment with a trial with data
@@ -220,8 +220,8 @@ def test_compute_modeled_can_use_ebts_for_no_gs(self) -> None:
         self.assertEqual(
             card.subtitle, "View a trial and its arms' predicted metric values"
         )
-        # high because it's on objective
-        self.assertEqual(card.level, AnalysisCardLevel.HIGH)
+        # +2 because it's on objective, +1 because it's modeled
+        self.assertEqual(card.level, AnalysisCardLevel.MID + 3)
 
     def test_compute_unmodeled_uses_thompson(self) -> None:
         # GIVEN an experiment with a trial with data
@@ -272,13 +272,13 @@ def test_compute_unmodeled_uses_thompson(self) -> None:
             )
 
         # AND THEN the card is labeled correctly
-        self.assertEqual(card.name, "RawEffectsPlot")
-        self.assertEqual(card.title, "Raw Effects for branin on trial 0")
+        self.assertEqual(card.name, "ObservedEffectsPlot")
+        self.assertEqual(card.title, "Observed Effects for branin on trial 0")
         self.assertEqual(
             card.subtitle, "View a trial and its arms' observed metric values"
         )
-        # high because it's on objective
-        self.assertEqual(card.level, AnalysisCardLevel.HIGH)
+        # +2 because it's on objective
+        self.assertEqual(card.level, AnalysisCardLevel.MID + 2)
 
     def test_compute_requires_data_for_the_metric_on_the_trial_without_a_model(
         self,
@@ -448,9 +448,9 @@ def test_level(self) -> None:
         experiment.fetch_data()
 
         metric_to_level = {
-            "branin": AnalysisCardLevel.HIGH,
-            "constraint_branin": AnalysisCardLevel.MID,
-            "tracking_branin": AnalysisCardLevel.LOW,
+            "branin": AnalysisCardLevel.MID + 2,
+            "constraint_branin": AnalysisCardLevel.MID + 1,
+            "tracking_branin": AnalysisCardLevel.MID,
         }
 
         for metric, level in metric_to_level.items():