From 3cf2adbf290ddba1b024bf133157024a6fb8b929 Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka Date: Tue, 25 Jun 2024 02:01:50 -0700 Subject: [PATCH] Add cqd_score --- ribs/archives/_unstructured_archive.py | 39 +++- tests/archives/unstructured_archive_test.py | 190 +++----------------- 2 files changed, 60 insertions(+), 169 deletions(-) diff --git a/ribs/archives/_unstructured_archive.py b/ribs/archives/_unstructured_archive.py index 03bf7261b..7e6eadf7f 100644 --- a/ribs/archives/_unstructured_archive.py +++ b/ribs/archives/_unstructured_archive.py @@ -496,5 +496,40 @@ def lower_bounds(self) -> np.ndarray: return np.min(self._store.data("measures"), axis=0) - # TODO: cqd_score needs to be fixed since it assumes lower_bounds and - # upper_bounds. + def cqd_score(self, + iterations, + target_points, + penalties, + obj_min, + obj_max, + dist_max=None, + dist_ord=None): + """Computes the CQD score of the archive. + + Refer to the documentation in :meth:`ArchiveBase.cqd_score` for more + info. The key difference from the base implementation is that the + implementation in ArchiveBase assumes the archive has a pre-defined + measure space with lower and upper bounds. However, by nature of being + unstructured, this archive has lower and upper bounds that change over + time. Thus, it is required to directly pass in ``target_points`` and + ``dist_max``. + + Raises: + ValueError: dist_max and target_points were not passed in. + """ + + if dist_max is None or np.isscalar(target_points): + raise ValueError( + "In UnstructuredArchive, dist_max must be passed " + "in, and target_points must be passed in as a custom " + "array of points.") + + return super().cqd_score( + iterations=iterations, + target_points=target_points, + penalties=penalties, + obj_min=obj_min, + obj_max=obj_max, + dist_max=dist_max, + dist_ord=dist_ord, + ) diff --git a/tests/archives/unstructured_archive_test.py b/tests/archives/unstructured_archive_test.py index c8efb304e..e623c41c5 100644 --- a/tests/archives/unstructured_archive_test.py +++ b/tests/archives/unstructured_archive_test.py @@ -531,170 +531,26 @@ def test_nonfinite_inputs(data): data.archive.index_of_single(data.measures) -# TODO: cqd_score - -# def test_cqd_score_detects_wrong_shapes(data): -# with pytest.raises(ValueError): -# data.archive.cqd_score( -# iterations=1, -# target_points=np.array([1.0]), # Should be 3D. -# penalties=2, -# obj_min=0.0, -# obj_max=1.0, -# ) - -# with pytest.raises(ValueError): -# data.archive.cqd_score( -# iterations=1, -# target_points=3, -# penalties=[[1.0, 1.0]], # Should be 1D. -# obj_min=0.0, -# obj_max=1.0, -# ) - -# def test_cqd_score_with_one_elite(): -# archive = UnstructuredArchive( -# solution_dim=2, -# measure_dim=2, -# k_neighbors=5, -# novelty_threshold=1.0, -# ) -# archive.add_single([4.0, 4.0], 1.0, [0.0, 0.0]) - -# score = archive.cqd_score( -# iterations=1, -# # With this target point, the solution above at [0, 0] has a normalized -# # distance of 0.5, since it is halfway between the archive bounds of -# # (-1, -1) and (1, 1). -# target_points=np.array([[[1.0, 1.0]]]), -# penalties=2, -# obj_min=0.0, -# obj_max=1.0, -# ).mean - -# # For theta=0, the score should be 1.0 - 0 * 0.5 = 1.0 -# # For theta=1, the score should be 1.0 - 1 * 0.5 = 0.5 -# assert np.isclose(score, 1.0 + 0.5) - -# def test_cqd_score_with_max_dist(): -# archive = UnstructuredArchive( -# solution_dim=2, -# measure_dim=2, -# k_neighbors=5, -# novelty_threshold=1.0, -# ) -# archive.add_single([4.0, 4.0], 0.5, [0.0, 1.0]) - -# score = archive.cqd_score( -# iterations=1, -# # With this target point and dist_max, the solution above at [0, 1] -# # has a normalized distance of 0.5, since it is one unit away. -# target_points=np.array([[[1.0, 1.0]]]), -# penalties=2, -# obj_min=0.0, -# obj_max=1.0, -# dist_max=2.0, -# ).mean - -# # For theta=0, the score should be 0.5 - 0 * 0.5 = 0.5 -# # For theta=1, the score should be 0.5 - 1 * 0.5 = 0.0 -# assert np.isclose(score, 0.5 + 0.0) - -# def test_cqd_score_l1_norm(): -# archive = UnstructuredArchive( -# solution_dim=2, -# measure_dim=2, -# k_neighbors=5, -# novelty_threshold=1.0, -# ) -# archive.add_single([4.0, 4.0], 0.5, [0.0, 0.0]) - -# score = archive.cqd_score( -# iterations=1, -# # With this target point and dist_max, the solution above at [0, 0] -# # has a normalized distance of 1.0, since it is two units away. -# target_points=np.array([[[1.0, 1.0]]]), -# penalties=2, -# obj_min=0.0, -# obj_max=1.0, -# dist_max=2.0, -# # L1 norm. -# dist_ord=1, -# ).mean - -# # For theta=0, the score should be 0.5 - 0 * 1.0 = 0.5 -# # For theta=1, the score should be 0.5 - 1 * 1.0 = -0.5 -# assert np.isclose(score, 0.5 + -0.5) - -# def test_cqd_score_full_output(): -# archive = UnstructuredArchive( -# solution_dim=2, -# measure_dim=2, -# k_neighbors=5, -# novelty_threshold=1.0, -# ) -# archive.add_single([4.0, 4.0], 1.0, [0.0, 0.0]) - -# result = archive.cqd_score( -# iterations=5, -# # With this target point, the solution above at [0, 0] has a normalized -# # distance of 0.5, since it is halfway between the archive bounds of -# # (-1, -1) and (1, 1). -# target_points=np.array([ -# [[1.0, 1.0]], -# [[1.0, 1.0]], -# [[1.0, 1.0]], -# [[-1.0, -1.0]], -# [[-1.0, -1.0]], -# ]), -# penalties=2, -# obj_min=0.0, -# obj_max=1.0, -# ) - -# # For theta=0, the score should be 1.0 - 0 * 0.5 = 1.0 -# # For theta=1, the score should be 1.0 - 1 * 0.5 = 0.5 -# assert result.iterations == 5 -# assert np.isclose(result.mean, 1.0 + 0.5) -# assert np.all(np.isclose(result.scores, 1.0 + 0.5)) -# assert np.all( -# np.isclose( -# result.target_points, -# np.array([ -# [[1.0, 1.0]], -# [[1.0, 1.0]], -# [[1.0, 1.0]], -# [[-1.0, -1.0]], -# [[-1.0, -1.0]], -# ]))) -# assert np.all(np.isclose(result.penalties, [0.0, 1.0])) -# assert np.isclose(result.obj_min, 0.0) -# assert np.isclose(result.obj_max, 1.0) -# # Distance from (-1,-1) to (1,1). -# assert np.isclose(result.dist_max, 2 * np.sqrt(2)) -# assert result.dist_ord is None - -# def test_cqd_score_with_two_elites(): -# archive = UnstructuredArchive( -# solution_dim=2, -# measure_dim=2, -# k_neighbors=5, -# novelty_threshold=1.0, -# ) -# archive.add_single([4.0, 4.0], 0.25, [-2.0, -2.0]) # Elite 2. -# archive.add_single([4.0, 4.0], 0.0, [2.0, 2.0]) # Elite 3. - -# score = archive.cqd_score( -# iterations=1, -# # With this target point, Elite 1 has a normalized distance of 1, since -# # it is exactly at [-2, -2]. -# # -# # Elite 2 has a normalized distance of 0, since it is exactly at [2, 2]. -# target_points=np.array([[[2.0, 2.0]]]), -# penalties=2, # Penalties of 0 and 1. -# obj_min=0.0, -# obj_max=1.0, -# ).mean -# # For theta=0, the score should be max(0.25 - 0 * 1.0, 0 - 0 * 0.0) = 0.25 -# # For theta=1, the score should be max(0.25 - 1 * 1.0, 0 - 1 * 0.0) = 0.0 -# assert np.isclose(score, 0.25 + 0) +def test_cqd_score_with_max_dist(): + archive = UnstructuredArchive( + solution_dim=2, + measure_dim=2, + k_neighbors=5, + novelty_threshold=1.0, + ) + archive.add_single([4.0, 4.0], 0.5, [0.0, 1.0]) + + score = archive.cqd_score( + iterations=1, + # With this target point and dist_max, the solution above at [0, 1] + # has a normalized distance of 0.5, since it is one unit away. + target_points=np.array([[[1.0, 1.0]]]), + penalties=2, + obj_min=0.0, + obj_max=1.0, + dist_max=2.0, + ).mean + + # For theta=0, the score should be 0.5 - 0 * 0.5 = 0.5 + # For theta=1, the score should be 0.5 - 1 * 0.5 = 0.0 + assert np.isclose(score, 0.5 + 0.0)