Skip to content

Commit

Permalink
Add cqd_score
Browse files Browse the repository at this point in the history
  • Loading branch information
@btjanaka
btjanaka committed Jun 25, 2024
1 parent 73d04a2 commit 3cf2adb
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 169 deletions.
39 changes: 37 additions & 2 deletions ribs/archives/_unstructured_archive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,5 +496,40 @@ def lower_bounds(self) -> np.ndarray:

return np.min(self._store.data("measures"), axis=0)

# TODO: cqd_score needs to be fixed since it assumes lower_bounds and
# upper_bounds.
def cqd_score(self,
iterations,
target_points,
penalties,
obj_min,
obj_max,
dist_max=None,
dist_ord=None):
"""Computes the CQD score of the archive.
Refer to the documentation in :meth:`ArchiveBase.cqd_score` for more
info. The key difference from the base implementation is that the
implementation in ArchiveBase assumes the archive has a pre-defined
measure space with lower and upper bounds. However, by nature of being
unstructured, this archive has lower and upper bounds that change over
time. Thus, it is required to directly pass in ``target_points`` and
``dist_max``.
Raises:
ValueError: dist_max and target_points were not passed in.
"""

if dist_max is None or np.isscalar(target_points):
raise ValueError(
"In UnstructuredArchive, dist_max must be passed "
"in, and target_points must be passed in as a custom "
"array of points.")

return super().cqd_score(
iterations=iterations,
target_points=target_points,
penalties=penalties,
obj_min=obj_min,
obj_max=obj_max,
dist_max=dist_max,
dist_ord=dist_ord,
)
190 changes: 23 additions & 167 deletions tests/archives/unstructured_archive_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,170 +531,26 @@ def test_nonfinite_inputs(data):
data.archive.index_of_single(data.measures)


# TODO: cqd_score

# def test_cqd_score_detects_wrong_shapes(data):
# with pytest.raises(ValueError):
# data.archive.cqd_score(
# iterations=1,
# target_points=np.array([1.0]), # Should be 3D.
# penalties=2,
# obj_min=0.0,
# obj_max=1.0,
# )

# with pytest.raises(ValueError):
# data.archive.cqd_score(
# iterations=1,
# target_points=3,
# penalties=[[1.0, 1.0]], # Should be 1D.
# obj_min=0.0,
# obj_max=1.0,
# )

# def test_cqd_score_with_one_elite():
# archive = UnstructuredArchive(
# solution_dim=2,
# measure_dim=2,
# k_neighbors=5,
# novelty_threshold=1.0,
# )
# archive.add_single([4.0, 4.0], 1.0, [0.0, 0.0])

# score = archive.cqd_score(
# iterations=1,
# # With this target point, the solution above at [0, 0] has a normalized
# # distance of 0.5, since it is halfway between the archive bounds of
# # (-1, -1) and (1, 1).
# target_points=np.array([[[1.0, 1.0]]]),
# penalties=2,
# obj_min=0.0,
# obj_max=1.0,
# ).mean

# # For theta=0, the score should be 1.0 - 0 * 0.5 = 1.0
# # For theta=1, the score should be 1.0 - 1 * 0.5 = 0.5
# assert np.isclose(score, 1.0 + 0.5)

# def test_cqd_score_with_max_dist():
# archive = UnstructuredArchive(
# solution_dim=2,
# measure_dim=2,
# k_neighbors=5,
# novelty_threshold=1.0,
# )
# archive.add_single([4.0, 4.0], 0.5, [0.0, 1.0])

# score = archive.cqd_score(
# iterations=1,
# # With this target point and dist_max, the solution above at [0, 1]
# # has a normalized distance of 0.5, since it is one unit away.
# target_points=np.array([[[1.0, 1.0]]]),
# penalties=2,
# obj_min=0.0,
# obj_max=1.0,
# dist_max=2.0,
# ).mean

# # For theta=0, the score should be 0.5 - 0 * 0.5 = 0.5
# # For theta=1, the score should be 0.5 - 1 * 0.5 = 0.0
# assert np.isclose(score, 0.5 + 0.0)

# def test_cqd_score_l1_norm():
# archive = UnstructuredArchive(
# solution_dim=2,
# measure_dim=2,
# k_neighbors=5,
# novelty_threshold=1.0,
# )
# archive.add_single([4.0, 4.0], 0.5, [0.0, 0.0])

# score = archive.cqd_score(
# iterations=1,
# # With this target point and dist_max, the solution above at [0, 0]
# # has a normalized distance of 1.0, since it is two units away.
# target_points=np.array([[[1.0, 1.0]]]),
# penalties=2,
# obj_min=0.0,
# obj_max=1.0,
# dist_max=2.0,
# # L1 norm.
# dist_ord=1,
# ).mean

# # For theta=0, the score should be 0.5 - 0 * 1.0 = 0.5
# # For theta=1, the score should be 0.5 - 1 * 1.0 = -0.5
# assert np.isclose(score, 0.5 + -0.5)

# def test_cqd_score_full_output():
# archive = UnstructuredArchive(
# solution_dim=2,
# measure_dim=2,
# k_neighbors=5,
# novelty_threshold=1.0,
# )
# archive.add_single([4.0, 4.0], 1.0, [0.0, 0.0])

# result = archive.cqd_score(
# iterations=5,
# # With this target point, the solution above at [0, 0] has a normalized
# # distance of 0.5, since it is halfway between the archive bounds of
# # (-1, -1) and (1, 1).
# target_points=np.array([
# [[1.0, 1.0]],
# [[1.0, 1.0]],
# [[1.0, 1.0]],
# [[-1.0, -1.0]],
# [[-1.0, -1.0]],
# ]),
# penalties=2,
# obj_min=0.0,
# obj_max=1.0,
# )

# # For theta=0, the score should be 1.0 - 0 * 0.5 = 1.0
# # For theta=1, the score should be 1.0 - 1 * 0.5 = 0.5
# assert result.iterations == 5
# assert np.isclose(result.mean, 1.0 + 0.5)
# assert np.all(np.isclose(result.scores, 1.0 + 0.5))
# assert np.all(
# np.isclose(
# result.target_points,
# np.array([
# [[1.0, 1.0]],
# [[1.0, 1.0]],
# [[1.0, 1.0]],
# [[-1.0, -1.0]],
# [[-1.0, -1.0]],
# ])))
# assert np.all(np.isclose(result.penalties, [0.0, 1.0]))
# assert np.isclose(result.obj_min, 0.0)
# assert np.isclose(result.obj_max, 1.0)
# # Distance from (-1,-1) to (1,1).
# assert np.isclose(result.dist_max, 2 * np.sqrt(2))
# assert result.dist_ord is None

# def test_cqd_score_with_two_elites():
# archive = UnstructuredArchive(
# solution_dim=2,
# measure_dim=2,
# k_neighbors=5,
# novelty_threshold=1.0,
# )
# archive.add_single([4.0, 4.0], 0.25, [-2.0, -2.0]) # Elite 2.
# archive.add_single([4.0, 4.0], 0.0, [2.0, 2.0]) # Elite 3.

# score = archive.cqd_score(
# iterations=1,
# # With this target point, Elite 1 has a normalized distance of 1, since
# # it is exactly at [-2, -2].
# #
# # Elite 2 has a normalized distance of 0, since it is exactly at [2, 2].
# target_points=np.array([[[2.0, 2.0]]]),
# penalties=2, # Penalties of 0 and 1.
# obj_min=0.0,
# obj_max=1.0,
# ).mean
# # For theta=0, the score should be max(0.25 - 0 * 1.0, 0 - 0 * 0.0) = 0.25
# # For theta=1, the score should be max(0.25 - 1 * 1.0, 0 - 1 * 0.0) = 0.0
# assert np.isclose(score, 0.25 + 0)
def test_cqd_score_with_max_dist():
archive = UnstructuredArchive(
solution_dim=2,
measure_dim=2,
k_neighbors=5,
novelty_threshold=1.0,
)
archive.add_single([4.0, 4.0], 0.5, [0.0, 1.0])

score = archive.cqd_score(
iterations=1,
# With this target point and dist_max, the solution above at [0, 1]
# has a normalized distance of 0.5, since it is one unit away.
target_points=np.array([[[1.0, 1.0]]]),
penalties=2,
obj_min=0.0,
obj_max=1.0,
dist_max=2.0,
).mean

# For theta=0, the score should be 0.5 - 0 * 0.5 = 0.5
# For theta=1, the score should be 0.5 - 1 * 0.5 = 0.0
assert np.isclose(score, 0.5 + 0.0)

0 comments on commit 3cf2adb

Please sign in to comment.