From 6a00111701d516f999274c0b717ba68fb5535abf Mon Sep 17 00:00:00 2001 From: Feiran Jia Date: Fri, 26 May 2023 09:20:10 -0400 Subject: [PATCH 1/6] icl api --- flaml/autogen/oai/__init__.py | 3 +- flaml/autogen/oai/exemplar_selector.py | 55 ++++++++++++++++++++++++++ flaml/autogen/oai/selection_methods.py | 13 ++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 flaml/autogen/oai/exemplar_selector.py create mode 100644 flaml/autogen/oai/selection_methods.py diff --git a/flaml/autogen/oai/__init__.py b/flaml/autogen/oai/__init__.py index 5457351b1b..519bdd6575 100644 --- a/flaml/autogen/oai/__init__.py +++ b/flaml/autogen/oai/__init__.py @@ -1,4 +1,5 @@ from flaml.autogen.oai.completion import Completion, ChatCompletion from flaml.autogen.oai.openai_utils import get_config_list, config_list_gpt4_gpt35, config_list_openai_aoai +from flaml.autogen.oai.exemplar_selector import ExemplarSelector -__all__ = ["Completion", "ChatCompletion", "get_config_list", "config_list_gpt4_gpt35", "config_list_openai_aoai"] +__all__ = ["Completion", "ChatCompletion", "get_config_list", "config_list_gpt4_gpt35", "config_list_openai_aoai", "ExemplarSelector"] diff --git a/flaml/autogen/oai/exemplar_selector.py b/flaml/autogen/oai/exemplar_selector.py new file mode 100644 index 0000000000..93ae03b261 --- /dev/null +++ b/flaml/autogen/oai/exemplar_selector.py @@ -0,0 +1,55 @@ +from functools import partial +import random +import numpy as np +from flaml.autogen.oai.selection_methods import RandomSelect + +class ExemplarSelector: + METHOD_MAPPING = { + "random": RandomSelect, + # You can add more methods here... + } + + @classmethod + def get_few_shot_template(cls, train_data, method=None, few_shot_template=None, method_params=None, template_params=None): + if isinstance(method, str): + method_class = cls.METHOD_MAPPING.get(method, None) + if method_class is not None: + method = method_class(train_data, **method_params).select + else: + raise ValueError(f"The specified method '{method}' is not recognized.") + return partial(cls.construct_template, train_data=train_data, method=method, + few_shot_template=few_shot_template, method_params=method_params or {}, + template_params=template_params or {}) + + @staticmethod + def construct_template(context, train_data, method=None, few_shot_template=None, method_params=None, template_params=None): + + if method is None: + k = method_params.get('k', np.inf) + exemplars = train_data[:k] if len(train_data) >= k else train_data + else: + exemplars = method(context) + + if few_shot_template is not None: + return few_shot_template(context, exemplars=exemplars) + else: + key_order = template_params.get('key_order', None) + return ExemplarSelector.default_template(context, exemplars, key_order) + + @staticmethod + def default_template(context, exemplars, key_order): + few_shot_prompt = "" + for examplar in exemplars: + few_shot_prompt += "\n".join( + [ + key + ": " + str(examplar[key]) for key in key_order + ] + ) + "\n" + few_shot_prompt += "\n".join( + [ + key + ": " + str(context[key]) for key in key_order[:-1] + ] + ) + few_shot_prompt += "\n" + key_order[-1] + ": " + "\n" + return few_shot_prompt + diff --git a/flaml/autogen/oai/selection_methods.py b/flaml/autogen/oai/selection_methods.py new file mode 100644 index 0000000000..b5bb5aff7d --- /dev/null +++ b/flaml/autogen/oai/selection_methods.py @@ -0,0 +1,13 @@ +import random +class RandomSelect: + def __init__(self, data, k=None): + self.data = data + self.k = k + # You can also add code to learn parameters here. + + def select(self, context): + data_without_context = [item for item in self.data if item != context] + if self.k is None: + return data_without_context + else: + return random.sample(data_without_context, self.k) From a1413fe32412d99dfe1459df184c0b392fa7d5fd Mon Sep 17 00:00:00 2001 From: Feiran Jia Date: Sat, 27 May 2023 21:36:50 -0400 Subject: [PATCH 2/6] add test for selector --- test/autogen/oai/test_selection.py | 93 ++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 test/autogen/oai/test_selection.py diff --git a/test/autogen/oai/test_selection.py b/test/autogen/oai/test_selection.py new file mode 100644 index 0000000000..1a22dac07b --- /dev/null +++ b/test/autogen/oai/test_selection.py @@ -0,0 +1,93 @@ +import unittest +import datasets +from flaml import oai + +class TestExemplarSelector(unittest.TestCase): + ''' + template: default or user-defined, if default, key_order is specific + method: existing (str) or user-defined (func) or None (return in the order of train_data) + ''' + @classmethod + def setUpClass(cls): + seed = 41 + cls.data = datasets.load_dataset("piqa") + cls.train_data = cls.data["train"].shuffle(seed=seed) + cls.test_data = cls.data["test"].shuffle(seed=seed) + cls.key_order = ["goal", "sol1", "sol2", "label"] + cls.exemplars = list(cls.train_data)[:8] + cls.context = list(cls.test_data)[0] + + def test_case_existing_method_default_template(self): + # Most cases should use the default template and existing methods + prompt_fn = oai.ExemplarSelector.get_few_shot_template(self.exemplars, method="random", + method_params={"k": 3}, template_params={"key_order": self.key_order}) + output= prompt_fn(self.context) + #print("Existing method + default template: prompt = ", output) + self.assertIsInstance(output, str) + self.assertIn(self.context[self.key_order[0]], output) + + def test_case_user_template_no_method(self): + # User specify their own template + method is not specified, k is specific + key_order = ["sol1", "sol2", "label"] + def few_shot_template(context, exemplars=None): + few_shot_prompt = "User template:\n" + + for exemplar in exemplars: + few_shot_prompt += "\n".join( + [ + key + ": " + str(exemplar[key]) for key in key_order + ] + ) + "\n" + few_shot_prompt += "\n".join( + [ + key + ": " + str(context[key]) for key in key_order[:-1] + ] + ) + few_shot_prompt += "\n" + key_order[-1] + ": " + "\n" + return few_shot_prompt + prompt_fn = oai.ExemplarSelector.get_few_shot_template(self.exemplars, + few_shot_template=few_shot_template, + method_params={"k": 3}) + output= prompt_fn(self.context) + #print("test_user_template_No_Method: prompt = ", output) + self.assertIsInstance(output, str) + self.assertIn(self.context[key_order[0]], output) + self.assertIn(self.context[key_order[1]], output) + # should pick first 3 exemplars + self.assertIn(self.exemplars[0][key_order[1]], output) + self.assertIn(self.exemplars[1][key_order[1]], output) + self.assertIn(self.exemplars[2][key_order[1]], output) + self.assertNotIn(self.exemplars[3][key_order[1]], output) + + + def test_case_user_method_no_k(self): + # User specify their method is not specified, k is not specific + def user_method(context): + return self.exemplars[3:5] + # key_order should be provided if we use the default template + prompt_fn = oai.ExemplarSelector.get_few_shot_template(self.exemplars, + method = user_method, + template_params={"key_order": self.key_order}) + output= prompt_fn(self.context) + #print("test_user_method_no_k: prompt = ", output) + self.assertIsInstance(output, str) + self.assertIn(self.context[self.key_order[0]], output) + self.assertIn(self.context[self.key_order[1]], output) + self.assertIn(self.context[self.key_order[2]], output) + # should pick the 3rd,4th exemplars + self.assertIn(self.exemplars[3][self.key_order[2]], output) + self.assertIn(self.exemplars[4][self.key_order[2]], output) + self.assertNotIn(self.exemplars[5][self.key_order[2]], output) + self.assertNotIn(self.exemplars[2][self.key_order[2]], output) + + def test_invalid_method(self): + with self.assertRaises(ValueError): + oai.ExemplarSelector.get_few_shot_template(self.exemplars, method="nonexistent") + + + + +if __name__ == '__main__': + unittest.main() + + From 7f703ef46558864758b31f7733430358b3871be2 Mon Sep 17 00:00:00 2001 From: Feiran Jia Date: Sat, 27 May 2023 21:47:19 -0400 Subject: [PATCH 3/6] add abstract class --- flaml/autogen/oai/exemplar_selector.py | 3 +++ flaml/autogen/oai/selection_methods.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/flaml/autogen/oai/exemplar_selector.py b/flaml/autogen/oai/exemplar_selector.py index 93ae03b261..ce7a01c5b0 100644 --- a/flaml/autogen/oai/exemplar_selector.py +++ b/flaml/autogen/oai/exemplar_selector.py @@ -33,6 +33,9 @@ def construct_template(context, train_data, method=None, few_shot_template=None, if few_shot_template is not None: return few_shot_template(context, exemplars=exemplars) else: + if 'key_order' not in template_params: + raise ValueError("No 'key_order' found in 'template_params'. 'key_order' is required when no 'few_shot_template' is provided.") + key_order = template_params.get('key_order', None) return ExemplarSelector.default_template(context, exemplars, key_order) diff --git a/flaml/autogen/oai/selection_methods.py b/flaml/autogen/oai/selection_methods.py index b5bb5aff7d..4f44d0019a 100644 --- a/flaml/autogen/oai/selection_methods.py +++ b/flaml/autogen/oai/selection_methods.py @@ -1,9 +1,19 @@ +from abc import ABC, abstractmethod import random -class RandomSelect: + +class SelectMethod(ABC): + @abstractmethod + def __init__(self, data): + pass + + @abstractmethod + def select(self, context): + pass + +class RandomSelect(SelectMethod): def __init__(self, data, k=None): self.data = data self.k = k - # You can also add code to learn parameters here. def select(self, context): data_without_context = [item for item in self.data if item != context] @@ -11,3 +21,5 @@ def select(self, context): return data_without_context else: return random.sample(data_without_context, self.k) + + From cb5df7aa526b8e8514b9c5cc800e13b33d81e81d Mon Sep 17 00:00:00 2001 From: Feiran Jia Date: Sat, 27 May 2023 21:53:00 -0400 Subject: [PATCH 4/6] exam to exem --- flaml/autogen/oai/exemplar_selector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flaml/autogen/oai/exemplar_selector.py b/flaml/autogen/oai/exemplar_selector.py index ce7a01c5b0..78d5ec2597 100644 --- a/flaml/autogen/oai/exemplar_selector.py +++ b/flaml/autogen/oai/exemplar_selector.py @@ -42,10 +42,10 @@ def construct_template(context, train_data, method=None, few_shot_template=None, @staticmethod def default_template(context, exemplars, key_order): few_shot_prompt = "" - for examplar in exemplars: + for exemplar in exemplars: few_shot_prompt += "\n".join( [ - key + ": " + str(examplar[key]) for key in key_order + key + ": " + str(exemplar[key]) for key in key_order ] ) + "\n" few_shot_prompt += "\n".join( From 0edafbd1b8aac0a35216e6c80b06d79c1b301d90 Mon Sep 17 00:00:00 2001 From: Feiran Jia Date: Sun, 28 May 2023 18:30:20 -0400 Subject: [PATCH 5/6] move to icl/ --- flaml/autogen/icl/__init__.py | 3 +++ flaml/autogen/{oai => icl}/exemplar_selector.py | 2 +- flaml/autogen/{oai => icl}/selection_methods.py | 0 flaml/autogen/oai/__init__.py | 3 +-- test/autogen/{oai => icl}/test_selection.py | 10 +++++----- 5 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 flaml/autogen/icl/__init__.py rename flaml/autogen/{oai => icl}/exemplar_selector.py (97%) rename flaml/autogen/{oai => icl}/selection_methods.py (100%) rename test/autogen/{oai => icl}/test_selection.py (93%) diff --git a/flaml/autogen/icl/__init__.py b/flaml/autogen/icl/__init__.py new file mode 100644 index 0000000000..a2ed99aba3 --- /dev/null +++ b/flaml/autogen/icl/__init__.py @@ -0,0 +1,3 @@ +from flaml.autogen.icl.exemplar_selector import ExemplarSelector + +__all__ = ["ExemplarSelector"] diff --git a/flaml/autogen/oai/exemplar_selector.py b/flaml/autogen/icl/exemplar_selector.py similarity index 97% rename from flaml/autogen/oai/exemplar_selector.py rename to flaml/autogen/icl/exemplar_selector.py index 78d5ec2597..3d79a162a3 100644 --- a/flaml/autogen/oai/exemplar_selector.py +++ b/flaml/autogen/icl/exemplar_selector.py @@ -1,7 +1,7 @@ from functools import partial import random import numpy as np -from flaml.autogen.oai.selection_methods import RandomSelect +from flaml.autogen.icl.selection_methods import RandomSelect class ExemplarSelector: METHOD_MAPPING = { diff --git a/flaml/autogen/oai/selection_methods.py b/flaml/autogen/icl/selection_methods.py similarity index 100% rename from flaml/autogen/oai/selection_methods.py rename to flaml/autogen/icl/selection_methods.py diff --git a/flaml/autogen/oai/__init__.py b/flaml/autogen/oai/__init__.py index 519bdd6575..2cf4820587 100644 --- a/flaml/autogen/oai/__init__.py +++ b/flaml/autogen/oai/__init__.py @@ -1,5 +1,4 @@ from flaml.autogen.oai.completion import Completion, ChatCompletion from flaml.autogen.oai.openai_utils import get_config_list, config_list_gpt4_gpt35, config_list_openai_aoai -from flaml.autogen.oai.exemplar_selector import ExemplarSelector -__all__ = ["Completion", "ChatCompletion", "get_config_list", "config_list_gpt4_gpt35", "config_list_openai_aoai", "ExemplarSelector"] +__all__ = ["Completion", "ChatCompletion", "get_config_list", "config_list_gpt4_gpt35", "config_list_openai_aoai"] \ No newline at end of file diff --git a/test/autogen/oai/test_selection.py b/test/autogen/icl/test_selection.py similarity index 93% rename from test/autogen/oai/test_selection.py rename to test/autogen/icl/test_selection.py index 1a22dac07b..05c45fa13a 100644 --- a/test/autogen/oai/test_selection.py +++ b/test/autogen/icl/test_selection.py @@ -1,6 +1,6 @@ import unittest import datasets -from flaml import oai +from flaml import icl class TestExemplarSelector(unittest.TestCase): ''' @@ -19,7 +19,7 @@ def setUpClass(cls): def test_case_existing_method_default_template(self): # Most cases should use the default template and existing methods - prompt_fn = oai.ExemplarSelector.get_few_shot_template(self.exemplars, method="random", + prompt_fn = icl.ExemplarSelector.get_few_shot_template(self.exemplars, method="random", method_params={"k": 3}, template_params={"key_order": self.key_order}) output= prompt_fn(self.context) #print("Existing method + default template: prompt = ", output) @@ -45,7 +45,7 @@ def few_shot_template(context, exemplars=None): ) few_shot_prompt += "\n" + key_order[-1] + ": " + "\n" return few_shot_prompt - prompt_fn = oai.ExemplarSelector.get_few_shot_template(self.exemplars, + prompt_fn = icl.ExemplarSelector.get_few_shot_template(self.exemplars, few_shot_template=few_shot_template, method_params={"k": 3}) output= prompt_fn(self.context) @@ -65,7 +65,7 @@ def test_case_user_method_no_k(self): def user_method(context): return self.exemplars[3:5] # key_order should be provided if we use the default template - prompt_fn = oai.ExemplarSelector.get_few_shot_template(self.exemplars, + prompt_fn = icl.ExemplarSelector.get_few_shot_template(self.exemplars, method = user_method, template_params={"key_order": self.key_order}) output= prompt_fn(self.context) @@ -82,7 +82,7 @@ def user_method(context): def test_invalid_method(self): with self.assertRaises(ValueError): - oai.ExemplarSelector.get_few_shot_template(self.exemplars, method="nonexistent") + icl.ExemplarSelector.get_few_shot_template(self.exemplars, method="nonexistent") From cc752df4c0f3cf96f403268876c72047e9f407fa Mon Sep 17 00:00:00 2001 From: Feiran Jia Date: Sun, 28 May 2023 18:32:57 -0400 Subject: [PATCH 6/6] add icl --- flaml/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flaml/__init__.py b/flaml/__init__.py index 9fca486499..9cd8ead0da 100644 --- a/flaml/__init__.py +++ b/flaml/__init__.py @@ -3,6 +3,7 @@ from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearch from flaml.onlineml.autovw import AutoVW from flaml.autogen import oai +from flaml.autogen import icl from flaml.version import __version__