clean config

.env.template

@@ -49,6 +49,14 @@ OPENAI_API_KEY=your-openai-api-key
 # FAST_TOKEN_LIMIT=4000
 # SMART_TOKEN_LIMIT=8000
 
+### EMBEDDINGS
+## EMBEDDING_MODEL       - Model to use for creating embeddings
+## EMBEDDING_TOKENIZER   - Tokenizer to use for chunking large inputs
+## EMBEDDING_TOKEN_LIMIT - Chunk size limit for large inputs
+# EMBEDDING_MODEL=text-embedding-ada-002
+# EMBEDDING_TOKENIZER=cl100k_base
+# EMBEDDING_TOKEN_LIMIT=8191
+
 ################################################################################
 ### MEMORY
 ################################################################################

autogpt/agent/agent.py

@@ -5,6 +5,7 @@
 from autogpt.json_utils.json_fix_llm import fix_json_using_multiple_techniques
 from autogpt.json_utils.utilities import LLM_DEFAULT_RESPONSE_FORMAT, validate_json
 from autogpt.llm import chat_with_ai, create_chat_completion, create_chat_message
+from autogpt.llm.token_counter import count_string_tokens
 from autogpt.logs import logger, print_assistant_thoughts
 from autogpt.speech import say_text
 from autogpt.spinner import Spinner
@@ -233,6 +234,16 @@ def start_interaction_loop(self):
                 )
                 result = f"Command {command_name} returned: " f"{command_result}"
 
+                result_tlength = count_string_tokens(
+                    str(command_result), cfg.fast_llm_model
+                )
+                memory_tlength = count_string_tokens(
+                    str(self.summary_memory), cfg.fast_llm_model
+                )
+                if result_tlength + memory_tlength + 600 > cfg.fast_token_limit:
+                    result = f"Failure: command {command_name} returned too much output. \
+                        Do not execute this command again with the same arguments."
+
                 for plugin in cfg.plugins:
                     if not plugin.can_handle_post_command():
                         continue

autogpt/command_list.json

@@ -0,0 +1,14 @@
+[
+    "autogpt.commands.analyze_code",
+    "autogpt.commands.audio_text",
+    "autogpt.commands.execute_code",
+    "autogpt.commands.file_operations",
+    "autogpt.commands.git_operations",
+    "autogpt.commands.google_search",
+    "autogpt.commands.image_gen",
+    "autogpt.commands.improve_code",
+    "autogpt.commands.twitter",
+    "autogpt.commands.web_selenium",
+    "autogpt.commands.write_tests",
+    "autogpt.app"
+]

autogpt/commands/command.py

@@ -49,8 +49,10 @@ class CommandRegistry:
     directory.
     """
 
-    def __init__(self):
+    def __init__(self, module_names: list = []):
         self.commands = {}
+        if module_names:
+            self.import_command_list(module_names)
 
     def _import_module(self, module_name: str) -> Any:
         return importlib.import_module(module_name)
@@ -123,6 +125,18 @@ def import_commands(self, module_name: str) -> None:
                 cmd_instance = attr()
                 self.register(cmd_instance)
 
+    def import_command_list(self, module_names: list) -> None:
+        """
+        Imports the specified Python modules containing command plugins.
+
+        This method calls import_commands for each command in the list.
+
+        Args:
+            module_name (list): The name of the module to import for command plugins.
+        """
+        for module_name in module_names:
+            self.import_commands(module_name)
+
 
 def command(
     name: str,

autogpt/config/config.py

@@ -35,6 +35,9 @@ def __init__(self) -> None:
         self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4")
         self.fast_token_limit = int(os.getenv("FAST_TOKEN_LIMIT", 4000))
         self.smart_token_limit = int(os.getenv("SMART_TOKEN_LIMIT", 8000))
+        self.embedding_model = os.getenv("EMBEDDING_MODEL", "text-embedding-ada-002")
+        self.embedding_tokenizer = os.getenv("EMBEDDING_TOKENIZER", "cl100k_base")
+        self.embedding_token_limit = int(os.getenv("EMBEDDING_TOKEN_LIMIT", 8191))
         self.browse_chunk_max_length = int(os.getenv("BROWSE_CHUNK_MAX_LENGTH", 3000))
         self.browse_spacy_language_model = os.getenv(
             "BROWSE_SPACY_LANGUAGE_MODEL", "en_core_web_sm"
@@ -216,6 +219,18 @@ def set_smart_token_limit(self, value: int) -> None:
         """Set the smart token limit value."""
         self.smart_token_limit = value
 
+    def set_embedding_model(self, value: str) -> None:
+        """Set the model to use for creating embeddings."""
+        self.embedding_model = value
+
+    def set_embedding_tokenizer(self, value: str) -> None:
+        """Set the tokenizer to use when creating embeddings."""
+        self.embedding_tokenizer = value
+
+    def set_embedding_token_limit(self, value: int) -> None:
+        """Set the token limit for creating embeddings."""
+        self.embedding_token_limit = value
+
     def set_browse_chunk_max_length(self, value: int) -> None:
         """Set the browse_website command chunk max length value."""
         self.browse_chunk_max_length = value

autogpt/configurator.py

@@ -7,8 +7,6 @@
 from autogpt.logs import logger
 from autogpt.memory import get_supported_memory_backends
 
-CFG = Config()
-
 
 def create_config(
     continuous: bool,
@@ -23,7 +21,7 @@ def create_config(
     browser_name: str,
     allow_downloads: bool,
     skip_news: bool,
-) -> None:
+) -> Config:
     """Updates the config object with the given arguments.
 
     Args:
@@ -40,6 +38,7 @@ def create_config(
         allow_downloads (bool): Whether to allow Auto-GPT to download files natively
         skips_news (bool): Whether to suppress the output of latest news on startup
     """
+    CFG = Config()
     CFG.set_debug_mode(False)
     CFG.set_continuous_mode(False)
     CFG.set_speak_mode(False)
@@ -132,3 +131,5 @@ def create_config(
 
     if skip_news:
         CFG.skip_news = True
+
+    return CFG

autogpt/json_utils/utilities.py

@@ -1,5 +1,6 @@
 """Utilities for the json_fixes package."""
 import json
+import os
 import re
 
 from jsonschema import Draft7Validator
@@ -35,7 +36,8 @@ def validate_json(json_object: object, schema_name: str) -> dict | None:
     :param schema_name: str
     :type json_object: object
     """
-    with open(f"autogpt/json_utils/{schema_name}.json", "r") as f:
+    scheme_file = os.path.join(os.path.dirname(__file__), f"{schema_name}.json")
+    with open(scheme_file, "r") as f:
         schema = json.load(f)
     validator = Draft7Validator(schema)
 

autogpt/llm/__init__.py

@@ -11,6 +11,7 @@
 from autogpt.llm.chat import chat_with_ai, create_chat_message, generate_context
 from autogpt.llm.llm_utils import (
     call_ai_function,
+    chunked_tokens,
     create_chat_completion,
     get_ada_embedding,
 )
@@ -32,6 +33,7 @@
     "call_ai_function",
     "create_chat_completion",
     "get_ada_embedding",
+    "chunked_tokens",
     "COSTS",
     "count_message_tokens",
     "count_string_tokens",

autogpt/llm/llm_utils.py

@@ -2,9 +2,12 @@
 
 import functools
 import time
+from itertools import islice
 from typing import List, Optional
 
+import numpy as np
 import openai
+import tiktoken
 from colorama import Fore, Style
 from openai.error import APIError, RateLimitError, Timeout
 
@@ -207,6 +210,23 @@ def create_chat_completion(
     return resp
 
 
+def batched(iterable, n):
+    """Batch data into tuples of length n. The last batch may be shorter."""
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    it = iter(iterable)
+    while batch := tuple(islice(it, n)):
+        yield batch
+
+
+def chunked_tokens(text, tokenizer_name, chunk_length):
+    tokenizer = tiktoken.get_encoding(tokenizer_name)
+    tokens = tokenizer.encode(text)
+    chunks_iterator = batched(tokens, chunk_length)
+    yield from chunks_iterator
+
+
 def get_ada_embedding(text: str) -> List[float]:
     """Get an embedding from the ada model.
 
@@ -217,7 +237,7 @@ def get_ada_embedding(text: str) -> List[float]:
         List[float]: The embedding.
     """
     cfg = Config()
-    model = "text-embedding-ada-002"
+    model = cfg.embedding_model
     text = text.replace("\n", " ")
 
     if cfg.use_azure:
@@ -226,13 +246,7 @@ def get_ada_embedding(text: str) -> List[float]:
         kwargs = {"model": model}
 
     embedding = create_embedding(text, **kwargs)
-    api_manager = ApiManager()
-    api_manager.update_cost(
-        prompt_tokens=embedding.usage.prompt_tokens,
-        completion_tokens=0,
-        model=model,
-    )
-    return embedding["data"][0]["embedding"]
+    return embedding
 
 
 @retry_openai_api()
@@ -251,8 +265,31 @@ def create_embedding(
         openai.Embedding: The embedding object.
     """
     cfg = Config()
-    return openai.Embedding.create(
-        input=[text],
-        api_key=cfg.openai_api_key,
-        **kwargs,
-    )
+    chunk_embeddings = []
+    chunk_lengths = []
+    for chunk in chunked_tokens(
+        text,
+        tokenizer_name=cfg.embedding_tokenizer,
+        chunk_length=cfg.embedding_token_limit,
+    ):
+        embedding = openai.Embedding.create(
+            input=[chunk],
+            api_key=cfg.openai_api_key,
+            **kwargs,
+        )
+        api_manager = ApiManager()
+        api_manager.update_cost(
+            prompt_tokens=embedding.usage.prompt_tokens,
+            completion_tokens=0,
+            model=cfg.embedding_model,
+        )
+        chunk_embeddings.append(embedding["data"][0]["embedding"])
+        chunk_lengths.append(len(chunk))
+
+    # do weighted avg
+    chunk_embeddings = np.average(chunk_embeddings, axis=0, weights=chunk_lengths)
+    chunk_embeddings = chunk_embeddings / np.linalg.norm(
+        chunk_embeddings
+    )  # normalize the length to one
+    chunk_embeddings = chunk_embeddings.tolist()
+    return chunk_embeddings

autogpt/llm/modelsinfo.py

@@ -3,5 +3,8 @@
     "gpt-3.5-turbo-0301": {"prompt": 0.002, "completion": 0.002},
     "gpt-4-0314": {"prompt": 0.03, "completion": 0.06},
     "gpt-4": {"prompt": 0.03, "completion": 0.06},
+    "gpt-4-0314": {"prompt": 0.03, "completion": 0.06},
+    "gpt-4-32k": {"prompt": 0.06, "completion": 0.12},
+    "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12},
     "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
 }