[WIP] Version 4 with better open source model support and no langchain (

#223)

* First draft without langchain

* Improved unit tests to be closer

* Fixed remaining tests

* Added new dependencies

* Refactored LLMs to allow swapping

* Added unit tests for custom embeds/llms

* Fixed langchain compatibility and updated README

* Refactored vector stores to maybe support langchain

* Addressed Matt's comments

* Finished langchain vector store

* Unit test prompt adjustments

* Added warning to README

* Fixed some typos in README

* Fixed problem for very short texts

* Made it easier to access LLM names

* Fixed text embedding errors

dev-requirements.txt

@@ -6,4 +6,6 @@ python-dotenv
 pymupdf
 build
 types-requests
-numpy
+langchain_openai
+langchain_community
+faiss-cpu

paperqa/__init__.py

@@ -1,5 +1,17 @@
 from .docs import Answer, Docs, PromptCollection, Doc, Text, Context
 from .version import __version__
+from .llms import (
+    LLMModel,
+    EmbeddingModel,
+    LangchainEmbeddingModel,
+    OpenAIEmbeddingModel,
+    LangchainLLMModel,
+    OpenAILLMModel,
+    LlamaEmbeddingModel,
+    NumpyVectorStore,
+    LangchainVectorStore,
+    SentenceTransformerEmbeddingModel,
+)
 
 __all__ = [
     "Docs",
@@ -9,4 +21,14 @@
     "Doc",
     "Text",
     "Context",
+    "LLMModel",
+    "EmbeddingModel",
+    "OpenAIEmbeddingModel",
+    "OpenAILLMModel",
+    "LangchainLLMModel",
+    "LlamaEmbeddingModel",
+    "SentenceTransformerEmbeddingModel",
+    "LangchainEmbeddingModel",
+    "NumpyVectorStore",
+    "LangchainVectorStore",
 ]

paperqa/contrib/zotero.py

@@ -4,10 +4,7 @@
 from pathlib import Path
 from typing import List, Optional, Union, cast
 
-try:
-    from pydantic.v1 import BaseModel
-except ImportError:
-    from pydantic import BaseModel
+from pydantic import BaseModel
 
 try:
     from pyzotero import zotero