Meilisearch Demo
+
+ + Use this demo to verify that the sync between Appwrite Databases and + Meilisearch was successful. Search your Meilisearch index using the + input below. +
+diff --git a/python/sync_with_meilisearch/.gitignore b/python/sync_with_meilisearch/.gitignore new file mode 100644 index 00000000..68bc17f9 --- /dev/null +++ b/python/sync_with_meilisearch/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/python/sync_with_meilisearch/README.md b/python/sync_with_meilisearch/README.md new file mode 100644 index 00000000..f218cd72 --- /dev/null +++ b/python/sync_with_meilisearch/README.md @@ -0,0 +1,106 @@ +# ⚡ Python Sync with Meilisearch Function + +Syncs documents in an Appwrite database collection to a Meilisearch index. + +## 🧰 Usage + +### GET / + +Returns HTML page where search can be performed to test the indexing. + +### POST / + +Triggers indexing of the Appwrite database collection to Meilisearch. + +**Response** + +Sample `204` Response: No content. + +## ⚙️ Configuration + +| Setting | Value | +| ----------------- | --------------------------------- | +| Runtime | Python (3.9) | +| Entrypoint | `src/main.py` | +| Build Commands | `pip install -r requirements.txt` | +| Permissions | `any` | +| Timeout (Seconds) | 15 | + +## 🔒 Environment Variables + +### APPWRITE_API_KEY + +API Key to talk to Appwrite backend APIs. + +| Question | Answer | +| ------------- | -------------------------------------------------------------------------------------------------- | +| Required | Yes | +| Sample Value | `d1efb...aec35` | +| Documentation | [Appwrite: Getting Started for Server](https://appwrite.io/docs/getting-started-for-server#apiKey) | + +### APPWRITE_DATABASE_ID + +The ID of the Appwrite database that contains the collection to sync. + +| Question | Answer | +| ------------- | --------------------------------------------------------- | +| Required | Yes | +| Sample Value | `612a3...5b6c9` | +| Documentation | [Appwrite: Databases](https://appwrite.io/docs/databases) | + +### APPWRITE_COLLECTION_ID + +The ID of the collection in the Appwrite database to sync. + +| Question | Answer | +| ------------- | ------------------------------------------------------------- | +| Required | Yes | +| Sample Value | `7c3e8...2a9f1` | +| Documentation | [Appwrite: Collections](https://appwrite.io/docs/databases#collection) | + +### APPWRITE_ENDPOINT + +The URL endpoint of the Appwrite server. If not provided, it defaults to the Appwrite Cloud server: `https://cloud.appwrite.io/v1`. + +| Question | Answer | +| ------------ | ------------------------------ | +| Required | No | +| Sample Value | `https://cloud.appwrite.io/v1` | + +### MEILISEARCH_ENDPOINT + +The host URL of the Meilisearch server. + +| Question | Answer | +| ------------ | ----------------------- | +| Required | Yes | +| Sample Value | `http://127.0.0.1:7700` | + +### MEILISEARCH_ADMIN_API_KEY + +The admin API key for Meilisearch. + +| Question | Answer | +| ------------- | ------------------------------------------------------------------------ | +| Required | Yes | +| Sample Value | `masterKey1234` | +| Documentation | [Meilisearch: API Keys](https://docs.meilisearch.com/reference/api/keys) | + +### MEILISEARCH_INDEX_NAME + +Name of the Meilisearch index to which the documents will be synchronized. + +| Question | Answer | +| ------------ | ---------- | +| Required | Yes | +| Sample Value | `my_index` | + +### MEILISEARCH_SEARCH_API_KEY + +API Key for Meilisearch search operations. + +| Question | Answer | +| ------------- | ------------------------------------------------------------------------ | +| Required | Yes | +| Sample Value | `searchKey1234` | +| Documentation | [Meilisearch: API Keys](https://docs.meilisearch.com/reference/api/keys) | diff --git a/python/sync_with_meilisearch/requirements.txt b/python/sync_with_meilisearch/requirements.txt new file mode 100644 index 00000000..a4e76ddd --- /dev/null +++ b/python/sync_with_meilisearch/requirements.txt @@ -0,0 +1,2 @@ +appwrite +meilisearch \ No newline at end of file diff --git a/python/sync_with_meilisearch/src/main.py b/python/sync_with_meilisearch/src/main.py new file mode 100644 index 00000000..918721bf --- /dev/null +++ b/python/sync_with_meilisearch/src/main.py @@ -0,0 +1,65 @@ +import os +from appwrite.client import Client +from appwrite.services.databases import Databases +from meilisearch import Client as MeiliClient +from .utils import get_static_file, interpolate, throw_if_missing + +def main(context): + throw_if_missing(os.environ, [ + 'APPWRITE_API_KEY', + 'APPWRITE_DATABASE_ID', + 'APPWRITE_COLLECTION_ID', + 'MEILISEARCH_ENDPOINT', + 'MEILISEARCH_INDEX_NAME', + 'MEILISEARCH_ADMIN_API_KEY', + 'MEILISEARCH_SEARCH_API_KEY', + ]) + + if context.req.method == 'GET': + html = interpolate(get_static_file('index.html'), { + 'MEILISEARCH_ENDPOINT': os.environ['MEILISEARCH_ENDPOINT'], + 'MEILISEARCH_INDEX_NAME': os.environ['MEILISEARCH_INDEX_NAME'], + 'MEILISEARCH_SEARCH_API_KEY': os.environ['MEILISEARCH_SEARCH_API_KEY'], + }) + + return context.res.send(html, 200, {'content-type': 'text/html; charset=utf-8'}) + + client = Client() + client.set_endpoint(os.environ.get('APPWRITE_ENDPOINT', 'https://cloud.appwrite.io/v1')) + client.set_project(os.environ['APPWRITE_FUNCTION_PROJECT_ID']) + client.set_key(os.environ['APPWRITE_API_KEY']) + + databases = Databases(client) + + meilisearch = MeiliClient(os.environ['MEILISEARCH_ENDPOINT'], os.environ['MEILISEARCH_ADMIN_API_KEY']) + index = meilisearch.index(os.environ['MEILISEARCH_INDEX_NAME']) + + cursor = None + + while True: + queries = [{'limit': 100}] + + if cursor: + queries.append({'cursorAfter': cursor}) + + response = databases.list_documents( + os.environ['APPWRITE_DATABASE_ID'], + os.environ['APPWRITE_COLLECTION_ID'], + queries + ) + + documents = response['documents'] + + if len(documents) > 0: + cursor = documents[-1]['$id'] + else: + context.log('No more documents found.') + cursor = None + break + + context.log(f'Syncing chunk of {len(documents)} documents...') + index.add_documents(documents, {'primaryKey': '$id'}) + + context.log('Sync finished.') + + return context.res.send('Sync finished.', 200) diff --git a/python/sync_with_meilisearch/src/utils.py b/python/sync_with_meilisearch/src/utils.py new file mode 100644 index 00000000..44c2cc61 --- /dev/null +++ b/python/sync_with_meilisearch/src/utils.py @@ -0,0 +1,52 @@ +import os +import re + +__dirname = os.path.dirname(os.path.abspath(__file__)) +static_folder = os.path.join(__dirname, "../static") + +def throw_if_missing(obj: dict, keys: list[str]) -> None: + """ + Throws an error if any of the keys are missing from the object + + Parameters: + obj (dict): Dictionary to check + keys (list[str]): List of keys to check + + Raises: + ValueError: If any keys are missing + """ + missing = [key for key in keys if key not in obj or not obj[key]] + if missing: + raise ValueError(f"Missing required fields: {', '.join(missing)}") + +def get_static_file(file_name: str) -> str: + """ + Returns the contents of a file in the static folder + + Parameters: + file_name (str): Name of the file to read + + Returns: + (str): Contents of static/{file_name} + """ + file_path = os.path.join(static_folder, file_name) + with open(file_path, "r") as file: + return file.read() + +def interpolate(template: str, values: dict[str, str]) -> str: + """ + Interpolates a template string with the given values + + Parameters: + template(str): Template string to interpolate + values(dict): Dictionary of values to interpolate + + Returns: + (str): Interpolated string + """ + + def replace_match(match): + key = match.group(1) + return values.get(key, "") + + return re.sub(r"{{([^}]+)}}", replace_match, template) diff --git a/python/sync_with_meilisearch/static/index.html b/python/sync_with_meilisearch/static/index.html new file mode 100644 index 00000000..ca23d017 --- /dev/null +++ b/python/sync_with_meilisearch/static/index.html @@ -0,0 +1,72 @@ + + +
+ + + +
+ + Use this demo to verify that the sync between Appwrite Databases and + Meilisearch was successful. Search your Meilisearch index using the + input below. +
+