From 34de288b52e7c254e995430bb60b9d07a52884e3 Mon Sep 17 00:00:00 2001 From: slatinsky Date: Sun, 12 Nov 2023 06:42:06 +0100 Subject: [PATCH] rename to a more inclusive synonyms - allowlist and denylist --- .gitignore | 2 +- backend/configurator/main.py | 118 +++++++++++++++---------------- backend/fastapi/Autocomplete.py | 6 +- backend/fastapi/app.py | 46 ++++++------ backend/fastapi/helpers.py | 22 +++--- backend/preprocess/main_mongo.py | 29 ++++++-- 6 files changed, 119 insertions(+), 104 deletions(-) diff --git a/.gitignore b/.gitignore index be3b319e..872820ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# global blacklist +# global denylist .DS_Store node_modules .env diff --git a/backend/configurator/main.py b/backend/configurator/main.py index 49f97a17..0383c2fa 100644 --- a/backend/configurator/main.py +++ b/backend/configurator/main.py @@ -14,32 +14,32 @@ def pad_id(id): def get_guilds(): return collection_guilds.find().sort("msg_count", pymongo.DESCENDING) -def get_whitelisted_guild_ids(): - whitelisted_guild_ids = collection_config.find_one({"key": "whitelisted_guild_ids"})["value"] - whitelisted_guild_ids = [pad_id(id) for id in whitelisted_guild_ids] - return whitelisted_guild_ids - -def get_blacklisted_user_ids(): - blacklisted_user_ids = collection_config.find_one({"key": "blacklisted_user_ids"})["value"] - blacklisted_user_ids = [pad_id(id) for id in blacklisted_user_ids] - return blacklisted_user_ids - -def set_whitelisted_guild_ids(whitelisted_guild_ids): - print("Applying whitelisted guild ids...") - collection_config.update_one({"key": "whitelisted_guild_ids"}, {"$set": {"value": whitelisted_guild_ids}}) - if len(whitelisted_guild_ids) == 0: - print("all guilds are now whitelisted") +def get_allowlisted_guild_ids(): + allowlisted_guild_ids = collection_config.find_one({"key": "allowlisted_guild_ids"})["value"] + allowlisted_guild_ids = [pad_id(id) for id in allowlisted_guild_ids] + return allowlisted_guild_ids + +def get_denylisted_user_ids(): + denylisted_user_ids = collection_config.find_one({"key": "denylisted_user_ids"})["value"] + denylisted_user_ids = [pad_id(id) for id in denylisted_user_ids] + return denylisted_user_ids + +def set_allowlisted_guild_ids(allowlisted_guild_ids): + print("Applying allowlisted guild ids...") + collection_config.update_one({"key": "allowlisted_guild_ids"}, {"$set": {"value": allowlisted_guild_ids}}) + if len(allowlisted_guild_ids) == 0: + print("all guilds are now allowlisted") else: - print(f"whitelisted {len(whitelisted_guild_ids)} guilds") + print(f"allowlisted {len(allowlisted_guild_ids)} guilds") print("") -def set_blacklisted_user_ids(blacklisted_user_ids): - print("Applying blacklisted user ids...") - collection_config.update_one({"key": "blacklisted_user_ids"}, {"$set": {"value": blacklisted_user_ids}}) - if len(blacklisted_user_ids) == 0: - print("no users are blacklisted") +def set_denylisted_user_ids(denylisted_user_ids): + print("Applying denylisted user ids...") + collection_config.update_one({"key": "denylisted_user_ids"}, {"$set": {"value": denylisted_user_ids}}) + if len(denylisted_user_ids) == 0: + print("no users are denylisted") else: - print(f"{len(blacklisted_user_ids)} users are now blacklisted") + print(f"{len(denylisted_user_ids)} users are now denylisted") def resolve_user_id(user_id): user_id = pad_id(user_id) @@ -52,33 +52,33 @@ def resolve_user_id(user_id): if user != None: return user['names'] + user['nicknames'] -def menu_whitelist(): +def menu_allowlist(): guilds = list(get_guilds()) - whitelisted_ids = get_whitelisted_guild_ids() - whitelisted_ids = [id for id in whitelisted_ids if id in [guild['_id'] for guild in guilds]] # remove invalid ids + allowlisted_ids = get_allowlisted_guild_ids() + allowlisted_ids = [id for id in allowlisted_ids if id in [guild['_id'] for guild in guilds]] # remove invalid ids while True: - print("choice | action | whitelist status | guild_name") + print("choice | action | allowlist status | guild_name") print("-------|------------------|------------------|--------------------------") print("CTRL+C | back | - | - ") for i, guild in enumerate(guilds): - is_whitelisted = guild['_id'] in whitelisted_ids - whitelisted_status = "" - if is_whitelisted: - whitelisted_status = "whitelisted " - elif len(whitelisted_ids) == 0: - whitelisted_status = "not-configured " + is_allowlisted = guild['_id'] in allowlisted_ids + allowlisted_status = "" + if is_allowlisted: + allowlisted_status = "allowlisted " + elif len(allowlisted_ids) == 0: + allowlisted_status = "not-configured " else: - whitelisted_status = "not-whitelisted" - print(f"{str(i).ljust(6)} | toggle whitelist | {whitelisted_status} | {guild['name']}") + allowlisted_status = "not-allowlisted" + print(f"{str(i).ljust(6)} | toggle allowlist | {allowlisted_status} | {guild['name']}") try: - index = int(input(f"Index of guild to toggle whitelist status (0-{len(guilds)-1}): ")) + index = int(input(f"Index of guild to toggle allowlist status (0-{len(guilds)-1}): ")) guild_id = guilds[index]['_id'] - if guild_id in whitelisted_ids: - whitelisted_ids.remove(guild_id) + if guild_id in allowlisted_ids: + allowlisted_ids.remove(guild_id) else: - whitelisted_ids.append(guild_id) + allowlisted_ids.append(guild_id) - set_whitelisted_guild_ids(whitelisted_ids) + set_allowlisted_guild_ids(allowlisted_ids) except ValueError: print("Invalid input") @@ -88,7 +88,7 @@ def menu_whitelist(): print("\nExiting...") break -def menu_add_user_to_blacklist(blacklisted_user_ids): +def menu_add_user_to_denylist(denylisted_user_ids): while True: try: print("\n(right click profile picture in DCEF -> Copy author ID)") @@ -100,40 +100,40 @@ def menu_add_user_to_blacklist(blacklisted_user_ids): continue print("Found user:", ', '.join(user_names)) - if user_id in blacklisted_user_ids: - print("This user is already blacklisted") + if user_id in denylisted_user_ids: + print("This user is already denylisted") continue - blacklisted_user_ids.append(user_id) - set_blacklisted_user_ids(blacklisted_user_ids) + denylisted_user_ids.append(user_id) + set_denylisted_user_ids(denylisted_user_ids) break except KeyboardInterrupt: print("\nExiting...") break - return blacklisted_user_ids + return denylisted_user_ids -def menu_blacklist_users(): - blacklisted_user_ids = list(get_blacklisted_user_ids()) +def menu_denylist_users(): + denylisted_user_ids = list(get_denylisted_user_ids()) while True: print("") print("choice | action | user_id | also_known_as") print("-------|-----------------------|--------------------------|--------------------------") print("CTRL+C | back | - | - ") - print("0 | blacklist new user | - | - ") - for i, user_id in enumerate(blacklisted_user_ids): - print(f"{str(i+1).ljust(5)} | remove from blacklist | {user_id} | {', '.join(resolve_user_id(user_id))}") + print("0 | hide new user | - | - ") + for i, user_id in enumerate(denylisted_user_ids): + print(f"{str(i+1).ljust(5)} | remove from denylist | {user_id} | {', '.join(resolve_user_id(user_id))}") try: - index = int(input(f"Choice (0-{len(blacklisted_user_ids)}): ")) + index = int(input(f"Choice (0-{len(denylisted_user_ids)}): ")) if index == 0: - blacklisted_user_ids = menu_add_user_to_blacklist(blacklisted_user_ids) + denylisted_user_ids = menu_add_user_to_denylist(denylisted_user_ids) else: - user_id = blacklisted_user_ids[index-1] - blacklisted_user_ids.remove(user_id) - set_blacklisted_user_ids(blacklisted_user_ids) + user_id = denylisted_user_ids[index-1] + denylisted_user_ids.remove(user_id) + set_denylisted_user_ids(denylisted_user_ids) continue @@ -166,15 +166,15 @@ def main(): print("choice | action") print("-------|-----------------") print("CTRL+C | exit ") - print("0 | Whitelist server") - print("1 | Blacklist user ") + print("0 | Show/hide servers") + print("1 | Hide users ") print("") try: choice = int(input("Choice (0-1): ")) if choice == 0: - menu_whitelist() + menu_allowlist() elif choice == 1: - menu_blacklist_users() + menu_denylist_users() else: print("Invalid choice") except ValueError: diff --git a/backend/fastapi/Autocomplete.py b/backend/fastapi/Autocomplete.py index e7e3c9bf..151ece3a 100644 --- a/backend/fastapi/Autocomplete.py +++ b/backend/fastapi/Autocomplete.py @@ -2,7 +2,7 @@ import pymongo -from helpers import get_blacklisted_user_ids, get_guild_collection +from helpers import get_denylisted_user_ids, get_guild_collection def autocomplete_categories(guild_id: str, partial_category: str, limit: int): @@ -208,7 +208,7 @@ def autocomplete_users(guild_id: str, partial_user_name: str, limit: int): collection_authors = get_guild_collection(guild_id, "authors") print("collection_authors", collection_authors) - blacklisted_user_ids = get_blacklisted_user_ids() + denylisted_user_ids = get_denylisted_user_ids() query = { "$or": [ @@ -224,7 +224,7 @@ def autocomplete_users(guild_id: str, partial_user_name: str, limit: int): } ], "_id": { - "$nin": blacklisted_user_ids + "$nin": denylisted_user_ids } } cursor = collection_authors.find(query, { diff --git a/backend/fastapi/app.py b/backend/fastapi/app.py index 88a08cf4..1366f0bb 100644 --- a/backend/fastapi/app.py +++ b/backend/fastapi/app.py @@ -13,7 +13,7 @@ from pydantic import BaseModel import Autocomplete -from helpers import get_blacklisted_user_ids, get_global_collection, get_whitelisted_guild_ids, is_db_online, pad_id, get_guild_collection +from helpers import get_denylisted_user_ids, get_global_collection, get_allowlisted_guild_ids, is_db_online, pad_id, get_guild_collection # fix PIPE encoding error on Windows, auto flush print sys.stdout.reconfigure(encoding='utf-8') @@ -59,20 +59,20 @@ async def api_status(): async def get_guilds(): """ Returns a list of guilds - If whitelist is enabled (by not being an empty list), only whitelisted guilds will be returned. + If allowlist is enabled (by not being an empty list), only allowlisted guilds will be returned. - all other whitelist logic is handled by get_guild_collection() method - it won't return a collection for non-whitelisted guilds + all other allowlist logic is handled by get_guild_collection() method - it won't return a collection for non-allowlisted guilds """ collection_guilds = get_global_collection("guilds") - whitelisted_guild_ids = get_whitelisted_guild_ids() + allowlisted_guild_ids = get_allowlisted_guild_ids() - if len(whitelisted_guild_ids) == 0: + if len(allowlisted_guild_ids) == 0: cursor = collection_guilds.find().sort([("msg_count", pymongo.DESCENDING)]) else: cursor = collection_guilds.find( { "_id": { - "$in": whitelisted_guild_ids + "$in": allowlisted_guild_ids } } ).sort([("msg_count", pymongo.DESCENDING)]) @@ -143,15 +143,15 @@ async def get_message_ids(channel_id: str, guild_id: str): cache_dir = "../../release/dcef/storage/cache/message-ids" cache_path = f"{cache_dir}/{channel_id}.json" - blacklisted_user_ids_path = f"{cache_dir}/blacklisted_user_ids.json" + denylisted_user_ids_path = f"{cache_dir}/denylisted_user_ids.json" - # clear entire cache if blacklisted user ids changed - blacklisted_user_ids = get_blacklisted_user_ids() + # clear entire cache if denylisted user ids changed + denylisted_user_ids = get_denylisted_user_ids() if os.path.exists(cache_path): - with open(blacklisted_user_ids_path, "r", encoding="utf-8") as f: + with open(denylisted_user_ids_path, "r", encoding="utf-8") as f: file_content = f.read() - if file_content != str(blacklisted_user_ids): + if file_content != str(denylisted_user_ids): shutil.rmtree(cache_dir) os.makedirs(cache_dir) @@ -167,7 +167,7 @@ async def get_message_ids(channel_id: str, guild_id: str): query = { "channelId": channel_id, "author._id": { - "$nin": blacklisted_user_ids + "$nin": denylisted_user_ids } } ids = collection_messages.find(query, {"_id": 1}).sort([("_id", pymongo.ASCENDING)]) @@ -178,8 +178,8 @@ async def get_message_ids(channel_id: str, guild_id: str): with open(cache_path, "w", encoding="utf-8") as f: f.write(file_content) - file_content = re.sub(r"'", '"', str(blacklisted_user_ids)) - with open(blacklisted_user_ids_path, "w", encoding="utf-8") as f: + file_content = re.sub(r"'", '"', str(denylisted_user_ids)) + with open(denylisted_user_ids_path, "w", encoding="utf-8") as f: f.write(file_content) return new_ids @@ -200,7 +200,7 @@ async def get_multiple_message_content(message_req_obj: MessageRequest): guild_id = message_req_obj.guild_id collection_messages = get_guild_collection(guild_id, "messages") - blacklisted_user_ids = get_blacklisted_user_ids() + denylisted_user_ids = get_denylisted_user_ids() messages = collection_messages.find( { @@ -208,7 +208,7 @@ async def get_multiple_message_content(message_req_obj: MessageRequest): "$in": message_ids }, "author._id": { - "$nin": blacklisted_user_ids + "$nin": denylisted_user_ids } } ) @@ -861,20 +861,20 @@ async def search_messages(guild_id: str, prompt: str = None, only_ids: bool = Tr message_contains = [word.lower() for word in message_contains] message_ids = [pad_id(id) for id in message_ids] - blacklisted_user_ids = get_blacklisted_user_ids() + denylisted_user_ids = get_denylisted_user_ids() from_user_ids = [pad_id(id) for id in from_user_ids] from_user_ids = extend_users(from_user_ids, from_users, guild_id) - from_user_ids = [user_id for user_id in from_user_ids if user_id not in blacklisted_user_ids] # remove blacklisted users + from_user_ids = [user_id for user_id in from_user_ids if user_id not in denylisted_user_ids] # remove denylisted users print("from_user_ids", from_user_ids) reaction_from_ids = [pad_id(id) for id in reaction_from_ids] reaction_from_ids = extend_users(reaction_from_ids, reaction_from, guild_id) - reaction_from_ids = [user_id for user_id in reaction_from_ids if user_id not in blacklisted_user_ids] # remove blacklisted users + reaction_from_ids = [user_id for user_id in reaction_from_ids if user_id not in denylisted_user_ids] # remove denylisted users mentions_user_ids = [pad_id(id) for id in mentions_user_ids] mentions_user_ids = extend_users(mentions_user_ids, mentions_users, guild_id) - mentions_user_ids = [user_id for user_id in mentions_user_ids if user_id not in blacklisted_user_ids] # remove blacklisted users + mentions_user_ids = [user_id for user_id in mentions_user_ids if user_id not in denylisted_user_ids] # remove denylisted users reaction_ids = [pad_id(id) for id in reaction_ids] reactions = [reaction.lower() for reaction in reactions] @@ -908,12 +908,12 @@ async def search_messages(guild_id: str, prompt: str = None, only_ids: bool = Tr if len(message_ids) > 0: query["_id"] = {"$in": message_ids} - if len(from_user_ids) > 0 or len(blacklisted_user_ids) > 0: + if len(from_user_ids) > 0 or len(denylisted_user_ids) > 0: query["author._id"] = {} if len(from_user_ids) > 0: query["author._id"]["$in"] = from_user_ids - if len(blacklisted_user_ids) > 0: - query["author._id"]["$nin"] = blacklisted_user_ids + if len(denylisted_user_ids) > 0: + query["author._id"]["$nin"] = denylisted_user_ids if len(reaction_from_ids) > 0: query["reactions.users._id"] = {"$in": reaction_from_ids} diff --git a/backend/fastapi/helpers.py b/backend/fastapi/helpers.py index e12cbaf0..dc433931 100644 --- a/backend/fastapi/helpers.py +++ b/backend/fastapi/helpers.py @@ -13,22 +13,22 @@ def pad_id(id): collection_guilds = db["guilds"] collection_config = db["config"] -def get_whitelisted_guild_ids(): - whitelisted_guild_ids = collection_config.find_one({"key": "whitelisted_guild_ids"})["value"] - whitelisted_guild_ids = [pad_id(id) for id in whitelisted_guild_ids] - return whitelisted_guild_ids +def get_allowlisted_guild_ids(): + allowlisted_guild_ids = collection_config.find_one({"key": "allowlisted_guild_ids"})["value"] + allowlisted_guild_ids = [pad_id(id) for id in allowlisted_guild_ids] + return allowlisted_guild_ids -def get_blacklisted_user_ids(): - blacklisted_user_ids = collection_config.find_one({"key": "blacklisted_user_ids"})["value"] - blacklisted_user_ids = [pad_id(id) for id in blacklisted_user_ids] - return blacklisted_user_ids +def get_denylisted_user_ids(): + denylisted_user_ids = collection_config.find_one({"key": "denylisted_user_ids"})["value"] + denylisted_user_ids = [pad_id(id) for id in denylisted_user_ids] + return denylisted_user_ids def get_guild_collection(guild_id, collection_name): - whitelisted_guild_ids = get_whitelisted_guild_ids() + allowlisted_guild_ids = get_allowlisted_guild_ids() padded_guild_id = pad_id(guild_id) - if len(whitelisted_guild_ids) > 0 and padded_guild_id not in whitelisted_guild_ids: - raise Exception(f"Guild {guild_id} not whitelisted") + if len(allowlisted_guild_ids) > 0 and padded_guild_id not in allowlisted_guild_ids: + raise Exception(f"Guild {guild_id} not allowlisted") return db[f"g{padded_guild_id}_{collection_name}"] diff --git a/backend/preprocess/main_mongo.py b/backend/preprocess/main_mongo.py index 6d71a95b..3d1a8fc6 100644 --- a/backend/preprocess/main_mongo.py +++ b/backend/preprocess/main_mongo.py @@ -16,6 +16,17 @@ sys.stderr.reconfigure(encoding='utf-8') print = functools.partial(print, flush=True) +def rename_config_key(config, old_key: str, new_key: str): + """ + Renames config key in config collection + """ + old_config = config.find_one({"key": old_key}) + if old_config is None: + return + old_config["key"] = new_key + config.update_one({"key": old_key}, {"$set": old_config}) + print(f"Renamed config key {old_key} --> {new_key}") + def wipe_database(database: MongoDatabase): """ @@ -25,14 +36,18 @@ def wipe_database(database: MongoDatabase): EXPECTED_VERSION = 15 # <---- change this to wipe database config = database.get_collection("config") - # add empty whitelisted_guild_ids config if it does not exist - whitelisted_guild_ids = config.find_one({"key": "whitelisted_guild_ids"}) - if whitelisted_guild_ids is None: - config.insert_one({"key": "whitelisted_guild_ids", "value": []}) + # migrate config keys to new names + rename_config_key(config, "whitelisted_guild_ids", "allowlisted_guild_ids") + rename_config_key(config, "blacklisted_user_ids", "denylisted_user_ids") + + # add empty allowlisted_guild_ids config if it does not exist + allowlisted_guild_ids = config.find_one({"key": "allowlisted_guild_ids"}) + if allowlisted_guild_ids is None: + config.insert_one({"key": "allowlisted_guild_ids", "value": []}) - blacklisted_user_ids = config.find_one({"key": "blacklisted_user_ids"}) - if blacklisted_user_ids is None: - config.insert_one({"key": "blacklisted_user_ids", "value": []}) + denylisted_user_ids = config.find_one({"key": "denylisted_user_ids"}) + if denylisted_user_ids is None: + config.insert_one({"key": "denylisted_user_ids", "value": []}) version = config.find_one({"key": "version"}) if version is None: