improve web search

src/lib/server/websearch/runWebSearch.ts

@@ -20,7 +20,7 @@ import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators";
 import { MetricsServer } from "../metrics";
 import { logger } from "$lib/server/logger";
 
-const MAX_N_PAGES_TO_SCRAPE = 8 as const;
+const MAX_N_PAGES_TO_SCRAPE = 15 as const;
 const MAX_N_PAGES_TO_EMBED = 5 as const;
 
 export async function* runWebSearch(

src/lib/server/websearch/search/generateQuery.ts

@@ -3,7 +3,7 @@ import { format } from "date-fns";
 import type { EndpointMessage } from "../../endpoints/endpoints";
 import { generateFromDefaultEndpoint } from "../../generateFromDefaultEndpoint";
 
-export async function generateQuery(messages: Message[]) {
+export async function generateQuery(messages: Message[], generatedQueries?: string[]) {
 	const currentDate = format(new Date(), "MMMM d, yyyy");
 	const userMessages = messages.filter(({ from }) => from === "user");
 	const previousUserMessages = userMessages.slice(0, -1);
@@ -17,6 +17,9 @@ export async function generateQuery(messages: Message[]) {
 - Who is the president of France?
 
 Current Question: What about Mexico?
+
+Previously generated queries:
+- President list of Mexico
 `,
 		},
 		{
@@ -47,8 +50,26 @@ Current Question: Where is it being hosted?`,
 			from: "assistant",
 			content: `news ${format(new Date(Date.now() - 864e5), "MMMM d, yyyy")}`,
 		},
-		{ from: "user", content: "What is the current weather in Paris?" },
-		{ from: "assistant", content: `weather in Paris ${currentDate}` },
+		{
+			from: "user",
+			content: `Current Question: My dog has been bitten, what should the gums look like so that he is healthy and when does he need an infusion?
+
+Previously generated queries:
+- What healthy gums look like in dogs
+- What unhealthy gums look like in dogs
+`,
+		},
+		{ from: "assistant", content: `dog, gums, indications of necessary infusion` },
+		{
+			from: "user",
+			content: `Current Question: Who is Elon Musk ?
+
+Previously generated queries:
+- Elon Musk biography
+- Who is Elon Musk
+`,
+		},
+		{ from: "assistant", content: `Elon Musk` },
 		{
 			from: "user",
 			content:
@@ -58,13 +79,19 @@ Current Question: Where is it being hosted?`,
 							.join("\n")}`
 					: "") +
 				"\n\nCurrent Question: " +
-				lastMessage.content,
+				lastMessage.content +
+				"\n" +
+				(generatedQueries && generatedQueries.length > 0
+					? `Previously generated queries:\n${generatedQueries.map((q) => `- ${q}`).join("\n")}`
+					: ""),
 		},
 	];
 
+	const preprompt = `You are tasked with generating precise and effective web search queries to answer the user's question. Provide a concise and specific query for Google search that will yield the most relevant and up-to-date results. Include key terms and related phrases, and avoid unnecessary words. Answer with only the query. Avoid duplicates, make the prompts as divers as you can. You are not allowed to repeat queries. Today is ${currentDate}`;
+
 	const webQuery = await generateFromDefaultEndpoint({
 		messages: convQuery,
-		preprompt: `You are tasked with generating web search queries. Give me an appropriate query to answer my question for google search. Answer with only the query. Today is ${currentDate}`,
+		preprompt,
 		generateSettings: {
 			max_new_tokens: 30,
 		},

src/lib/server/websearch/search/search.ts

@@ -15,6 +15,7 @@ import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate";
 const listSchema = z.array(z.string()).default([]);
 const allowList = listSchema.parse(JSON5.parse(env.WEBSEARCH_ALLOWLIST));
 const blockList = listSchema.parse(JSON5.parse(env.WEBSEARCH_BLOCKLIST));
+const num_searches = 3;
 
 export async function* search(
 	messages: Message[],
@@ -25,6 +26,8 @@ export async function* search(
 	{ searchQuery: string; pages: WebSearchSource[] },
 	undefined
 > {
+	const searchQueries: string[] = [];
+
 	if (ragSettings && ragSettings?.allowedLinks.length > 0) {
 		yield makeGeneralUpdate({ message: "Using links specified in Assistant" });
 		return {
@@ -33,24 +36,48 @@ export async function* search(
 		};
 	}
 
-	const searchQuery = query ?? (await generateQuery(messages));
-	yield makeGeneralUpdate({ message: `Searching ${getWebSearchProvider()}`, args: [searchQuery] });
+	for (let i = 0; i < num_searches; i++) {
+		const searchQuery = query ?? (await generateQuery(messages, searchQueries));
+		searchQueries.push(searchQuery);
+		yield makeGeneralUpdate({
+			message: `Searching ${getWebSearchProvider()}`,
+			args: [searchQuery],
+		});
+	}
+
+	let combinedResults: WebSearchSource[] = [];
 
-	// handle the global and (optional) rag lists
-	if (ragSettings && ragSettings?.allowedDomains.length > 0) {
-		yield makeGeneralUpdate({ message: "Filtering on specified domains" });
+	for (const searchQuery of searchQueries) {
+		// handle the global and (optional) rag lists
+		if (ragSettings && ragSettings?.allowedDomains.length > 0) {
+			yield makeGeneralUpdate({ message: "Filtering on specified domains" });
+		}
+		const filters = buildQueryFromSiteFilters(
+			[...(ragSettings?.allowedDomains ?? []), ...allowList],
+			blockList
+		);
+
+		const searchQueryWithFilters = `${filters} ${searchQuery}`;
+		const searchResults = await searchWeb(searchQueryWithFilters).then(filterByBlockList);
+		combinedResults = [...combinedResults, ...searchResults];
 	}
-	const filters = buildQueryFromSiteFilters(
-		[...(ragSettings?.allowedDomains ?? []), ...allowList],
-		blockList
-	);
 
-	const searchQueryWithFilters = `${filters} ${searchQuery}`;
-	const searchResults = await searchWeb(searchQueryWithFilters).then(filterByBlockList);
+	// re-sort the results by relevance
+	// all results are appended to the end of the list
+	// so the most relevant results are at the beginning
+	// using num_searches iterating over the list to get the most relevant results
+	// example input: [a1,a2,a3,a4,a5,b1,b2,b3,b4,b5,c1,c2,c3,c4,c5]
+	// example output: [a1,b1,c1,a2,b2,c2,a3,b3,c3,a4,b4,c4,a5,b5,c5]
+	const sortedResults = [];
+	for (let i = 0; i < num_searches; i++) {
+		for (let j = i; j < combinedResults.length; j += num_searches) {
+			sortedResults.push(combinedResults[j]);
+		}
+	}
 
 	return {
-		searchQuery: searchQueryWithFilters,
-		pages: searchResults,
+		searchQuery: searchQueries.join(" | "),
+		pages: sortedResults,
 	};
 }