Noob question on parsing search results (how to rank?)

[jasonpolites]

Hi folks,

First time using the lib.. so far it's awesome, but I'm not sure how to deal with duplicates in the search result set. My use case is indexing email (after it's been parsed). Here is the sample/demo app I tried:

const { Document } = require("flexsearch");

const doc1 = {
  "id" : "tempId1",
  "from" : "[email protected]",
  "to" : "[email protected]",
  "subject" : "Hello fox",
  "body" : "The quick brown fox",
  "hasAttachments": true,
  "attachments" : [
    {
      "name" : "foo.pdf",
      "contentType" : "text/pdf",
      "path" : "/foo/bar/foo.pdf"
    },
    {
      "name" : "bar.pdf",
      "contentType" : "text/pdf",
      "path" : "/foo/bar/bar.pdf"      
    }
  ]
}

const doc2 = {
  "id" : "tempId2",
  "from" : "[email protected]",
  "to" : "[email protected]",
  "subject" : "Hello dog",
  "body" : "Jumps over the lazy dog",
  "hasAttachments": true,
  "attachments" : [
    {
      "name" : "foo.jpg",
      "contentType" : "image/jpg",
      "path" : "/foo/bar/foo.jpg"
    }
  ]
}

const index = new Document({
  document: {
      id: "id",
      index: [
          "from",
          "to",
          "subject",
          "body",
          "from",
          "hasAttachments",
          "attachments[]:name",
          "attachments[]:contentType",
          "attachments[]:path"
      ],
      store: [
        "from",
        "to",
        "subject",
        "hasAttachments"
      ]
  }
});

index.add(doc1);
index.add(doc2);

let result = index.search(`foo`,  { enrich: true});

console.log(JSON.stringify(result, null, 2));

Running this, the console prints:

[
  {
    "field": "attachments[]:name",
    "result": [
      {
        "id": "tempId",
        "doc": {
          "from": "[email protected]",
          "to": "[email protected]",
          "subject": "Hello fox",
          "hasAttachments": true
        }
      },
      {
        "id": "tempId2",
        "doc": {
          "from": "[email protected]",
          "to": "[email protected]",
          "subject": "Hello dog",
          "hasAttachments": true
        }
      }
    ]
  },
  {
    "field": "attachments[]:path",
    "result": [
      {
        "id": "tempId",
        "doc": {
          "from": "[email protected]",
          "to": "[email protected]",
          "subject": "Hello fox",
          "hasAttachments": true
        }
      },
      {
        "id": "tempId",
        "doc": {
          "from": "[email protected]",
          "to": "[email protected]",
          "subject": "Hello fox",
          "hasAttachments": true
        }
      },
      {
        "id": "tempId2",
        "doc": {
          "from": "[email protected]",
          "to": "[email protected]",
          "subject": "Hello dog",
          "hasAttachments": true
        }
      }
    ]
  }
]

This result looks "correct", but I'm not sure how to easily iterate the results without running into a bunch of duplicates. The goal here is to list matching docs, in this case there are 2, but the field-centric arrangements of the result schema means it doesn't seem easy to just naively iterate the result set to see matching documents.

Is the idea that, when displaying results, I should maintain a local hashmap of all documents in the result set? And if so, what is the recommendation for sort order? I feel like the challenge here is more one of sorting than just retrieval (?)

Updated

Still hoping for authoritative guidance, but in the meantime I implemented a hacky ranking function

/**
 * Collapses the search results into single rows, ranked by the number of 
 * fields that matched the search term
 */
function rank(resultSet) {
  let resultMap = new Map();
  resultSet.forEach(result => {
    let docs = result.result;
    docs.forEach(doc => {
      let currentVal = resultMap.get(doc.id);
      if(!currentVal) {
        currentVal = {
          score: 1,
          doc: doc.doc
        }
        resultMap.set(doc.id, currentVal);
      } else {
        // More fields matching means a higher score/rank
        currentVal.score++
      }
    });
  });

  // Sort the map based on the score property
  return new Map([...resultMap].sort((a, b) => {
    return b[1].score - a[1].score;
  }))
}

[RupertBarrow]

You could change the initial JSON data structure before indexing it, into a shape which will fit with how you want to use the results.
You could also imagine creating 2 indices instead of 1 :

• one for the email body
• one for the email attachments
• You can then search both indices in parallel, and mix and match results.

[jasonpolites]

Good ideas. I guess I was hoping for a rough approximation of how something like Lucene works. The schema of the document in my case is "correct" (i.e. one email can have many attachments), and I could munge all these together into one uber-attachment node/child but this feels like a problem that may arise in many contexts. tl,dr; IMHO matching multiple fields from a single doc should still only return that doc once in the resultset. (ideally with metadata associated with the index itself, like how many fields matched etc)

[stereobooster]

I'm also a bit confused. I want to search more than one filed, but I need only one search set. For example, UI looks like this

I want to search through all searchable fields at once and get one set of results sorted by rank (cumulative). I can write function which will combine results, but it feels clumsy.

const search = (index, query, options = {}) => {
  let { boostFactors, offset, limit, ...rest } = options;
  offset = offset || 0;
  limit = limit || 20;
  const results = index.search(query, {
    ...rest,
    offset: 0,
    limit: offset + limit,
  });
  if (results.length === 1) return results[0].result;

  boostFactors = boostFactors || {};
  const resultsByRelevance = new Map();
  results.forEach(({ field, result }) => {
    const factor = boostFactors[field] || 1;
    result.forEach((id, i) => {
      resultsByRelevance.set(
        id,
        (resultsByRelevance.get(id) || 0) + (i + 1) * factor
      );
    });
  });

  return [...resultsByRelevance.entries()]
    .sort((a, b) => a[1] - b[1])
    .slice(offset, offset + limit)
    .map((a) => a[0]);
};

console.log(search(index, "apple", { boostFactors: { description: 0.7 } }));
// or
console.log(search(index, "apple"));

Or do I miss something?

PS: other less correct options:

const searchNaive = (index, query, options = {}) => {
  let { limit, ...rest } = options;
  limit = limit || 20;
  const results = index.search(query, {
    ...rest,
    limit,
  });
  if (results.length === 1) return results[0].result;
  return [
    ...new Set(results.reduce((acc, val) => acc.concat(val.result), [])),
  ].slice(0, limit);
};

const searchNaive2 = (index, query, options = {}) => {
  let { limit, ...rest } = options;
  limit = limit || 20;
  const results = index.search(query, {
    ...rest,
    limit,
  });
  if (results.length === 1) return results[0].result;

  const combinedResult = new Set();
  for (let i = 0; i < limit; i++) {
    results.forEach(({ result }) => {
      if (result[i] !== undefined) combinedResult.add(result[i]);
    });
    if (combinedResult.size >= limit) break;
  }

  return [...combinedResult];
};