Skip to content

Commit

Permalink
refactor: optimization of syncTaggedLogsAsSender (#10811)
Browse files Browse the repository at this point in the history
  • Loading branch information
@benesjan
benesjan authored Dec 18, 2024
1 parent 921febb commit a4be4a7
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 76 deletions.
80 changes: 37 additions & 43 deletions yarn-project/pxe/src/simulator_oracle/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import { type IncomingNoteDao } from '../database/incoming_note_dao.js';
import { type PxeDatabase } from '../database/index.js';
import { produceNoteDaos } from '../note_decryption_utils/produce_note_daos.js';
import { getAcirSimulator } from '../simulator/index.js';
import { getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js';
import { WINDOW_HALF_SIZE, getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js';

/**
* A data oracle that provides information needed for simulating a transaction.
Expand Down Expand Up @@ -353,59 +353,56 @@ export class SimulatorOracle implements DBOracle {
recipient: AztecAddress,
): Promise<void> {
const appTaggingSecret = await this.#calculateAppTaggingSecret(contractAddress, sender, recipient);
let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]);

const INDEX_OFFSET = 10;

let previousEmptyBack = 0;
let currentEmptyBack = 0;
let currentEmptyFront: number;

// The below code is trying to find the index of the start of the first window in which for all elements of window, we do not see logs.
// We take our window size, and fetch the node for these logs. We store both the amount of empty consecutive slots from the front and the back.
// We use our current empty consecutive slots from the front, as well as the previous consecutive empty slots from the back to see if we ever hit a time where there
// is a window in which we see the combination of them to be greater than the window's size. If true, we rewind current index to the start of said window and use it.
// Assuming two windows of 5:
// [0, 1, 0, 1, 0], [0, 0, 0, 0, 0]
// We can see that when processing the second window, the previous amount of empty slots from the back of the window (1), added with the empty elements from the front of the window (5)
// is greater than 5 (6) and therefore we have found a window to use.
// We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (1) = 4;
// This is the first index of our desired window.
// Note that if we ever see a situation like so:
// [0, 1, 0, 1, 0], [0, 0, 0, 0, 1]
// This also returns the correct index (4), but this is indicative of a problem / desync. i.e. we should never have a window that has a log that exists after the window.
const [oldIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]);

// This algorithm works such that:
// 1. If we find minimum consecutive empty logs in a window of logs we set the index to the index of the last log
// we found and quit.
// 2. If we don't find minimum consecutive empty logs in a window of logs we slide the window to latest log index
// and repeat the process.
const MIN_CONSECUTIVE_EMPTY_LOGS = 10;
const WINDOW_SIZE = MIN_CONSECUTIVE_EMPTY_LOGS * 2;

let [numConsecutiveEmptyLogs, currentIndex] = [0, oldIndex];
do {
const currentTags = [...new Array(INDEX_OFFSET)].map((_, i) => {
// We compute the tags for the current window of indexes
const currentTags = [...new Array(WINDOW_SIZE)].map((_, i) => {
const indexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex + i);
return indexedAppTaggingSecret.computeSiloedTag(recipient, contractAddress);
});
previousEmptyBack = currentEmptyBack;

// We fetch the logs for the tags
const possibleLogs = await this.aztecNode.getLogsByTags(currentTags);

const indexOfFirstLog = possibleLogs.findIndex(possibleLog => possibleLog.length !== 0);
currentEmptyFront = indexOfFirstLog === -1 ? INDEX_OFFSET : indexOfFirstLog;

// We find the index of the last log in the window that is not empty
const indexOfLastLog = possibleLogs.findLastIndex(possibleLog => possibleLog.length !== 0);
currentEmptyBack = indexOfLastLog === -1 ? INDEX_OFFSET : INDEX_OFFSET - 1 - indexOfLastLog;

currentIndex += INDEX_OFFSET;
} while (currentEmptyFront + previousEmptyBack < INDEX_OFFSET);
if (indexOfLastLog === -1) {
// We haven't found any logs in the current window so we stop looking
break;
}

// We unwind the entire current window and the amount of consecutive empty slots from the previous window
const newIndex = currentIndex - (INDEX_OFFSET + previousEmptyBack);
// We move the current index to that of the last log we found
currentIndex += indexOfLastLog + 1;

await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, newIndex)]);
// We compute the number of consecutive empty logs we found and repeat the process if we haven't found enough.
numConsecutiveEmptyLogs = WINDOW_SIZE - indexOfLastLog - 1;
} while (numConsecutiveEmptyLogs < MIN_CONSECUTIVE_EMPTY_LOGS);

const contractName = await this.contractDataOracle.getDebugContractName(contractAddress);
this.log.debug(`Syncing logs for sender ${sender} at contract ${contractName}(${contractAddress})`, {
sender,
secret: appTaggingSecret,
index: currentIndex,
contractName,
contractAddress,
});
if (currentIndex !== oldIndex) {
await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, currentIndex)]);

this.log.debug(`Syncing logs for sender ${sender} at contract ${contractName}(${contractAddress})`, {
sender,
secret: appTaggingSecret,
index: currentIndex,
contractName,
contractAddress,
});
} else {
this.log.debug(`No new logs found for sender ${sender} at contract ${contractName}(${contractAddress})`);
}
}

/**
Expand All @@ -421,9 +418,6 @@ export class SimulatorOracle implements DBOracle {
maxBlockNumber: number,
scopes?: AztecAddress[],
): Promise<Map<string, TxScopedL2Log[]>> {
// Half the size of the window we slide over the tagging secret indexes.
const WINDOW_HALF_SIZE = 10;

// Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles.
// However it is impossible at the moment due to the language not supporting nested slices.
// This nesting is necessary because for a given set of tags we don't
Expand Down
69 changes: 36 additions & 33 deletions yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import { type PxeDatabase } from '../database/index.js';
import { KVPxeDatabase } from '../database/kv_pxe_database.js';
import { ContractDataOracle } from '../index.js';
import { SimulatorOracle } from './index.js';
import { WINDOW_HALF_SIZE } from './tagging_utils.js';

const TXS_PER_BLOCK = 4;
const NUM_NOTE_HASHES_PER_BLOCK = TXS_PER_BLOCK * MAX_NOTE_HASHES_PER_TX;
Expand Down Expand Up @@ -138,16 +139,15 @@ describe('Simulator oracle', () => {

describe('sync tagged logs', () => {
const NUM_SENDERS = 10;
const SENDER_OFFSET_WINDOW_SIZE = 10;
let senders: { completeAddress: CompleteAddress; ivsk: Fq; secretKey: Fr }[];

function generateMockLogs(senderOffset: number) {
function generateMockLogs(tagIndex: number) {
const logs: { [k: string]: TxScopedL2Log[] } = {};

// Add a random note from every address in the address book for our account with index senderOffset
// Add a random note from every address in the address book for our account with index tagIndex
// Compute the tag as sender (knowledge of preaddress and ivsk)
for (const sender of senders) {
const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, senderOffset);
const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, tagIndex);
const blockNumber = 1;
const randomNote = new MockNoteRequest(
getRandomNoteLogPayload(tag, contractAddress),
Expand All @@ -164,18 +164,18 @@ describe('Simulator oracle', () => {
// Add a random note from the first sender in the address book, repeating the tag
// Compute the tag as sender (knowledge of preaddress and ivsk)
const firstSender = senders[0];
const tag = computeSiloedTagForIndex(firstSender, recipient.address, contractAddress, senderOffset);
const tag = computeSiloedTagForIndex(firstSender, recipient.address, contractAddress, tagIndex);
const payload = getRandomNoteLogPayload(tag, contractAddress);
const logData = payload.generatePayload(GrumpkinScalar.random(), recipient.address).toBuffer();
const log = new TxScopedL2Log(TxHash.random(), 1, 0, false, logData);
logs[tag.toString()].push(log);
// Accumulated logs intended for recipient: NUM_SENDERS + 1

// Add a random note from half the address book for our account with index senderOffset + 1
// Add a random note from half the address book for our account with index tagIndex + 1
// Compute the tag as sender (knowledge of preaddress and ivsk)
for (let i = NUM_SENDERS / 2; i < NUM_SENDERS; i++) {
const sender = senders[i];
const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, senderOffset + 1);
const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, tagIndex + 1);
const blockNumber = 2;
const randomNote = new MockNoteRequest(
getRandomNoteLogPayload(tag, contractAddress),
Expand All @@ -189,13 +189,13 @@ describe('Simulator oracle', () => {
}
// Accumulated logs intended for recipient: NUM_SENDERS + 1 + NUM_SENDERS / 2

// Add a random note from every address in the address book for a random recipient with index senderOffset
// Add a random note from every address in the address book for a random recipient with index tagIndex
// Compute the tag as sender (knowledge of preaddress and ivsk)
for (const sender of senders) {
const keys = deriveKeys(Fr.random());
const partialAddress = Fr.random();
const randomRecipient = computeAddress(keys.publicKeys, partialAddress);
const tag = computeSiloedTagForIndex(sender, randomRecipient, contractAddress, senderOffset);
const tag = computeSiloedTagForIndex(sender, randomRecipient, contractAddress, tagIndex);
const blockNumber = 3;
const randomNote = new MockNoteRequest(
getRandomNoteLogPayload(tag, contractAddress),
Expand Down Expand Up @@ -232,8 +232,8 @@ describe('Simulator oracle', () => {
});

it('should sync tagged logs', async () => {
const senderOffset = 0;
generateMockLogs(senderOffset);
const tagIndex = 0;
generateMockLogs(tagIndex);
const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3);
// We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first
// one + half of the logs for the second index
Expand Down Expand Up @@ -266,8 +266,8 @@ describe('Simulator oracle', () => {
await keyStore.addAccount(sender.secretKey, sender.completeAddress.partialAddress);
}

let senderOffset = 0;
generateMockLogs(senderOffset);
let tagIndex = 0;
generateMockLogs(tagIndex);

// Recompute the secrets (as recipient) to ensure indexes are updated
const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address);
Expand All @@ -292,13 +292,14 @@ describe('Simulator oracle', () => {
let indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets);
expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]);

// Two windows are fetch for each sender
expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2);
// Only 1 window is obtained for each sender
expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS);
aztecNode.getLogsByTags.mockReset();

// We add more logs at the end of the window to make sure we only detect them and bump the indexes if it lies within our window
senderOffset = 10;
generateMockLogs(senderOffset);
// We add more logs to the second half of the window to test that a second iteration in `syncTaggedLogsAsSender`
// is handled correctly.
tagIndex = 11;
generateMockLogs(tagIndex);
for (let i = 0; i < senders.length; i++) {
await simulatorOracle.syncTaggedLogsAsSender(
contractAddress,
Expand All @@ -308,14 +309,14 @@ describe('Simulator oracle', () => {
}

indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets);
expect(indexesAsSenderAfterSync).toStrictEqual([11, 11, 11, 11, 11, 12, 12, 12, 12, 12]);
expect(indexesAsSenderAfterSync).toStrictEqual([12, 12, 12, 12, 12, 13, 13, 13, 13, 13]);

expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2);
});

it('should sync tagged logs with a sender index offset', async () => {
const senderOffset = 5;
generateMockLogs(senderOffset);
const tagIndex = 5;
generateMockLogs(tagIndex);
const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3);
// We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first one + half of the logs for the second index
expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2);
Expand All @@ -341,8 +342,8 @@ describe('Simulator oracle', () => {
});

it("should sync tagged logs for which indexes are not updated if they're inside the window", async () => {
const senderOffset = 1;
generateMockLogs(senderOffset);
const tagIndex = 1;
generateMockLogs(tagIndex);

// Recompute the secrets (as recipient) to update indexes
const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address);
Expand All @@ -361,8 +362,8 @@ describe('Simulator oracle', () => {
expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2);

// First sender should have 2 logs, but keep index 2 since they were built using the same tag
// Next 4 senders should also have index 2 = offset + 1
// Last 5 senders should have index 3 = offset + 2
// Next 4 senders should also have index 2 = tagIndex + 1
// Last 5 senders should have index 3 = tagIndex + 2
const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets);

expect(indexes).toHaveLength(NUM_SENDERS);
Expand All @@ -374,8 +375,8 @@ describe('Simulator oracle', () => {
});

it("should not sync tagged logs for which indexes are not updated if they're outside the window", async () => {
const senderOffset = 0;
generateMockLogs(senderOffset);
const tagIndex = 0;
generateMockLogs(tagIndex);

// Recompute the secrets (as recipient) to update indexes
const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address);
Expand All @@ -384,8 +385,10 @@ describe('Simulator oracle', () => {
return poseidon2Hash([firstSenderSecretPoint.x, firstSenderSecretPoint.y, contractAddress]);
});

// We set the indexes to WINDOW_HALF_SIZE + 1 so that it's outside the window and for this reason no updates
// should be triggered.
await database.setTaggingSecretsIndexesAsRecipient(
secrets.map(secret => new IndexedTaggingSecret(secret, SENDER_OFFSET_WINDOW_SIZE + 1)),
secrets.map(secret => new IndexedTaggingSecret(secret, WINDOW_HALF_SIZE + 1)),
);

const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3);
Expand All @@ -404,8 +407,8 @@ describe('Simulator oracle', () => {
});

it('should sync tagged logs from scratch after a DB wipe', async () => {
const senderOffset = 0;
generateMockLogs(senderOffset);
const tagIndex = 0;
generateMockLogs(tagIndex);

// Recompute the secrets (as recipient) to update indexes
const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address);
Expand All @@ -415,7 +418,7 @@ describe('Simulator oracle', () => {
});

await database.setTaggingSecretsIndexesAsRecipient(
secrets.map(secret => new IndexedTaggingSecret(secret, SENDER_OFFSET_WINDOW_SIZE + 2)),
secrets.map(secret => new IndexedTaggingSecret(secret, WINDOW_HALF_SIZE + 2)),
);

let syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3);
Expand Down Expand Up @@ -447,8 +450,8 @@ describe('Simulator oracle', () => {
});

it('should not sync tagged logs with a blockNumber > maxBlockNumber', async () => {
const senderOffset = 0;
generateMockLogs(senderOffset);
const tagIndex = 0;
generateMockLogs(tagIndex);
const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 1);

// Only NUM_SENDERS + 1 logs should be synched, since the rest have blockNumber > 1
Expand Down
3 changes: 3 additions & 0 deletions yarn-project/pxe/src/simulator_oracle/tagging_utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { type Fr, IndexedTaggingSecret } from '@aztec/circuits.js';

// Half the size of the window we slide over the tagging secret indexes.
export const WINDOW_HALF_SIZE = 10;

export function getIndexedTaggingSecretsForTheWindow(
secretsAndWindows: { appTaggingSecret: Fr; leftMostIndex: number; rightMostIndex: number }[],
): IndexedTaggingSecret[] {
Expand Down

0 comments on commit a4be4a7

Please sign in to comment.