From 3879fb0a5f10d738463ecc5ed3b3dfb005493f54 Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Tue, 3 Sep 2024 11:57:35 -0700 Subject: [PATCH 1/4] CBG-4216 Don't increment high sequence cached for unused sequences on pendingLogs (#7098) * CBG-4215 Move duplicate checks for pending sequences to read side Reduces complexity and avoids loop bug when duplicates are present when running _addPendingLogs. When popping an unused sequence range from pendingLogs, checks for overlap with the next entry in pending. When found, truncates range at next pending entry. Adds check in addPendingLogs for the case where the pending includes sequences earlier than nextSequence. Info logging emitted for any unexpected duplication. * CBG-3216 Don't increment high sequence cached for unused sequences on pending heap If unused sequences ranges are pushed to pendingLogs, they shouldn't increment the channel cache's highSequenceCached until they are processed. --- db/change_cache.go | 8 ++------ db/change_cache_test.go | 6 +++--- db/channel_cache.go | 6 +++++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/db/change_cache.go b/db/change_cache.go index 19995d5ae7..bbeecdcbee 100644 --- a/db/change_cache.go +++ b/db/change_cache.go @@ -576,7 +576,6 @@ func (c *changeCache) releaseUnusedSequence(ctx context.Context, sequence uint64 } else { changedChannels.Add(unusedSeq) } - c.channelCache.AddUnusedSequence(change) if c.notifyChange != nil && len(changedChannels) > 0 { c.notifyChange(ctx, changedChannels) } @@ -599,7 +598,6 @@ func (c *changeCache) releaseUnusedSequenceRange(ctx context.Context, fromSequen } changedChannels := c.processEntry(ctx, change) allChangedChannels = allChangedChannels.Update(changedChannels) - c.channelCache.AddUnusedSequence(change) if c.notifyChange != nil { c.notifyChange(ctx, allChangedChannels) } @@ -609,9 +607,6 @@ func (c *changeCache) releaseUnusedSequenceRange(ctx context.Context, fromSequen // push unused range to either pending or skipped lists based on current state of the change cache allChangedChannels = c.processUnusedRange(ctx, fromSequence, toSequence, allChangedChannels, timeReceived) - // update high seq cached - c.channelCache.AddUnusedSequence(&LogEntry{Sequence: toSequence}) - if c.notifyChange != nil { c.notifyChange(ctx, allChangedChannels) } @@ -804,8 +799,9 @@ func (c *changeCache) _addToCache(ctx context.Context, change *LogEntry) []chann } delete(c.receivedSeqs, change.Sequence) - // If unused sequence or principal, we're done after updating sequence + // If unused sequence, notify the cache and return if change.DocID == "" { + c.channelCache.AddUnusedSequence(change) return nil } diff --git a/db/change_cache_test.go b/db/change_cache_test.go index 2e500eba5f..0a6b245c2d 100644 --- a/db/change_cache_test.go +++ b/db/change_cache_test.go @@ -2350,7 +2350,7 @@ func TestReleasedSequenceRangeHandlingEverythingPending(t *testing.T) { assert.Equal(c, int64(1), dbContext.DbStats.CacheStats.PendingSeqLen.Value()) assert.Equal(c, uint64(2), testChangeCache.nextSequence) dbContext.UpdateCalculatedStats(ctx) - assert.Equal(c, int64(25), dbContext.DbStats.CacheStats.HighSeqCached.Value()) + assert.Equal(c, int64(1), dbContext.DbStats.CacheStats.HighSeqCached.Value()) }, time.Second*10, time.Millisecond*100) } @@ -2456,7 +2456,7 @@ func TestReleasedSequenceRangeHandlingEverythingPendingLowPendingCapacity(t *tes defer testChangeCache.Stop(ctx) require.NoError(t, err) - // process unused sequence range + // process unused sequence range, will be sent to pending. Triggers seq 1 being sent to skipped testChangeCache.releaseUnusedSequenceRange(ctx, 2, 25, time.Now()) require.EventuallyWithT(t, func(c *assert.CollectT) { @@ -2567,7 +2567,7 @@ func TestReleasedSequenceRangeHandlingSingleSequence(t *testing.T) { assert.Equal(c, int64(1), dbContext.DbStats.CacheStats.PendingSeqLen.Value()) assert.Equal(c, uint64(1), testChangeCache.nextSequence) dbContext.UpdateCalculatedStats(ctx) - assert.Equal(c, int64(2), dbContext.DbStats.CacheStats.HighSeqCached.Value()) + assert.Equal(c, int64(0), dbContext.DbStats.CacheStats.HighSeqCached.Value()) }, time.Second*10, time.Millisecond*100) // process change that should overload pending and push sequence 1 to skipped diff --git a/db/channel_cache.go b/db/channel_cache.go index d4ab42371d..716a996a92 100644 --- a/db/channel_cache.go +++ b/db/channel_cache.go @@ -188,7 +188,11 @@ func (c *channelCacheImpl) AddPrincipal(change *LogEntry) { // Add unused Sequence notifies the cache of an unused sequence update. Updates the cache's high sequence func (c *channelCacheImpl) AddUnusedSequence(change *LogEntry) { - c.updateHighCacheSequence(change.Sequence) + if change.EndSequence > 0 { + c.updateHighCacheSequence(change.EndSequence) + } else { + c.updateHighCacheSequence(change.Sequence) + } } // Adds an entry to the appropriate channels' caches, returning the affected channels. lateSequence From 6d848a9eabb21f685dc7f3855796e6b0122cd2fc Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Tue, 3 Sep 2024 14:55:47 -0700 Subject: [PATCH 2/4] CBG-4227 Set up 3.1.11 builds (#7104) --- manifest/3.1.xml | 4 ++-- manifest/3.1/3.1.10.1.xml | 30 ++++++++++++++++++++++++++++++ manifest/product-config.json | 14 ++++++++++++-- 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 manifest/3.1/3.1.10.1.xml diff --git a/manifest/3.1.xml b/manifest/3.1.xml index fedaa0f327..21559591f3 100644 --- a/manifest/3.1.xml +++ b/manifest/3.1.xml @@ -18,13 +18,13 @@ licenses/APL2.txt. - + - + diff --git a/manifest/3.1/3.1.10.1.xml b/manifest/3.1/3.1.10.1.xml new file mode 100644 index 0000000000..dba19c2c93 --- /dev/null +++ b/manifest/3.1/3.1.10.1.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/manifest/product-config.json b/manifest/product-config.json index 3dbe784773..97082f6095 100644 --- a/manifest/product-config.json +++ b/manifest/product-config.json @@ -450,8 +450,8 @@ "start_build": 1 }, "manifest/3.1.xml": { - "release": "3.1.10.1", - "release_name": "Couchbase Sync Gateway 3.1.10.1", + "release": "3.1.11", + "release_name": "Couchbase Sync Gateway 3.1.11", "production": true, "interval": 120, "go_version": "1.22.5", @@ -478,6 +478,16 @@ "trigger_blackduck": true, "start_build": 5 }, + "manifest/3.1/3.1.10.1.xml": { + "do-build": false, + "release": "3.1.10.1", + "release_name": "Couchbase Sync Gateway 3.1.10.1", + "production": true, + "interval": 1440, + "go_version": "1.22.5", + "trigger_blackduck": true, + "start_build": 3 + }, "manifest/3.1/3.1.8.xml": { "do-build": false, "release": "3.1.8", From 51c00da4c789fed54e31ad9d512659420dee73f1 Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Wed, 4 Sep 2024 10:20:40 -0400 Subject: [PATCH 3/4] CBG-4223 correct log message (#7101) --- base/redactable_string.go | 17 +++++++++++++++++ rest/config_manager.go | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 base/redactable_string.go diff --git a/base/redactable_string.go b/base/redactable_string.go new file mode 100644 index 0000000000..1a9bc58542 --- /dev/null +++ b/base/redactable_string.go @@ -0,0 +1,17 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package base + +import "fmt" + +// RedactSprintf is a wrapper around fmt.Sprintf that redacts any sensitive data. +func RedactSprintf(format string, args ...any) string { + redactedArgs := redact(args) + return fmt.Sprintf(format, redactedArgs...) +} diff --git a/rest/config_manager.go b/rest/config_manager.go index dbb8be15d4..b50bff05b4 100644 --- a/rest/config_manager.go +++ b/rest/config_manager.go @@ -519,9 +519,10 @@ func (b *bootstrapContext) waitForConfigDelete(ctx context.Context, bucketName, timeout = b.configRetryTimeout } + dbConfigName := PersistentConfigKey(ctx, groupID, dbName) retryWorker := func() (shouldRetry bool, err error, value interface{}) { config := &DatabaseConfig{} - cas, getErr := b.Connection.GetMetadataDocument(ctx, bucketName, PersistentConfigKey(ctx, groupID, dbName), config) + cas, getErr := b.Connection.GetMetadataDocument(ctx, bucketName, dbConfigName, config) // Success case - delete has been completed if base.IsDocNotFoundError(getErr) { return false, nil, nil @@ -541,7 +542,7 @@ func (b *bootstrapContext) waitForConfigDelete(ctx context.Context, bucketName, // Kick off the retry loop err, retryResult := base.RetryLoop( ctx, - "Wait for config version match", + base.RedactSprintf("Wait for %q to be deleted in %q", base.MD(dbConfigName), base.MD(bucketName)), retryWorker, base.CreateDoublingSleeperDurationFunc(50, timeout), ) From 26926af475a387384bac2f0465682df3c011b883 Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Thu, 5 Sep 2024 09:20:25 -0400 Subject: [PATCH 4/4] CBG-4228 correct docs for /db/_config/audit (#7108) --- docs/api/components/schemas.yaml | 135 +++++++++++---------- docs/api/paths/admin/db-_config-audit.yaml | 6 +- 2 files changed, 75 insertions(+), 66 deletions(-) diff --git a/docs/api/components/schemas.yaml b/docs/api/components/schemas.yaml index 6bdfb7e8ad..4b9466514c 100644 --- a/docs/api/components/schemas.yaml +++ b/docs/api/components/schemas.yaml @@ -1847,69 +1847,78 @@ Database: example: ["CRUD", "HTTP", "Query"] audit: description: Audit logging configuration. - type: object - properties: - enabled: - description: Whether audit logging is enabled. - type: boolean - default: false - enabled_events: - description: List of enabled audit events for this database. - type: array - items: - type: number - example: [1234, 5678] - disabled_users: - description: List of users for which audit logging is disabled - type: array - items: - type: object - properties: - domain: - description: The domain of the user for which audit logging is disabled. - type: string - name: - description: The name of the user for which audit logging is disabled. - type: string - disabled_roles: - description: List of roles for which audit logging is disabled - type: array - items: - type: object - properties: - domain: - description: The domain of the role for which audit logging is disabled. - type: string - name: - description: The name of the role for which audit logging is disabled. - type: string + allOf: + - type: object + properties: + enabled: + description: Whether audit logging is enabled. + type: boolean + default: false + enabled_events: + description: List of enabled audit events for this database. + type: array + items: + type: number + example: [1234, 5678] + - $ref: '#/Disabled-users-and-roles' title: Database-config -Database-audit: - title: Database-audit - description: A map of audit events and whether they are enabled or not. +Disabled-users-and-roles: type: object properties: - enabled: - type: boolean - events: - type: object - additionalProperties: - x-additionalPropertiesName: audit_id - description: The audit event ID and whether it is enabled or not. - type: boolean + disabled_users: + description: List of users for which audit logging is disabled. + type: array + items: + type: object + properties: + domain: + description: The domain of the user for which audit logging is disabled. Either cbs or sgw. + type: string + name: + description: The name of the user for which audit logging is disabled. + type: string + disabled_roles: + description: List of roles for which audit logging is disabled. Either cbs or sgw. + type: array + items: + type: object + properties: + domain: + description: The domain of the role for which audit logging is disabled. + type: string + name: + description: The name of the role for which audit logging is disabled. + type: string +Database-audit: + title: Simple + description: A map of audit events and whether they are enabled or not. + allOf: + - type: object + properties: + enabled: + type: boolean + events: + type: object + additionalProperties: + x-additionalPropertiesName: audit_id + description: The audit event ID and whether it is enabled or not. + type: boolean + - $ref: '#/Disabled-users-and-roles' Database-audit-verbose: - title: Database-audit-verbose + title: Verbose description: A map of detailed audit events. - type: object - properties: - enabled: - type: boolean - events: - type: object - additionalProperties: - x-additionalPropertiesName: audit_id - description: The audit event ID and whether it is enabled or not. - $ref: '#/AuditEventVerbose' + allOf: + - type: object + properties: + enabled: + type: boolean + events: + type: object + additionalProperties: + x-additionalPropertiesName: audit_id + description: The audit event ID and whether it is enabled or not. + $ref: '#/AuditEventVerbose' + - $ref: '#/Disabled-users-and-roles' AuditEventVerbose: title: audit-event-verbose description: Detailed information about an audit event. @@ -2390,7 +2399,7 @@ Logging-config: type: integer collation_buffer_size: description: The size of the log collation buffer. - default: 10 + default: 0 type: integer readOnly: true warn: @@ -2409,7 +2418,7 @@ Logging-config: type: integer collation_buffer_size: description: The size of the log collation buffer - default: 1000 + default: 0 type: integer readOnly: true info: @@ -2428,7 +2437,7 @@ Logging-config: type: integer collation_buffer_size: description: The size of the log collation buffer - default: 1000 + default: 0 type: integer readOnly: true debug: @@ -2447,7 +2456,7 @@ Logging-config: type: integer collation_buffer_size: description: The size of the log collation buffer - default: 0 + default: 1000 type: integer readOnly: true trace: @@ -2466,7 +2475,7 @@ Logging-config: type: integer collation_buffer_size: description: The size of the log collation buffer - default: 0 + default: 1000 type: integer readOnly: true stats: diff --git a/docs/api/paths/admin/db-_config-audit.yaml b/docs/api/paths/admin/db-_config-audit.yaml index a6b9dc28f2..986fa216e9 100644 --- a/docs/api/paths/admin/db-_config-audit.yaml +++ b/docs/api/paths/admin/db-_config-audit.yaml @@ -12,7 +12,7 @@ get: Required Sync Gateway RBAC roles: - * TODO + * Sync Gateway Architect parameters: - name: verbose in: query @@ -49,7 +49,7 @@ put: Required Sync Gateway RBAC roles: - * TODO + * Sync Gateway Architect requestBody: description: The new database audit configuration to use content: @@ -77,7 +77,7 @@ post: Required Sync Gateway RBAC roles: - * TODO + * Sync Gateway Architect parameters: - name: verbose in: query