From fa80d9913969a7da6869608ad07841eb401d54dc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 15 Aug 2024 16:14:00 +0100
Subject: [PATCH 001/455] Add initial new UI for views

---
 .../backend/DataTable/TableDataTable.svelte   | 110 +++++-----
 .../backend/DataTable/ViewV2DataTable.svelte  |  49 ++---
 .../DeleteConfirmationModal.svelte            |   2 +-
 .../TableNavItem/TableNavItem.svelte          |  16 +-
 .../TableNavigator/TableNavigator.svelte      |  15 --
 .../ViewNavItem/ViewNavItem.svelte            |  71 -------
 .../table/[tableId]/[viewId]/_layout.svelte   |  24 +++
 .../table/[tableId]/[viewId]/index.svelte     |   5 +
 .../_components/DeleteViewModal.svelte}       |   0
 .../_components}/EditViewModal.svelte         |   0
 .../[tableId]/_components/ViewNavBar.svelte   | 200 ++++++++++++++++++
 .../data/table/[tableId]/_layout.svelte       |  16 +-
 .../data/table/[tableId]/view/index.svelte    |  19 ++
 .../view/v1/[viewName]/_layout.svelte         |  24 +++
 .../[tableId]/view/v1/[viewName]/index.svelte |  18 ++
 .../data/table/[tableId]/view/v1/index.svelte |   5 +
 16 files changed, 386 insertions(+), 188 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
 rename packages/builder/src/{components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte => pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte} (100%)
 rename packages/builder/src/{components/backend/TableNavigator/ViewNavItem => pages/builder/app/[application]/data/table/[tableId]/_components}/EditViewModal.svelte (100%)
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 525421f996e..22a7fabf244 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -57,64 +57,52 @@
   }
 </script>
 
-<div class="wrapper">
-  <Grid
-    {API}
-    {darkMode}
-    datasource={gridDatasource}
-    canAddRows={!isUsersTable}
-    canDeleteRows={!isUsersTable}
-    canEditRows={!isUsersTable || !$appStore.features.disableUserMetadata}
-    canEditColumns={!isUsersTable || !$appStore.features.disableUserMetadata}
-    schemaOverrides={isUsersTable ? userSchemaOverrides : null}
-    showAvatars={false}
-    on:updatedatasource={handleGridTableUpdate}
-    isCloud={$admin.cloud}
-  >
-    <svelte:fragment slot="filter">
-      {#if isUsersTable && $appStore.features.disableUserMetadata}
-        <GridUsersTableButton />
-      {/if}
-      <GridFilterButton />
-    </svelte:fragment>
-    <svelte:fragment slot="controls">
-      {#if !isUsersTable}
-        <GridCreateViewButton />
-      {/if}
-      <GridManageAccessButton />
-      {#if !isUsersTable}
-        <GridCreateAutomationButton />
-      {/if}
-      {#if relationshipsEnabled}
-        <GridRelationshipButton />
-      {/if}
-      {#if isUsersTable}
-        <EditRolesButton />
-      {:else}
-        <GridImportButton />
-      {/if}
-      <GridExportButton />
-      {#if isUsersTable}
-        <GridEditUserModal />
-      {:else}
-        <GridCreateEditRowModal />
-      {/if}
-    </svelte:fragment>
-    <svelte:fragment slot="edit-column">
-      <GridEditColumnModal />
-    </svelte:fragment>
-    <svelte:fragment slot="add-column">
-      <GridAddColumnModal />
-    </svelte:fragment>
-  </Grid>
-</div>
-
-<style>
-  .wrapper {
-    flex: 1 1 auto;
-    margin: -28px -40px -40px -40px;
-    display: flex;
-    flex-direction: column;
-    background: var(--background);
-  }
-</style>
+<Grid
+  {API}
+  {darkMode}
+  datasource={gridDatasource}
+  canAddRows={!isUsersTable}
+  canDeleteRows={!isUsersTable}
+  canEditRows={!isUsersTable || !$appStore.features.disableUserMetadata}
+  canEditColumns={!isUsersTable || !$appStore.features.disableUserMetadata}
+  schemaOverrides={isUsersTable ? userSchemaOverrides : null}
+  showAvatars={false}
+  on:updatedatasource={handleGridTableUpdate}
+  isCloud={$admin.cloud}
+>
+  <svelte:fragment slot="filter">
+    {#if isUsersTable && $appStore.features.disableUserMetadata}
+      <GridUsersTableButton />
+    {/if}
+    <GridFilterButton />
+  </svelte:fragment>
+  <svelte:fragment slot="controls">
+    {#if !isUsersTable}
+      <GridCreateViewButton />
+    {/if}
+    <GridManageAccessButton />
+    {#if !isUsersTable}
+      <GridCreateAutomationButton />
+    {/if}
+    {#if relationshipsEnabled}
+      <GridRelationshipButton />
+    {/if}
+    {#if isUsersTable}
+      <EditRolesButton />
+    {:else}
+      <GridImportButton />
+    {/if}
+    <GridExportButton />
+    {#if isUsersTable}
+      <GridEditUserModal />
+    {:else}
+      <GridCreateEditRowModal />
+    {/if}
+  </svelte:fragment>
+  <svelte:fragment slot="edit-column">
+    <GridEditColumnModal />
+  </svelte:fragment>
+  <svelte:fragment slot="add-column">
+    <GridAddColumnModal />
+  </svelte:fragment>
+</Grid>
diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index 646b764a2cd..5f7a0feb674 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -19,34 +19,21 @@
   }
 </script>
 
-<div class="wrapper">
-  <Grid
-    {API}
-    {datasource}
-    allowAddRows
-    allowDeleteRows
-    showAvatars={false}
-    on:updatedatasource={handleGridViewUpdate}
-    isCloud={$admin.cloud}
-    allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
-  >
-    <svelte:fragment slot="filter">
-      <GridFilterButton />
-    </svelte:fragment>
-    <svelte:fragment slot="controls">
-      <GridCreateEditRowModal />
-      <GridManageAccessButton />
-    </svelte:fragment>
-  </Grid>
-</div>
-
-<style>
-  .wrapper {
-    flex: 1 1 auto;
-    margin: -28px -40px -40px -40px;
-    display: flex;
-    flex-direction: column;
-    background: var(--background);
-    overflow: hidden;
-  }
-</style>
+<Grid
+  {API}
+  {datasource}
+  allowAddRows
+  allowDeleteRows
+  showAvatars={false}
+  on:updatedatasource={handleGridViewUpdate}
+  isCloud={$admin.cloud}
+  allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
+>
+  <svelte:fragment slot="filter">
+    <GridFilterButton />
+  </svelte:fragment>
+  <svelte:fragment slot="controls">
+    <GridCreateEditRowModal />
+    <GridManageAccessButton />
+  </svelte:fragment>
+</Grid>
diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte
index 03da9f3fd3e..c0e030845f6 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte
@@ -104,7 +104,7 @@
         </InlineAlert>
       </div>
     {/if}
-    <p class="fourthWarning">Please enter the app name below to confirm.</p>
+    <p class="fourthWarning">Please enter the table name below to confirm.</p>
     <Input bind:value={deleteTableName} placeholder={table.name} />
   </div>
 </ConfirmDialog>
diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte
index ab79a8fff09..6b64096e2e7 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte
@@ -20,14 +20,6 @@
 
   const getContextMenuItems = () => {
     return [
-      {
-        icon: "Delete",
-        name: "Delete",
-        keyBind: null,
-        visible: true,
-        disabled: false,
-        callback: deleteConfirmationModal.show,
-      },
       {
         icon: "Edit",
         name: "Edit",
@@ -36,6 +28,14 @@
         disabled: false,
         callback: editModal.show,
       },
+      {
+        icon: "Delete",
+        name: "Delete",
+        keyBind: null,
+        visible: true,
+        disabled: false,
+        callback: deleteConfirmationModal.show,
+      },
     ]
   }
 
diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
index f21230d7a6c..6ab500b65d4 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
@@ -1,7 +1,5 @@
 <script>
-  import { goto } from "@roxi/routify"
   import TableNavItem from "./TableNavItem/TableNavItem.svelte"
-  import ViewNavItem from "./ViewNavItem/ViewNavItem.svelte"
 
   export let tables
   export let selectTable
@@ -16,18 +14,5 @@
 <div class="hierarchy-items-container">
   {#each sortedTables as table, idx}
     <TableNavItem {table} {idx} on:click={() => selectTable(table._id)} />
-    {#each [...Object.entries(table.views || {})].sort() as [name, view], idx (idx)}
-      <ViewNavItem
-        {view}
-        {name}
-        on:click={() => {
-          if (view.version === 2) {
-            $goto(`./view/v2/${encodeURIComponent(view.id)}`)
-          } else {
-            $goto(`./view/v1/${encodeURIComponent(name)}`)
-          }
-        }}
-      />
-    {/each}
   {/each}
 </div>
diff --git a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte b/packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte
deleted file mode 100644
index ba30cea165a..00000000000
--- a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte
+++ /dev/null
@@ -1,71 +0,0 @@
-<script>
-  import {
-    contextMenuStore,
-    views,
-    viewsV2,
-    userSelectedResourceMap,
-  } from "stores/builder"
-  import NavItem from "components/common/NavItem.svelte"
-  import { isActive } from "@roxi/routify"
-  import { Icon } from "@budibase/bbui"
-  import EditViewModal from "./EditViewModal.svelte"
-  import DeleteConfirmationModal from "./DeleteConfirmationModal.svelte"
-
-  export let view
-  export let name
-
-  let editModal
-  let deleteConfirmationModal
-
-  const getContextMenuItems = () => {
-    return [
-      {
-        icon: "Delete",
-        name: "Delete",
-        keyBind: null,
-        visible: true,
-        disabled: false,
-        callback: deleteConfirmationModal.show,
-      },
-      {
-        icon: "Edit",
-        name: "Edit",
-        keyBind: null,
-        visible: true,
-        disabled: false,
-        callback: editModal.show,
-      },
-    ]
-  }
-
-  const openContextMenu = e => {
-    e.preventDefault()
-    e.stopPropagation()
-
-    const items = getContextMenuItems()
-    contextMenuStore.open(view.id, items, { x: e.clientX, y: e.clientY })
-  }
-
-  const isViewActive = (view, isActive, views, viewsV2) => {
-    return (
-      (isActive("./view/v1") && views.selected?.name === view.name) ||
-      (isActive("./view/v2") && viewsV2.selected?.id === view.id)
-    )
-  }
-</script>
-
-<NavItem
-  on:contextmenu={openContextMenu}
-  indentLevel={2}
-  icon="Remove"
-  text={name}
-  selected={isViewActive(view, $isActive, $views, $viewsV2)}
-  hovering={view.id === $contextMenuStore.id}
-  on:click
-  selectedBy={$userSelectedResourceMap[name] ||
-    $userSelectedResourceMap[view.id]}
->
-  <Icon on:click={openContextMenu} s hoverable name="MoreSmallList" />
-</NavItem>
-<EditViewModal {view} bind:this={editModal} />
-<DeleteConfirmationModal {view} bind:this={deleteConfirmationModal} />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte
new file mode 100644
index 00000000000..9c1baa723e2
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte
@@ -0,0 +1,24 @@
+<script>
+  import { viewsV2, builderStore } from "stores/builder"
+  import { syncURLToState } from "helpers/urlStateSync"
+  import * as routify from "@roxi/routify"
+  import { onDestroy } from "svelte"
+
+  $: id = $viewsV2.selectedViewId
+  $: builderStore.selectResource(id)
+
+  const stopSyncing = syncURLToState({
+    urlParam: "viewId",
+    stateKey: "selectedViewId",
+    validate: id => $viewsV2.list?.some(view => view.id === id),
+    update: viewsV2.select,
+    fallbackUrl: "../",
+    store: viewsV2,
+    routify,
+    decode: decodeURIComponent,
+  })
+
+  onDestroy(stopSyncing)
+</script>
+
+<slot />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
new file mode 100644
index 00000000000..c2281710bad
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -0,0 +1,5 @@
+<script>
+  import ViewV2DataTable from "components/backend/DataTable/ViewV2DataTable.svelte"
+</script>
+
+<ViewV2DataTable />
diff --git a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
similarity index 100%
rename from packages/builder/src/components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte
rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
diff --git a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/EditViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte
similarity index 100%
rename from packages/builder/src/components/backend/TableNavigator/ViewNavItem/EditViewModal.svelte
rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
new file mode 100644
index 00000000000..20ecdc3f830
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -0,0 +1,200 @@
+<script>
+  import {
+    tables,
+    datasources,
+    userSelectedResourceMap,
+    contextMenuStore,
+  } from "stores/builder"
+  import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
+  import { Icon } from "@budibase/bbui"
+  import { params, url } from "@roxi/routify"
+  import EditViewModal from "./EditViewModal.svelte"
+  import DeleteViewModal from "./DeleteViewModal.svelte"
+  import EditTableModal from "components/backend/TableNavigator/TableNavItem/EditModal.svelte"
+  import DeleteTableModal from "components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte"
+  import { UserAvatars } from "@budibase/frontend-core"
+  import { tick } from "svelte"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
+  import { TableNames } from "constants"
+
+  // Editing table
+  let editTableModal
+  let deleteTableModal
+
+  // Editing views
+  let editableView
+  let editViewModal
+  let deleteViewModal
+
+  $: tableId = $params.tableId
+  $: table = $tables.list.find(x => x._id === tableId)
+  $: datasource = $datasources.list.find(x => x._id === table?.sourceId)
+  $: tableSelectedBy = $userSelectedResourceMap[table?._id]
+  $: tableEditable = table?._id !== TableNames.USERS
+  $: activeId = $params.viewId ?? $params.tableId
+  $: views = Object.values(table?.views || {})
+    .filter(x => x.version === 2)
+    .slice()
+    .sort(alphabetical)
+
+  const openTableContextMenu = e => {
+    if (!tableEditable) {
+      return
+    }
+    e.preventDefault()
+    e.stopPropagation()
+    contextMenuStore.open(
+      table._id,
+      [
+        {
+          icon: "Edit",
+          name: "Edit",
+          keyBind: null,
+          visible: table?.sourceType !== DB_TYPE_EXTERNAL,
+          disabled: false,
+          callback: editTableModal?.show,
+        },
+        {
+          icon: "Delete",
+          name: "Delete",
+          keyBind: null,
+          visible: true,
+          disabled: false,
+          callback: deleteTableModal?.show,
+        },
+      ],
+      {
+        x: e.clientX,
+        y: e.clientY,
+      }
+    )
+  }
+
+  const openViewContextMenu = async (e, view) => {
+    e.preventDefault()
+    e.stopPropagation()
+    editableView = view
+    await tick()
+    contextMenuStore.open(
+      view.id,
+      [
+        {
+          icon: "Edit",
+          name: "Edit",
+          keyBind: null,
+          visible: true,
+          disabled: false,
+          callback: editViewModal?.show,
+        },
+        {
+          icon: "Delete",
+          name: "Delete",
+          keyBind: null,
+          visible: true,
+          disabled: false,
+          callback: deleteViewModal?.show,
+        },
+      ],
+      {
+        x: e.clientX,
+        y: e.clientY,
+      }
+    )
+  }
+
+  const alphabetical = (a, b) => {
+    return a.name < b.name ? -1 : 1
+  }
+</script>
+
+<div class="view-nav-bar">
+  <IntegrationIcon
+    integrationType={datasource.source}
+    schema={datasource.schema}
+    size="24"
+  />
+  <a
+    href={$url(`../${tableId}`)}
+    class="nav-bar-item"
+    class:active={tableId === activeId}
+    on:contextmenu={openTableContextMenu}
+  >
+    <div class="title">
+      {table.name}
+    </div>
+    {#if tableSelectedBy}
+      <UserAvatars size="XS" users={tableSelectedBy} />
+    {/if}
+    {#if tableEditable}
+      <Icon on:click={openTableContextMenu} s hoverable name="MoreSmallList" />
+    {/if}
+  </a>
+  {#each views as view (view.id)}
+    {@const selectedBy = $userSelectedResourceMap[view.id]}
+    <a
+      href={$url(`../${tableId}/${encodeURIComponent(view.id)}`)}
+      class="nav-bar-item"
+      class:active={view.id === activeId}
+      on:contextmenu={e => openViewContextMenu(e, view)}
+    >
+      <div class="title">
+        {view.name}
+      </div>
+      {#if selectedBy}
+        <UserAvatars size="XS" users={selectedBy} />
+      {/if}
+      <Icon
+        on:click={e => openViewContextMenu(e, view)}
+        s
+        hoverable
+        name="MoreSmallList"
+      />
+    </a>
+  {/each}
+</div>
+
+{#if table}
+  <EditTableModal {table} bind:this={editTableModal} />
+  <DeleteTableModal {table} bind:this={deleteTableModal} />
+{/if}
+
+{#if editableView}
+  <EditViewModal view={editableView} bind:this={editViewModal} />
+  <DeleteViewModal view={editableView} bind:this={deleteViewModal} />
+{/if}
+
+<style>
+  .view-nav-bar {
+    height: 50px;
+    border-bottom: var(--border-light);
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    padding: 0 var(--spacing-xl);
+    gap: var(--spacing-m);
+  }
+  .nav-bar-item {
+    padding: 6px 8px;
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-m);
+    transition: background 130ms ease-out, color 130ms ease-out;
+    color: var(--spectrum-global-color-gray-700);
+  }
+  .title {
+    font-size: 16px;
+  }
+  .nav-bar-item.active,
+  .nav-bar-item:hover {
+    color: var(--spectrum-global-color-gray-900);
+    background: var(--spectrum-global-color-gray-300);
+    cursor: pointer;
+  }
+  .nav-bar-item:not(.active) :global(.icon) {
+    display: none;
+  }
+</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte
index 8c60dbdd69a..4a33f8a9673 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte
@@ -3,6 +3,7 @@
   import { tables, builderStore } from "stores/builder"
   import * as routify from "@roxi/routify"
   import { onDestroy } from "svelte"
+  import ViewNavBar from "./_components/ViewNavBar.svelte"
 
   $: tableId = $tables.selectedTableId
   $: builderStore.selectResource(tableId)
@@ -20,4 +21,17 @@
   onDestroy(stopSyncing)
 </script>
 
-<slot />
+<div class="wrapper">
+  <ViewNavBar />
+  <slot />
+</div>
+
+<style>
+  .wrapper {
+    flex: 1 1 auto;
+    margin: -28px -40px -40px -40px;
+    display: flex;
+    flex-direction: column;
+    background: var(--background);
+  }
+</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte
new file mode 100644
index 00000000000..623cd224dbc
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte
@@ -0,0 +1,19 @@
+<script>
+  import { onMount } from "svelte"
+  import { views, viewsV2 } from "stores/builder"
+  import { redirect } from "@roxi/routify"
+
+  onMount(async () => {
+    if ($viewsV2.selected) {
+      $redirect(`./v2/${$viewsV2.selected.id}`)
+    } else if ($viewsV2.list?.length) {
+      $redirect(`./v2/${$viewsV2.list[0].id}`)
+    } else if ($views.selected) {
+      $redirect(`./${encodeURIComponent($views.selected?.name)}`)
+    } else if ($views.list?.length) {
+      $redirect(`./${encodeURIComponent($views.list[0].name)}`)
+    } else {
+      $redirect("../")
+    }
+  })
+</script>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte
new file mode 100644
index 00000000000..df3a18c2bb8
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte
@@ -0,0 +1,24 @@
+<script>
+  import { views, builderStore } from "stores/builder"
+  import { syncURLToState } from "helpers/urlStateSync"
+  import * as routify from "@roxi/routify"
+  import { onDestroy } from "svelte"
+
+  $: name = $views.selectedViewName
+  $: builderStore.selectResource(name)
+
+  const stopSyncing = syncURLToState({
+    urlParam: "viewName",
+    stateKey: "selectedViewName",
+    validate: name => $views.list?.some(view => view.name === name),
+    update: views.select,
+    fallbackUrl: "../../",
+    store: views,
+    routify,
+    decode: decodeURIComponent,
+  })
+
+  onDestroy(stopSyncing)
+</script>
+
+<slot />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte
new file mode 100644
index 00000000000..51149b602d2
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte
@@ -0,0 +1,18 @@
+<script>
+  import ViewDataTable from "components/backend/DataTable/ViewDataTable.svelte"
+  import { views } from "stores/builder"
+
+  $: selectedView = $views.selected
+</script>
+
+{#if selectedView}
+  <ViewDataTable view={selectedView} />
+{:else}<i>Create your first table to start building</i>{/if}
+
+<style>
+  i {
+    font-size: var(--font-size-m);
+    color: var(--grey-4);
+    margin-top: 2px;
+  }
+</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte
new file mode 100644
index 00000000000..c11ca870230
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte
@@ -0,0 +1,5 @@
+<script>
+  import { redirect } from "@roxi/routify"
+
+  $redirect("../")
+</script>

From d313968eaa69f07306db5bf207135d96a8c1c380 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 15 Aug 2024 16:33:39 +0100
Subject: [PATCH 002/455] Add view CRUD and table editing via new UI

---
 .../backend/DataTable/TableDataTable.svelte   |  4 --
 .../DatasourceNavigator.svelte                |  4 +-
 .../TableNavigator/TableNavigator.svelte      |  5 +-
 .../backend/TableNavigator/utils.js           |  4 ++
 .../_components/CreateViewModal.svelte}       | 54 ++++++++++---------
 .../[tableId]/_components/ViewNavBar.svelte   | 41 ++++++++++----
 6 files changed, 66 insertions(+), 46 deletions(-)
 rename packages/builder/src/{components/backend/DataTable/modals/grid/GridCreateViewModal.svelte => pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte} (51%)

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
index 22a7fabf244..2dd1eec0a8e 100644
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
@@ -9,7 +9,6 @@
   import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
-  import GridCreateViewButton from "components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte"
   import GridImportButton from "components/backend/DataTable/buttons/grid/GridImportButton.svelte"
   import GridExportButton from "components/backend/DataTable/buttons/grid/GridExportButton.svelte"
   import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
@@ -77,9 +76,6 @@
     <GridFilterButton />
   </svelte:fragment>
   <svelte:fragment slot="controls">
-    {#if !isUsersTable}
-      <GridCreateViewButton />
-    {/if}
     <GridManageAccessButton />
     {#if !isUsersTable}
       <GridCreateAutomationButton />
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
index e0745c15a15..a6078e38fbd 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
@@ -39,9 +39,7 @@
 
   const selectTable = tableId => {
     tables.select(tableId)
-    if (!$isActive("./table/:tableId")) {
-      $goto(`./table/${tableId}`)
-    }
+    $goto(`./table/${tableId}`)
   }
 
   function openNode(datasource) {
diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
index 6ab500b65d4..f97bd2487b9 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
@@ -1,14 +1,11 @@
 <script>
   import TableNavItem from "./TableNavItem/TableNavItem.svelte"
+  import { alphabetical } from "./utils"
 
   export let tables
   export let selectTable
 
   $: sortedTables = tables.sort(alphabetical)
-
-  const alphabetical = (a, b) => {
-    return a.name?.toLowerCase() > b.name?.toLowerCase() ? 1 : -1
-  }
 </script>
 
 <div class="hierarchy-items-container">
diff --git a/packages/builder/src/components/backend/TableNavigator/utils.js b/packages/builder/src/components/backend/TableNavigator/utils.js
index ae7aaa0f0a5..a76a14c1a4f 100644
--- a/packages/builder/src/components/backend/TableNavigator/utils.js
+++ b/packages/builder/src/components/backend/TableNavigator/utils.js
@@ -66,3 +66,7 @@ export const parseFile = e => {
     reader.readAsText(file)
   })
 }
+
+export const alphabetical = (a, b) => {
+  return a.name?.toLowerCase() > b.name?.toLowerCase() ? 1 : -1
+}
diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
similarity index 51%
rename from packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
index eea64a4d9e7..77477054b8d 100644
--- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
@@ -1,16 +1,21 @@
 <script>
-  import { getContext } from "svelte"
-  import { Input, notifications, ModalContent } from "@budibase/bbui"
+  import { Modal, Input, notifications, ModalContent } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { viewsV2 } from "stores/builder"
 
-  const { filter, sort, definition } = getContext("grid")
+  export let table
 
   let name
+  let modal
 
-  $: views = Object.keys($definition?.views || {}).map(x => x.toLowerCase())
+  $: views = Object.keys(table?.views || {}).map(x => x.toLowerCase())
   $: nameExists = views.includes(name?.trim().toLowerCase())
 
+  export const show = () => {
+    name = null
+    modal?.show()
+  }
+
   const enrichSchema = schema => {
     // We need to sure that "visible" is set to true for any fields which have
     // not yet been saved with grid metadata attached
@@ -28,33 +33,30 @@
     try {
       const newView = await viewsV2.create({
         name,
-        tableId: $definition._id,
-        query: $filter,
-        sort: {
-          field: $sort.column,
-          order: $sort.order,
-        },
-        schema: enrichSchema($definition.schema),
-        primaryDisplay: $definition.primaryDisplay,
+        tableId: table._id,
+        schema: enrichSchema(table.schema),
+        primaryDisplay: table.primaryDisplay,
       })
       notifications.success(`View ${name} created`)
-      $goto(`../../view/v2/${newView.id}`)
+      $goto(`./${newView.id}`)
     } catch (error) {
       notifications.error("Error creating view")
     }
   }
 </script>
 
-<ModalContent
-  title="Create view"
-  confirmText="Create view"
-  onConfirm={saveView}
-  disabled={nameExists}
->
-  <Input
-    label="View name"
-    thin
-    bind:value={name}
-    error={nameExists ? "A view already exists with that name" : null}
-  />
-</ModalContent>
+<Modal bind:this={modal}>
+  <ModalContent
+    title="Create view"
+    confirmText="Create view"
+    onConfirm={saveView}
+    disabled={nameExists}
+  >
+    <Input
+      label="View name"
+      thin
+      bind:value={name}
+      error={nameExists ? "A view already exists with that name" : null}
+    />
+  </ModalContent>
+</Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 20ecdc3f830..fb7d46f2595 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -6,7 +6,7 @@
     contextMenuStore,
   } from "stores/builder"
   import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
-  import { Icon } from "@budibase/bbui"
+  import { Icon, Button } from "@budibase/bbui"
   import { params, url } from "@roxi/routify"
   import EditViewModal from "./EditViewModal.svelte"
   import DeleteViewModal from "./DeleteViewModal.svelte"
@@ -16,8 +16,11 @@
   import { tick } from "svelte"
   import { DB_TYPE_EXTERNAL } from "constants/backend"
   import { TableNames } from "constants"
+  import { alphabetical } from "components/backend/TableNavigator/utils"
+  import CreateViewModal from "./CreateViewModal.svelte"
 
   // Editing table
+  let createViewModal
   let editTableModal
   let deleteTableModal
 
@@ -101,10 +104,6 @@
       }
     )
   }
-
-  const alphabetical = (a, b) => {
-    return a.name < b.name ? -1 : 1
-  }
 </script>
 
 <div class="view-nav-bar">
@@ -126,7 +125,13 @@
       <UserAvatars size="XS" users={tableSelectedBy} />
     {/if}
     {#if tableEditable}
-      <Icon on:click={openTableContextMenu} s hoverable name="MoreSmallList" />
+      <Icon
+        on:click={openTableContextMenu}
+        hoverable
+        name="MoreSmallList"
+        color="var(--spectrum-global-color-gray-600)"
+        hoverColor="var(--spectrum-global-color-gray-900)"
+      />
     {/if}
   </a>
   {#each views as view (view.id)}
@@ -145,15 +150,33 @@
       {/if}
       <Icon
         on:click={e => openViewContextMenu(e, view)}
-        s
         hoverable
         name="MoreSmallList"
+        color="var(--spectrum-global-color-gray-600)"
+        hoverColor="var(--spectrum-global-color-gray-900)"
       />
     </a>
   {/each}
+  {#if !views.length && tableEditable}
+    <Button cta on:click={createViewModal?.show}>Create a view</Button>
+    <span>
+      To create subsets of data, control access and more, create a view.
+    </span>
+  {/if}
+  {#if views.length}
+    <Icon
+      name="Add"
+      size="L"
+      hoverable
+      color="var(--spectrum-global-color-gray-600)"
+      hoverColor="var(--spectrum-global-color-gray-900)"
+      on:click={createViewModal?.show}
+    />
+  {/if}
 </div>
 
-{#if table}
+{#if table && tableEditable}
+  <CreateViewModal {table} bind:this={createViewModal} />
   <EditTableModal {table} bind:this={editTableModal} />
   <DeleteTableModal {table} bind:this={deleteTableModal} />
 {/if}
@@ -183,7 +206,7 @@
     align-items: center;
     gap: var(--spacing-m);
     transition: background 130ms ease-out, color 130ms ease-out;
-    color: var(--spectrum-global-color-gray-700);
+    color: var(--spectrum-global-color-gray-600);
   }
   .title {
     font-size: 16px;

From 40e7f58131b7af6795201e5a887c2a2a9bb809bd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 15 Aug 2024 19:49:30 +0100
Subject: [PATCH 003/455] Add automatic overflow menu popover for views that
 don't fit

---
 packages/bbui/src/Menu/Item.svelte            |   2 +-
 .../_components/CreateViewModal.svelte        |  10 +-
 .../[tableId]/_components/ViewNavBar.svelte   | 165 ++++++++++++++----
 3 files changed, 134 insertions(+), 43 deletions(-)

diff --git a/packages/bbui/src/Menu/Item.svelte b/packages/bbui/src/Menu/Item.svelte
index 05a33adda97..758b819a62c 100644
--- a/packages/bbui/src/Menu/Item.svelte
+++ b/packages/bbui/src/Menu/Item.svelte
@@ -35,7 +35,7 @@
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <li
-  on:click|preventDefault={disabled ? null : onClick}
+  on:click={disabled ? null : onClick}
   class="spectrum-Menu-item"
   class:is-disabled={disabled}
   role="menuitem"
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
index 77477054b8d..a37b5ccf828 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
@@ -9,7 +9,9 @@
   let modal
 
   $: views = Object.keys(table?.views || {}).map(x => x.toLowerCase())
-  $: nameExists = views.includes(name?.trim().toLowerCase())
+  $: trimmedName = name?.trim()
+  $: nameExists = views.includes(trimmedName?.toLowerCase())
+  $: nameValid = trimmedName?.length && !nameExists
 
   export const show = () => {
     name = null
@@ -29,15 +31,15 @@
   }
 
   const saveView = async () => {
-    name = name?.trim()
     try {
       const newView = await viewsV2.create({
-        name,
+        name: trimmedName,
         tableId: table._id,
         schema: enrichSchema(table.schema),
         primaryDisplay: table.primaryDisplay,
       })
       notifications.success(`View ${name} created`)
+      name = null
       $goto(`./${newView.id}`)
     } catch (error) {
       notifications.error("Error creating view")
@@ -50,7 +52,7 @@
     title="Create view"
     confirmText="Create view"
     onConfirm={saveView}
-    disabled={nameExists}
+    disabled={!nameValid}
   >
     <Input
       label="View name"
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index fb7d46f2595..7cc125ae67e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -6,7 +6,7 @@
     contextMenuStore,
   } from "stores/builder"
   import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
-  import { Icon, Button } from "@budibase/bbui"
+  import { Icon, Button, Popover, ActionMenu, MenuItem } from "@budibase/bbui"
   import { params, url } from "@roxi/routify"
   import EditViewModal from "./EditViewModal.svelte"
   import DeleteViewModal from "./DeleteViewModal.svelte"
@@ -18,6 +18,13 @@
   import { TableNames } from "constants"
   import { alphabetical } from "components/backend/TableNavigator/utils"
   import CreateViewModal from "./CreateViewModal.svelte"
+  import { onDestroy } from "svelte"
+
+  let viewContainer
+  let observer
+  let viewVisibiltyMap = {}
+  let overflowPopover
+  let anchor
 
   // Editing table
   let createViewModal
@@ -30,8 +37,8 @@
   let deleteViewModal
 
   $: tableId = $params.tableId
-  $: table = $tables.list.find(x => x._id === tableId)
-  $: datasource = $datasources.list.find(x => x._id === table?.sourceId)
+  $: table = $tables.list.find(table => table._id === tableId)
+  $: datasource = $datasources.list.find(ds => ds._id === table?.sourceId)
   $: tableSelectedBy = $userSelectedResourceMap[table?._id]
   $: tableEditable = table?._id !== TableNames.USERS
   $: activeId = $params.viewId ?? $params.tableId
@@ -39,6 +46,8 @@
     .filter(x => x.version === 2)
     .slice()
     .sort(alphabetical)
+  $: setUpObserver(viewContainer, views)
+  $: overflowedViews = views.filter(view => !viewVisibiltyMap[view.id])
 
   const openTableContextMenu = e => {
     if (!tableEditable) {
@@ -104,9 +113,39 @@
       }
     )
   }
+
+  const setUpObserver = (viewContainer, views) => {
+    if (!views.length || !viewContainer) {
+      observer?.disconnect()
+      return
+    }
+    observer = new IntersectionObserver(
+      entries => {
+        let updates = {}
+        for (let entry of entries) {
+          updates[entry.target.dataset.id] = entry.isIntersecting
+        }
+        viewVisibiltyMap = {
+          ...viewVisibiltyMap,
+          ...updates,
+        }
+      },
+      {
+        threshold: 1,
+        root: viewContainer,
+      }
+    )
+    for (let child of viewContainer.children) {
+      observer.observe(child)
+    }
+  }
+
+  onDestroy(() => {
+    observer?.disconnect()
+  })
 </script>
 
-<div class="view-nav-bar">
+<div class="nav">
   <IntegrationIcon
     integrationType={datasource.source}
     schema={datasource.schema}
@@ -114,11 +153,11 @@
   />
   <a
     href={$url(`../${tableId}`)}
-    class="nav-bar-item"
+    class="nav-item"
     class:active={tableId === activeId}
     on:contextmenu={openTableContextMenu}
   >
-    <div class="title">
+    <div class="nav-item__title">
       {table.name}
     </div>
     {#if tableSelectedBy}
@@ -134,35 +173,66 @@
       />
     {/if}
   </a>
-  {#each views as view (view.id)}
-    {@const selectedBy = $userSelectedResourceMap[view.id]}
-    <a
-      href={$url(`../${tableId}/${encodeURIComponent(view.id)}`)}
-      class="nav-bar-item"
-      class:active={view.id === activeId}
-      on:contextmenu={e => openViewContextMenu(e, view)}
-    >
-      <div class="title">
-        {view.name}
-      </div>
-      {#if selectedBy}
-        <UserAvatars size="XS" users={selectedBy} />
-      {/if}
-      <Icon
-        on:click={e => openViewContextMenu(e, view)}
-        hoverable
-        name="MoreSmallList"
-        color="var(--spectrum-global-color-gray-600)"
-        hoverColor="var(--spectrum-global-color-gray-900)"
-      />
-    </a>
-  {/each}
+  {#if views.length}
+    <div class="nav__views" bind:this={viewContainer}>
+      {#each views as view (view.id)}
+        {@const selectedBy = $userSelectedResourceMap[view.id]}
+        <a
+          href={$url(`../${tableId}/${encodeURIComponent(view.id)}`)}
+          class="nav-item"
+          class:active={view.id === activeId}
+          class:hidden={!viewVisibiltyMap[view.id]}
+          on:contextmenu={e => openViewContextMenu(e, view)}
+          data-id={view.id}
+        >
+          <div class="nav-item__title">
+            {view.name}
+          </div>
+          {#if selectedBy}
+            <UserAvatars size="XS" users={selectedBy} />
+          {/if}
+          <Icon
+            on:click={e => openViewContextMenu(e, view)}
+            hoverable
+            name="MoreSmallList"
+            color="var(--spectrum-global-color-gray-600)"
+            hoverColor="var(--spectrum-global-color-gray-900)"
+          />
+        </a>
+      {/each}
+    </div>
+  {/if}
   {#if !views.length && tableEditable}
     <Button cta on:click={createViewModal?.show}>Create a view</Button>
     <span>
       To create subsets of data, control access and more, create a view.
     </span>
   {/if}
+  {#if overflowedViews.length}
+    <ActionMenu align="right">
+      <div slot="control">
+        <Icon
+          name="ChevronDown"
+          size="XL"
+          hoverable
+          color="var(--spectrum-global-color-gray-600)"
+          hoverColor="var(--spectrum-global-color-gray-900)"
+          on:click={overflowPopover?.show}
+        />
+      </div>
+      {#each overflowedViews as view}
+        <a
+          class="nav-overflow-item"
+          class:active={view.id === activeId}
+          href={$url(`../${tableId}/${encodeURIComponent(view.id)}`)}
+        >
+          <MenuItem>
+            {view.name}
+          </MenuItem>
+        </a>
+      {/each}
+    </ActionMenu>
+  {/if}
   {#if views.length}
     <Icon
       name="Add"
@@ -187,7 +257,8 @@
 {/if}
 
 <style>
-  .view-nav-bar {
+  /* Main containers */
+  .nav {
     height: 50px;
     border-bottom: var(--border-light);
     display: flex;
@@ -195,9 +266,21 @@
     justify-content: flex-start;
     align-items: center;
     padding: 0 var(--spacing-xl);
-    gap: var(--spacing-m);
+    gap: 8px;
+  }
+  .nav__views {
+    width: 0;
+    flex: 1 1 auto;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    overflow: hidden;
+    gap: 8px;
   }
-  .nav-bar-item {
+
+  /* Table and view items */
+  .nav-item {
     padding: 6px 8px;
     border-radius: 4px;
     display: flex;
@@ -208,16 +291,22 @@
     transition: background 130ms ease-out, color 130ms ease-out;
     color: var(--spectrum-global-color-gray-600);
   }
-  .title {
-    font-size: 16px;
+  .nav-item.hidden {
+    visibility: hidden;
   }
-  .nav-bar-item.active,
-  .nav-bar-item:hover {
-    color: var(--spectrum-global-color-gray-900);
+  .nav-item.active,
+  .nav-item:hover {
     background: var(--spectrum-global-color-gray-300);
     cursor: pointer;
+    color: var(--spectrum-global-color-gray-900);
   }
-  .nav-bar-item:not(.active) :global(.icon) {
+  .nav-item:not(.active) :global(.icon) {
     display: none;
   }
+  .nav-item__title {
+    max-width: 150px;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    overflow: hidden;
+  }
 </style>

From d377186f0da38460b3d8bcd709f6299d22b19d47 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 16 Aug 2024 18:43:48 +0100
Subject: [PATCH 004/455] Add double layer context menu for overflow views

---
 .../bbui/src/ActionButton/ActionButton.svelte |   1 +
 .../bbui/src/ActionMenu/ActionMenu.svelte     |  41 ++++++-
 .../bbui/src/Actions/position_dropdown.js     |  10 +-
 packages/bbui/src/Menu/Item.svelte            |   5 +-
 packages/bbui/src/Popover/Popover.svelte      |  29 ++++-
 .../[tableId]/_components/ViewNavBar.svelte   | 112 +++++++++++++-----
 6 files changed, 156 insertions(+), 42 deletions(-)

diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte
index d3cec0f3078..7019ad7d2c1 100644
--- a/packages/bbui/src/ActionButton/ActionButton.svelte
+++ b/packages/bbui/src/ActionButton/ActionButton.svelte
@@ -115,6 +115,7 @@
   }
   .spectrum-ActionButton--quiet.is-selected {
     color: var(--spectrum-global-color-gray-900);
+    background: var(--spectrum-global-color-gray-300);
   }
   .noPadding {
     padding: 0;
diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte
index 75ddd679daf..e3de70e3ee6 100644
--- a/packages/bbui/src/ActionMenu/ActionMenu.svelte
+++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte
@@ -1,14 +1,20 @@
 <script>
-  import { setContext } from "svelte"
+  import { setContext, getContext } from "svelte"
   import Popover from "../Popover/Popover.svelte"
   import Menu from "../Menu/Menu.svelte"
 
   export let disabled = false
   export let align = "left"
   export let portalTarget
+  export let openOnHover = false
+  export let animate
+  export let offset
+
+  const actionMenuContext = getContext("actionMenu")
 
   let anchor
   let dropdown
+  let timeout
 
   // This is needed because display: contents is considered "invisible".
   // It should only ever be an action button, so should be fine.
@@ -16,11 +22,19 @@
     anchor = node.firstChild
   }
 
+  export const show = e => {
+    cancelHide()
+    dropdown.show()
+  }
+
   export const hide = () => {
     dropdown.hide()
   }
-  export const show = () => {
-    dropdown.show()
+
+  // Hides this menu and all parent menus
+  const hideAll = () => {
+    hide()
+    actionMenuContext?.hide()
   }
 
   const openMenu = event => {
@@ -30,12 +44,25 @@
     }
   }
 
-  setContext("actionMenu", { show, hide })
+  const queueHide = () => {
+    timeout = setTimeout(hide, 10)
+  }
+
+  const cancelHide = () => {
+    clearTimeout(timeout)
+  }
+
+  setContext("actionMenu", { show, hide, hideAll })
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
-<div use:getAnchor on:click={openMenu}>
+<div
+  use:getAnchor
+  on:click={openOnHover ? null : openMenu}
+  on:mouseenter={openOnHover ? show : null}
+  on:mouseleave={openOnHover ? queueHide : null}
+>
   <slot name="control" />
 </div>
 <Popover
@@ -43,9 +70,13 @@
   {anchor}
   {align}
   {portalTarget}
+  {animate}
+  {offset}
   resizable={false}
   on:open
   on:close
+  on:mouseenter={openOnHover ? cancelHide : null}
+  on:mouseleave={openOnHover ? queueHide : null}
 >
   <Menu>
     <slot />
diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js
index 21635592d24..c3675cfbe6a 100644
--- a/packages/bbui/src/Actions/position_dropdown.js
+++ b/packages/bbui/src/Actions/position_dropdown.js
@@ -151,9 +151,9 @@ export default function positionDropdown(element, opts) {
       // Determine X strategy
       if (align === "right") {
         applyXStrategy(Strategies.EndToEnd)
-      } else if (align === "right-outside") {
+      } else if (align === "right-outside" || align === "right-context-menu") {
         applyXStrategy(Strategies.StartToEnd)
-      } else if (align === "left-outside") {
+      } else if (align === "left-outside" || align === "left-context-menu") {
         applyXStrategy(Strategies.EndToStart)
       } else if (align === "center") {
         applyXStrategy(Strategies.MidPoint)
@@ -164,6 +164,12 @@ export default function positionDropdown(element, opts) {
       // Determine Y strategy
       if (align === "right-outside" || align === "left-outside") {
         applyYStrategy(Strategies.MidPoint)
+      } else if (
+        align === "right-context-menu" ||
+        align === "left-context-menu"
+      ) {
+        applyYStrategy(Strategies.StartToStart)
+        styles.top -= 4 // Manual adjustment for action menu padding
       } else {
         applyYStrategy(Strategies.StartToEnd)
       }
diff --git a/packages/bbui/src/Menu/Item.svelte b/packages/bbui/src/Menu/Item.svelte
index 758b819a62c..5e5f6d840cc 100644
--- a/packages/bbui/src/Menu/Item.svelte
+++ b/packages/bbui/src/Menu/Item.svelte
@@ -27,7 +27,7 @@
 
   const onClick = () => {
     if (actionMenu && !noClose) {
-      actionMenu.hide()
+      actionMenu.hideAll()
     }
     dispatch("click")
   }
@@ -47,8 +47,9 @@
     </div>
   {/if}
   <span class="spectrum-Menu-itemLabel"><slot /></span>
-  {#if keys?.length}
+  {#if keys?.length || $$slots.right}
     <div class="keys">
+      <slot name="right" />
       {#each keys as key}
         <div class="key">
           {#if key.startsWith("!")}
diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte
index aa811afe1ef..696e246d28b 100644
--- a/packages/bbui/src/Popover/Popover.svelte
+++ b/packages/bbui/src/Popover/Popover.svelte
@@ -1,7 +1,7 @@
 <script>
   import "@spectrum-css/popover/dist/index-vars.css"
   import Portal from "svelte-portal"
-  import { createEventDispatcher, getContext } from "svelte"
+  import { createEventDispatcher, getContext, onDestroy } from "svelte"
   import positionDropdown from "../Actions/position_dropdown"
   import clickOutside from "../Actions/click_outside"
   import { fly } from "svelte/transition"
@@ -28,7 +28,21 @@
   export let resizable = true
   export let wrap = false
 
+  const animationDuration = 260
+
+  let timeout
+  let blockPointerEvents = false
+
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
+  $: {
+    if (open && animate) {
+      blockPointerEvents = true
+      clearTimeout(timeout)
+      timeout = setTimeout(() => {
+        blockPointerEvents = false
+      }, animationDuration / 2)
+    }
+  }
 
   export const show = () => {
     dispatch("open")
@@ -77,6 +91,10 @@
       hide()
     }
   }
+
+  onDestroy(() => {
+    clearTimeout(timeout)
+  })
 </script>
 
 {#if open}
@@ -104,9 +122,13 @@
       class="spectrum-Popover is-open"
       class:customZindex
       class:hidden={!showPopover}
+      class:blockPointerEvents
       role="presentation"
       style="height: {customHeight}; --customZindex: {customZindex};"
-      transition:fly|local={{ y: -20, duration: animate ? 260 : 0 }}
+      transition:fly|local={{
+        y: -20,
+        duration: animate ? animationDuration : 0,
+      }}
       on:mouseenter
       on:mouseleave
     >
@@ -122,6 +144,9 @@
     overflow: auto;
     transition: opacity 260ms ease-out;
   }
+  .blockPointerEvents {
+    pointer-events: none;
+  }
   .hidden {
     opacity: 0;
     pointer-events: none;
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 7cc125ae67e..7f92e0c0553 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -6,7 +6,13 @@
     contextMenuStore,
   } from "stores/builder"
   import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
-  import { Icon, Button, Popover, ActionMenu, MenuItem } from "@budibase/bbui"
+  import {
+    Icon,
+    Button,
+    ActionButton,
+    ActionMenu,
+    MenuItem,
+  } from "@budibase/bbui"
   import { params, url } from "@roxi/routify"
   import EditViewModal from "./EditViewModal.svelte"
   import DeleteViewModal from "./DeleteViewModal.svelte"
@@ -19,12 +25,13 @@
   import { alphabetical } from "components/backend/TableNavigator/utils"
   import CreateViewModal from "./CreateViewModal.svelte"
   import { onDestroy } from "svelte"
+  import { derived } from "svelte/store"
 
-  let viewContainer
+  // View overflow
   let observer
+  let viewContainer
   let viewVisibiltyMap = {}
-  let overflowPopover
-  let anchor
+  let overflowMenu
 
   // Editing table
   let createViewModal
@@ -46,8 +53,15 @@
     .filter(x => x.version === 2)
     .slice()
     .sort(alphabetical)
-  $: setUpObserver(viewContainer, views)
+  $: setUpObserver(views)
   $: overflowedViews = views.filter(view => !viewVisibiltyMap[view.id])
+  $: viewHidden = viewVisibiltyMap[activeId] === false
+
+  const viewUrl = derived([url, params], ([$url, $params]) => viewId => {
+    return $url(`../${$params.tableId}/${encodeURIComponent(viewId)}`)
+  })
+
+  const tableUrl = derived(url, $url => tableId => $url(`../${tableId}`))
 
   const openTableContextMenu = e => {
     if (!tableEditable) {
@@ -114,11 +128,24 @@
     )
   }
 
-  const setUpObserver = (viewContainer, views) => {
-    if (!views.length || !viewContainer) {
-      observer?.disconnect()
+  const editOverflowView = async view => {
+    editableView = view
+    await tick()
+    editViewModal?.show()
+  }
+
+  const deleteOverflowView = async view => {
+    editableView = view
+    await tick()
+    deleteViewModal?.show()
+  }
+
+  const setUpObserver = async views => {
+    observer?.disconnect()
+    if (!views.length) {
       return
     }
+    await tick()
     observer = new IntersectionObserver(
       entries => {
         let updates = {}
@@ -136,7 +163,9 @@
       }
     )
     for (let child of viewContainer.children) {
-      observer.observe(child)
+      if (child.dataset.id) {
+        observer.observe(child)
+      }
     }
   }
 
@@ -152,7 +181,7 @@
     size="24"
   />
   <a
-    href={$url(`../${tableId}`)}
+    href={$tableUrl(tableId)}
     class="nav-item"
     class:active={tableId === activeId}
     on:contextmenu={openTableContextMenu}
@@ -178,7 +207,7 @@
       {#each views as view (view.id)}
         {@const selectedBy = $userSelectedResourceMap[view.id]}
         <a
-          href={$url(`../${tableId}/${encodeURIComponent(view.id)}`)}
+          href={$viewUrl(view.id)}
           class="nav-item"
           class:active={view.id === activeId}
           class:hidden={!viewVisibiltyMap[view.id]}
@@ -209,27 +238,42 @@
     </span>
   {/if}
   {#if overflowedViews.length}
-    <ActionMenu align="right">
+    <ActionMenu align="right" bind:this={overflowMenu}>
       <div slot="control">
-        <Icon
-          name="ChevronDown"
-          size="XL"
-          hoverable
-          color="var(--spectrum-global-color-gray-600)"
-          hoverColor="var(--spectrum-global-color-gray-900)"
-          on:click={overflowPopover?.show}
-        />
+        <ActionButton icon="ChevronDown" quiet selected={viewHidden}>
+          {overflowedViews.length} more
+        </ActionButton>
       </div>
       {#each overflowedViews as view}
-        <a
-          class="nav-overflow-item"
-          class:active={view.id === activeId}
-          href={$url(`../${tableId}/${encodeURIComponent(view.id)}`)}
+        <ActionMenu
+          align="left-context-menu"
+          openOnHover
+          animate={false}
+          offset={-4}
         >
-          <MenuItem>
-            {view.name}
+          <div slot="control">
+            <a
+              href={$viewUrl(view.id)}
+              class="nav-overflow-item"
+              class:active={view.id === activeId}
+              on:click={overflowMenu?.hide}
+            >
+              <MenuItem icon={viewHidden ? "Checkmark" : null}>
+                {view.name}
+                <Icon slot="right" name="ChevronRight" />
+              </MenuItem>
+            </a>
+          </div>
+          <a href={$viewUrl(view.id)}>
+            <MenuItem icon="Checkmark">Select</MenuItem>
+          </a>
+          <MenuItem icon="Edit" on:click={() => editOverflowView(view)}>
+            Edit
           </MenuItem>
-        </a>
+          <MenuItem icon="Delete" on:click={() => deleteOverflowView(view)}>
+            Delete
+          </MenuItem>
+        </ActionMenu>
       {/each}
     </ActionMenu>
   {/if}
@@ -266,22 +310,23 @@
     justify-content: flex-start;
     align-items: center;
     padding: 0 var(--spacing-xl);
-    gap: 8px;
+    gap: 12px;
   }
   .nav__views {
-    width: 0;
-    flex: 1 1 auto;
+    flex: 0 1 auto;
     display: flex;
     flex-direction: row;
     justify-content: flex-start;
     align-items: center;
     overflow: hidden;
     gap: 8px;
+    margin-left: -4px;
   }
 
   /* Table and view items */
   .nav-item {
-    padding: 6px 8px;
+    padding: 0 8px;
+    height: 32px;
     border-radius: 4px;
     display: flex;
     flex-direction: row;
@@ -309,4 +354,9 @@
     white-space: nowrap;
     overflow: hidden;
   }
+
+  /* OVerflow items */
+  .nav-overflow-item:not(.active) :global(> .spectrum-Menu-item > .icon) {
+    visibility: hidden;
+  }
 </style>

From a322f56efd572744ceed1a673863cc075a875ed4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 19 Aug 2024 14:35:43 +0100
Subject: [PATCH 005/455] Remove selection option in nested nav bar

---
 .../data/table/[tableId]/_components/ViewNavBar.svelte         | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 7f92e0c0553..cc62eb78d54 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -264,9 +264,6 @@
               </MenuItem>
             </a>
           </div>
-          <a href={$viewUrl(view.id)}>
-            <MenuItem icon="Checkmark">Select</MenuItem>
-          </a>
           <MenuItem icon="Edit" on:click={() => editOverflowView(view)}>
             Edit
           </MenuItem>

From b76a0b6e89585e19582db040a973a449cbbd17f1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 19 Aug 2024 15:00:15 +0100
Subject: [PATCH 006/455] Move all grid buttons to builder codebase, update
 table grids

---
 .../backend/DataTable/TableDataTable.svelte   | 104 ---------------
 .../buttons/grid}/ColumnsSettingButton.svelte |   0
 .../DataTable/buttons/grid}/SizeButton.svelte |   1 +
 .../DataTable/buttons/grid}/SortButton.svelte |   0
 .../data/table/[tableId]/index.svelte         | 125 ++++++++++++++++--
 .../src/components/grid/layout/Grid.svelte    |  16 +--
 6 files changed, 115 insertions(+), 131 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/TableDataTable.svelte
 rename packages/{frontend-core/src/components/grid/controls => builder/src/components/backend/DataTable/buttons/grid}/ColumnsSettingButton.svelte (100%)
 rename packages/{frontend-core/src/components/grid/controls => builder/src/components/backend/DataTable/buttons/grid}/SizeButton.svelte (99%)
 rename packages/{frontend-core/src/components/grid/controls => builder/src/components/backend/DataTable/buttons/grid}/SortButton.svelte (100%)

diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
deleted file mode 100644
index 2dd1eec0a8e..00000000000
--- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte
+++ /dev/null
@@ -1,104 +0,0 @@
-<script>
-  import { datasources, tables, integrations, appStore } from "stores/builder"
-  import { themeStore, admin } from "stores/portal"
-  import EditRolesButton from "./buttons/EditRolesButton.svelte"
-  import { TableNames } from "constants"
-  import { Grid } from "@budibase/frontend-core"
-  import { API } from "api"
-  import GridCreateAutomationButton from "./buttons/grid/GridCreateAutomationButton.svelte"
-  import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
-  import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
-  import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
-  import GridImportButton from "components/backend/DataTable/buttons/grid/GridImportButton.svelte"
-  import GridExportButton from "components/backend/DataTable/buttons/grid/GridExportButton.svelte"
-  import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
-  import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
-  import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
-  import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
-  import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
-  import { DB_TYPE_EXTERNAL } from "constants/backend"
-
-  const userSchemaOverrides = {
-    firstName: { displayName: "First name", disabled: true },
-    lastName: { displayName: "Last name", disabled: true },
-    email: { displayName: "Email", disabled: true },
-    roleId: { displayName: "Role", disabled: true },
-    status: { displayName: "Status", disabled: true },
-  }
-
-  $: id = $tables.selected?._id
-  $: isUsersTable = id === TableNames.USERS
-  $: isInternal = $tables.selected?.sourceType !== DB_TYPE_EXTERNAL
-  $: gridDatasource = {
-    type: "table",
-    tableId: id,
-  }
-  $: tableDatasource = $datasources.list.find(datasource => {
-    return datasource._id === $tables.selected?.sourceId
-  })
-  $: relationshipsEnabled = relationshipSupport(tableDatasource)
-
-  $: currentTheme = $themeStore?.theme
-  $: darkMode = !currentTheme.includes("light")
-
-  const relationshipSupport = datasource => {
-    const integration = $integrations[datasource?.source]
-    return !isInternal && integration?.relationships !== false
-  }
-
-  const handleGridTableUpdate = async e => {
-    tables.replaceTable(id, e.detail)
-
-    // We need to refresh datasources when an external table changes.
-    if (e.detail?.sourceType === DB_TYPE_EXTERNAL) {
-      await datasources.fetch()
-    }
-  }
-</script>
-
-<Grid
-  {API}
-  {darkMode}
-  datasource={gridDatasource}
-  canAddRows={!isUsersTable}
-  canDeleteRows={!isUsersTable}
-  canEditRows={!isUsersTable || !$appStore.features.disableUserMetadata}
-  canEditColumns={!isUsersTable || !$appStore.features.disableUserMetadata}
-  schemaOverrides={isUsersTable ? userSchemaOverrides : null}
-  showAvatars={false}
-  on:updatedatasource={handleGridTableUpdate}
-  isCloud={$admin.cloud}
->
-  <svelte:fragment slot="filter">
-    {#if isUsersTable && $appStore.features.disableUserMetadata}
-      <GridUsersTableButton />
-    {/if}
-    <GridFilterButton />
-  </svelte:fragment>
-  <svelte:fragment slot="controls">
-    <GridManageAccessButton />
-    {#if !isUsersTable}
-      <GridCreateAutomationButton />
-    {/if}
-    {#if relationshipsEnabled}
-      <GridRelationshipButton />
-    {/if}
-    {#if isUsersTable}
-      <EditRolesButton />
-    {:else}
-      <GridImportButton />
-    {/if}
-    <GridExportButton />
-    {#if isUsersTable}
-      <GridEditUserModal />
-    {:else}
-      <GridCreateEditRowModal />
-    {/if}
-  </svelte:fragment>
-  <svelte:fragment slot="edit-column">
-    <GridEditColumnModal />
-  </svelte:fragment>
-  <svelte:fragment slot="add-column">
-    <GridAddColumnModal />
-  </svelte:fragment>
-</Grid>
diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingButton.svelte
similarity index 100%
rename from packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingButton.svelte
diff --git a/packages/frontend-core/src/components/grid/controls/SizeButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/SizeButton.svelte
similarity index 99%
rename from packages/frontend-core/src/components/grid/controls/SizeButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/SizeButton.svelte
index 320aa473457..a4db68f1aa5 100644
--- a/packages/frontend-core/src/components/grid/controls/SizeButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/SizeButton.svelte
@@ -66,6 +66,7 @@
   }
 </script>
 
+#
 <div bind:this={anchor}>
   <ActionButton
     icon="MoveUpDown"
diff --git a/packages/frontend-core/src/components/grid/controls/SortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/SortButton.svelte
similarity index 100%
rename from packages/frontend-core/src/components/grid/controls/SortButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/SortButton.svelte
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index d79a0bc0ad4..b862ab52aea 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -1,7 +1,68 @@
 <script>
-  import TableDataTable from "components/backend/DataTable/TableDataTable.svelte"
-  import { tables } from "stores/builder"
   import { Banner } from "@budibase/bbui"
+  import { datasources, tables, integrations, appStore } from "stores/builder"
+  import { themeStore, admin } from "stores/portal"
+  import EditRolesButton from "components/backend/DataTable/buttons/EditRolesButton.svelte"
+  import { TableNames } from "constants"
+  import { Grid } from "@budibase/frontend-core"
+  import { API } from "api"
+  import GridCreateAutomationButton from "components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte"
+  import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
+  import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
+  import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
+  import GridImportButton from "components/backend/DataTable/buttons/grid/GridImportButton.svelte"
+  import GridExportButton from "components/backend/DataTable/buttons/grid/GridExportButton.svelte"
+  import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
+  import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
+  import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
+  import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
+
+  const userSchemaOverrides = {
+    firstName: { displayName: "First name", disabled: true },
+    lastName: { displayName: "Last name", disabled: true },
+    email: { displayName: "Email", disabled: true },
+    roleId: { displayName: "Role", disabled: true },
+    status: { displayName: "Status", disabled: true },
+  }
+
+  $: autoColumnStatus = verifyAutocolumns($tables?.selected)
+  $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
+    if (status.length > 1) {
+      acc = [...acc, ...status]
+    }
+    return acc
+  }, [])
+  $: invalidColumnText = duplicates.map(entry => {
+    return `${entry.name} (${entry.subtype})`
+  })
+  $: id = $tables.selected?._id
+  $: isUsersTable = id === TableNames.USERS
+  $: isInternal = $tables.selected?.sourceType !== DB_TYPE_EXTERNAL
+  $: gridDatasource = {
+    type: "table",
+    tableId: id,
+  }
+  $: tableDatasource = $datasources.list.find(datasource => {
+    return datasource._id === $tables.selected?.sourceId
+  })
+  $: relationshipsEnabled = relationshipSupport(tableDatasource)
+  $: currentTheme = $themeStore?.theme
+  $: darkMode = !currentTheme.includes("light")
+
+  const relationshipSupport = datasource => {
+    const integration = $integrations[datasource?.source]
+    return !isInternal && integration?.relationships !== false
+  }
+
+  const handleGridTableUpdate = async e => {
+    tables.replaceTable(id, e.detail)
+
+    // We need to refresh datasources when an external table changes.
+    if (e.detail?.sourceType === DB_TYPE_EXTERNAL) {
+      await datasources.fetch()
+    }
+  }
 
   const verifyAutocolumns = table => {
     // Check for duplicates
@@ -17,17 +78,6 @@
       return acc
     }, {})
   }
-
-  $: autoColumnStatus = verifyAutocolumns($tables?.selected)
-  $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
-    if (status.length > 1) {
-      acc = [...acc, ...status]
-    }
-    return acc
-  }, [])
-  $: invalidColumnText = duplicates.map(entry => {
-    return `${entry.name} (${entry.subtype})`
-  })
 </script>
 
 {#if $tables?.selected?.name}
@@ -40,7 +90,54 @@
       </Banner>
     </div>
   {/if}
-  <TableDataTable />
+  <Grid
+    {API}
+    {darkMode}
+    datasource={gridDatasource}
+    canAddRows={!isUsersTable}
+    canDeleteRows={!isUsersTable}
+    canEditRows={!isUsersTable || !$appStore.features.disableUserMetadata}
+    canEditColumns={!isUsersTable || !$appStore.features.disableUserMetadata}
+    schemaOverrides={isUsersTable ? userSchemaOverrides : null}
+    showAvatars={false}
+    isCloud={$admin.cloud}
+    on:updatedatasource={handleGridTableUpdate}
+  >
+    <!-- Controls -->
+    <svelte:fragment slot="controls">
+      {#if isUsersTable && $appStore.features.disableUserMetadata}
+        <GridUsersTableButton />
+      {/if}
+      <GridManageAccessButton />
+      {#if !isUsersTable}
+        <GridCreateAutomationButton />
+      {/if}
+      {#if relationshipsEnabled}
+        <GridRelationshipButton />
+      {/if}
+      {#if isUsersTable}
+        <EditRolesButton />
+      {:else}
+        <GridImportButton />
+      {/if}
+      <GridExportButton />
+    </svelte:fragment>
+
+    <!-- Content for editing columns -->
+    <svelte:fragment slot="edit-column">
+      <GridEditColumnModal />
+    </svelte:fragment>
+    <svelte:fragment slot="add-column">
+      <GridAddColumnModal />
+    </svelte:fragment>
+
+    <!-- Listening to events for editing rows in modals -->
+    {#if isUsersTable}
+      <GridEditUserModal />
+    {:else}
+      <GridCreateEditRowModal />
+    {/if}
+  </Grid>
 {:else}
   <i>Create your first table to start building</i>
 {/if}
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 04a11da8121..86c154be952 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -19,9 +19,6 @@
   import StickyColumn from "./StickyColumn.svelte"
   import UserAvatars from "./UserAvatars.svelte"
   import KeyboardManager from "../overlays/KeyboardManager.svelte"
-  import SortButton from "../controls/SortButton.svelte"
-  import ColumnsSettingButton from "../controls/ColumnsSettingButton.svelte"
-  import SizeButton from "../controls/SizeButton.svelte"
   import NewRow from "./NewRow.svelte"
   import { createGridWebsocket } from "../lib/websocket"
   import {
@@ -47,7 +44,6 @@
   export let quiet = false
   export let collaboration = true
   export let showAvatars = true
-  export let showControls = true
   export let initialFilter = null
   export let initialSortColumn = null
   export let initialSortOrder = null
@@ -57,7 +53,6 @@
   export let buttons = null
   export let darkMode
   export let isCloud = null
-  export let allowViewReadonlyColumns = false
   export let rowConditions = null
 
   // Unique identifier for DOM nodes inside this instance
@@ -103,7 +98,6 @@
     quiet,
     collaboration,
     showAvatars,
-    showControls,
     initialFilter,
     initialSortColumn,
     initialSortOrder,
@@ -113,13 +107,12 @@
     buttons,
     darkMode,
     isCloud,
-    allowViewReadonlyColumns,
     rowConditions,
   })
 
   // Derive min height and make available in context
   const minHeight = derived(rowHeight, $height => {
-    const heightForControls = showControls ? ControlsHeight : 0
+    const heightForControls = $$slots.controls ? ControlsHeight : 0
     return VPadding + SmallRowHeight + $height + heightForControls
   })
   context = { ...context, minHeight }
@@ -150,13 +143,9 @@
   on:mouseleave={() => gridFocused.set(false)}
   style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-overflow:{MaxCellRenderOverflow}px; --content-lines:{$contentLines}; --min-height:{$minHeight}px; --controls-height:{ControlsHeight}px; --scroll-bar-size:{ScrollBarSize}px;"
 >
-  {#if showControls}
+  {#if $$slots.controls}
     <div class="controls">
       <div class="controls-left">
-        <slot name="filter" />
-        <SortButton />
-        <ColumnsSettingButton {allowViewReadonlyColumns} />
-        <SizeButton />
         <slot name="controls" />
       </div>
       <div class="controls-right">
@@ -216,6 +205,7 @@
   <BulkDeleteHandler />
   <ClipboardHandler />
   <KeyboardManager />
+  <slot />
 </div>
 
 <style>

From d316ba71e199f1153b37eb9f8f4d0376591d038d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 19 Aug 2024 15:30:21 +0100
Subject: [PATCH 007/455] Update buttons for legacy views

---
 .../backend/DataTable/ViewDataTable.svelte    | 80 -------------------
 .../data/table/[tableId]/index.svelte         |  4 +-
 .../data/table/[tableId]/view/index.svelte    | 19 -----
 .../view/v1/[viewName]/_layout.svelte         | 24 ------
 .../[tableId]/view/v1/[viewName]/index.svelte | 18 -----
 .../data/table/[tableId]/view/v1/index.svelte |  5 --
 .../data/view/v1/[viewName]/index.svelte      | 78 +++++++++++++++++-
 7 files changed, 75 insertions(+), 153 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte

diff --git a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
deleted file mode 100644
index 684cbd6cf49..00000000000
--- a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte
+++ /dev/null
@@ -1,80 +0,0 @@
-<script>
-  import { API } from "api"
-  import { tables } from "stores/builder"
-
-  import Table from "./Table.svelte"
-  import CalculateButton from "./buttons/CalculateButton.svelte"
-  import GroupByButton from "./buttons/GroupByButton.svelte"
-  import ViewFilterButton from "./buttons/ViewFilterButton.svelte"
-  import ExportButton from "./buttons/ExportButton.svelte"
-  import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
-  import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
-  import { notifications } from "@budibase/bbui"
-  import { ROW_EXPORT_FORMATS } from "constants/backend"
-
-  export let view = {}
-
-  let hideAutocolumns = true
-  let data = []
-  let loading = false
-
-  $: name = view.name
-  $: schema = view.schema
-  $: calculation = view.calculation
-
-  $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {
-    if (calculation && key === ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA) {
-      return false
-    }
-    return true
-  })
-
-  // Fetch rows for specified view
-  $: fetchViewData(name, view.field, view.groupBy, view.calculation)
-
-  async function fetchViewData(name, field, groupBy, calculation) {
-    loading = true
-    const _tables = $tables.list
-    const allTableViews = _tables.map(table => table.views)
-    const thisView = allTableViews.filter(
-      views => views != null && views[name] != null
-    )[0]
-
-    // Don't fetch view data if the view no longer exists
-    if (!thisView) {
-      loading = false
-      return
-    }
-    try {
-      data = await API.fetchViewData({
-        name,
-        calculation,
-        field,
-        groupBy,
-      })
-    } catch (error) {
-      notifications.error("Error fetching view data")
-    }
-    loading = false
-  }
-</script>
-
-<Table
-  title={decodeURI(name)}
-  {schema}
-  tableId={view.tableId}
-  {data}
-  {loading}
-  rowCount={10}
-  allowEditing={false}
-  bind:hideAutocolumns
->
-  <ViewFilterButton {view} />
-  <CalculateButton {view} />
-  {#if view.calculation}
-    <GroupByButton {view} />
-  {/if}
-  <ManageAccessButton resourceId={decodeURI(name)} />
-  <HideAutocolumnButton bind:hideAutocolumns />
-  <ExportButton view={view.name} formats={supportedFormats} />
-</Table>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index b862ab52aea..5b5132255b2 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -115,9 +115,7 @@
       {#if relationshipsEnabled}
         <GridRelationshipButton />
       {/if}
-      {#if isUsersTable}
-        <EditRolesButton />
-      {:else}
+      {#if !isUsersTable}
         <GridImportButton />
       {/if}
       <GridExportButton />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte
deleted file mode 100644
index 623cd224dbc..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/index.svelte
+++ /dev/null
@@ -1,19 +0,0 @@
-<script>
-  import { onMount } from "svelte"
-  import { views, viewsV2 } from "stores/builder"
-  import { redirect } from "@roxi/routify"
-
-  onMount(async () => {
-    if ($viewsV2.selected) {
-      $redirect(`./v2/${$viewsV2.selected.id}`)
-    } else if ($viewsV2.list?.length) {
-      $redirect(`./v2/${$viewsV2.list[0].id}`)
-    } else if ($views.selected) {
-      $redirect(`./${encodeURIComponent($views.selected?.name)}`)
-    } else if ($views.list?.length) {
-      $redirect(`./${encodeURIComponent($views.list[0].name)}`)
-    } else {
-      $redirect("../")
-    }
-  })
-</script>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte
deleted file mode 100644
index df3a18c2bb8..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/_layout.svelte
+++ /dev/null
@@ -1,24 +0,0 @@
-<script>
-  import { views, builderStore } from "stores/builder"
-  import { syncURLToState } from "helpers/urlStateSync"
-  import * as routify from "@roxi/routify"
-  import { onDestroy } from "svelte"
-
-  $: name = $views.selectedViewName
-  $: builderStore.selectResource(name)
-
-  const stopSyncing = syncURLToState({
-    urlParam: "viewName",
-    stateKey: "selectedViewName",
-    validate: name => $views.list?.some(view => view.name === name),
-    update: views.select,
-    fallbackUrl: "../../",
-    store: views,
-    routify,
-    decode: decodeURIComponent,
-  })
-
-  onDestroy(stopSyncing)
-</script>
-
-<slot />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte
deleted file mode 100644
index 51149b602d2..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/[viewName]/index.svelte
+++ /dev/null
@@ -1,18 +0,0 @@
-<script>
-  import ViewDataTable from "components/backend/DataTable/ViewDataTable.svelte"
-  import { views } from "stores/builder"
-
-  $: selectedView = $views.selected
-</script>
-
-{#if selectedView}
-  <ViewDataTable view={selectedView} />
-{:else}<i>Create your first table to start building</i>{/if}
-
-<style>
-  i {
-    font-size: var(--font-size-m);
-    color: var(--grey-4);
-    margin-top: 2px;
-  }
-</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte
deleted file mode 100644
index c11ca870230..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/view/v1/index.svelte
+++ /dev/null
@@ -1,5 +0,0 @@
-<script>
-  import { redirect } from "@roxi/routify"
-
-  $redirect("../")
-</script>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
index 51149b602d2..b8a8dc514b6 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
@@ -1,12 +1,82 @@
 <script>
-  import ViewDataTable from "components/backend/DataTable/ViewDataTable.svelte"
   import { views } from "stores/builder"
+  import { API } from "api"
+  import { tables } from "stores/builder"
+  import Table from "./Table.svelte"
+  import CalculateButton from "./buttons/CalculateButton.svelte"
+  import GroupByButton from "./buttons/GroupByButton.svelte"
+  import ViewFilterButton from "./buttons/ViewFilterButton.svelte"
+  import ExportButton from "./buttons/ExportButton.svelte"
+  import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
+  import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
+  import { notifications } from "@budibase/bbui"
+  import { ROW_EXPORT_FORMATS } from "constants/backend"
 
-  $: selectedView = $views.selected
+  let hideAutocolumns = true
+  let data = []
+  let loading = false
+
+  $: view = $views.selected
+  $: name = view.name
+  $: schema = view.schema
+  $: calculation = view.calculation
+  $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {
+    if (calculation && key === ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA) {
+      return false
+    }
+    return true
+  })
+
+  // Fetch rows for specified view
+  $: fetchViewData(name, view.field, view.groupBy, view.calculation)
+
+  async function fetchViewData(name, field, groupBy, calculation) {
+    loading = true
+    const _tables = $tables.list
+    const allTableViews = _tables.map(table => table.views)
+    const thisView = allTableViews.filter(
+      views => views != null && views[name] != null
+    )[0]
+
+    // Don't fetch view data if the view no longer exists
+    if (!thisView) {
+      loading = false
+      return
+    }
+    try {
+      data = await API.fetchViewData({
+        name,
+        calculation,
+        field,
+        groupBy,
+      })
+    } catch (error) {
+      notifications.error("Error fetching view data")
+    }
+    loading = false
+  }
 </script>
 
-{#if selectedView}
-  <ViewDataTable view={selectedView} />
+{#if view}
+  <Table
+    title={decodeURI(name)}
+    {schema}
+    tableId={view.tableId}
+    {data}
+    {loading}
+    rowCount={10}
+    allowEditing={false}
+    bind:hideAutocolumns
+  >
+    <ViewFilterButton {view} />
+    <CalculateButton {view} />
+    {#if view.calculation}
+      <GroupByButton {view} />
+    {/if}
+    <ManageAccessButton resourceId={decodeURI(name)} />
+    <HideAutocolumnButton bind:hideAutocolumns />
+    <ExportButton view={view.name} formats={supportedFormats} />
+  </Table>
 {:else}<i>Create your first table to start building</i>{/if}
 
 <style>

From 38bd64cebdadb1069026ff69af01418965d6c960 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 19 Aug 2024 15:36:57 +0100
Subject: [PATCH 008/455] Fix imports

---
 .../data/view/v1/[viewName]/index.svelte           | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
index b8a8dc514b6..d96a9239ba7 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
@@ -2,13 +2,13 @@
   import { views } from "stores/builder"
   import { API } from "api"
   import { tables } from "stores/builder"
-  import Table from "./Table.svelte"
-  import CalculateButton from "./buttons/CalculateButton.svelte"
-  import GroupByButton from "./buttons/GroupByButton.svelte"
-  import ViewFilterButton from "./buttons/ViewFilterButton.svelte"
-  import ExportButton from "./buttons/ExportButton.svelte"
-  import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
-  import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
+  import Table from "components/backend/DataTable/Table.svelte"
+  import CalculateButton from "components/backend/DataTable/buttons/CalculateButton.svelte"
+  import GroupByButton from "components/backend/DataTable/buttons/GroupByButton.svelte"
+  import ViewFilterButton from "components/backend/DataTable/buttons/ViewFilterButton.svelte"
+  import ExportButton from "components/backend/DataTable/buttons/ExportButton.svelte"
+  import ManageAccessButton from "components/backend/DataTable/buttons/ManageAccessButton.svelte"
+  import HideAutocolumnButton from "components/backend/DataTable/buttons/HideAutocolumnButton.svelte"
   import { notifications } from "@budibase/bbui"
   import { ROW_EXPORT_FORMATS } from "constants/backend"
 

From 54f9507e53c7226e2d175e31f5b50ca326f7beca Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 20 Aug 2024 08:24:23 +0100
Subject: [PATCH 009/455] Update buttons for views

---
 packages/bbui/package.json                    |  2 +-
 .../DataTable/RelationshipDataTable.svelte    | 44 -------------------
 .../backend/DataTable/ViewV2DataTable.svelte  | 39 ----------------
 ...svelte => GridColumnsSettingButton.svelte} |  6 +--
 ...izeButton.svelte => GridSizeButton.svelte} | 27 ++++++------
 ...ortButton.svelte => GridSortButton.svelte} |  0
 .../common}/ToggleActionButtonGroup.svelte    |  9 ++--
 .../table/[tableId]/[viewId]/index.svelte     | 44 ++++++++++++++++++-
 .../relationship/[rowId]/[field]/index.svelte | 10 -----
 .../relationship/[rowId]/index.svelte         |  7 ---
 .../table/[tableId]/relationship/index.svelte |  7 ---
 .../data/view/v2/[viewId]/_layout.svelte      | 24 ----------
 .../data/view/v2/[viewId]/index.svelte        |  5 ---
 .../[application]/data/view/v2/index.svelte   |  5 ---
 .../components/grid/cells/HeaderCell.svelte   |  2 +-
 .../src/components/grid/layout/Grid.svelte    | 22 +++++-----
 .../src/components/grid/lib/utils.js          | 21 ---------
 packages/frontend-core/src/utils/index.js     |  1 +
 packages/frontend-core/src/utils/schema.js    | 20 +++++++++
 19 files changed, 95 insertions(+), 200 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte
 delete mode 100644 packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
 rename packages/builder/src/components/backend/DataTable/buttons/grid/{ColumnsSettingButton.svelte => GridColumnsSettingButton.svelte} (94%)
 rename packages/builder/src/components/backend/DataTable/buttons/grid/{SizeButton.svelte => GridSizeButton.svelte} (86%)
 rename packages/builder/src/components/backend/DataTable/buttons/grid/{SortButton.svelte => GridSortButton.svelte} (100%)
 rename packages/{frontend-core/src/components/grid/controls => builder/src/components/common}/ToggleActionButtonGroup.svelte (82%)
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte
 create mode 100644 packages/frontend-core/src/utils/schema.js

diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index 1b3510cf3bf..272e0937e10 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -34,7 +34,7 @@
     "dist"
   ],
   "dependencies": {
-    "@adobe/spectrum-css-workflow-icons": "1.2.1",
+    "@adobe/spectrum-css-workflow-icons": "3.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@spectrum-css/accordion": "3.0.24",
diff --git a/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte b/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte
deleted file mode 100644
index f107acc3158..00000000000
--- a/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte
+++ /dev/null
@@ -1,44 +0,0 @@
-<script>
-  import { API } from "api"
-  import Table from "./Table.svelte"
-  import { tables } from "stores/builder"
-  import { notifications } from "@budibase/bbui"
-
-  export let tableId
-  export let rowId
-  export let fieldName
-
-  let row
-  let title
-
-  $: data = row?.[fieldName] ?? []
-  $: linkedTableId = data?.length ? data[0].tableId : null
-  $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
-  $: schema = linkedTable?.schema
-  $: table = $tables.list.find(table => table._id === tableId)
-  $: fetchData(tableId, rowId)
-  $: {
-    let rowLabel = row?.[table?.primaryDisplay]
-    if (rowLabel) {
-      title = `${rowLabel} - ${fieldName}`
-    } else {
-      title = fieldName
-    }
-  }
-
-  async function fetchData(tableId, rowId) {
-    try {
-      row = await API.fetchRelationshipData({
-        tableId,
-        rowId,
-      })
-    } catch (error) {
-      row = null
-      notifications.error("Error fetching relationship data")
-    }
-  }
-</script>
-
-{#if row && row._id === rowId}
-  <Table {title} {schema} {data} />
-{/if}
diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
deleted file mode 100644
index 5f7a0feb674..00000000000
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ /dev/null
@@ -1,39 +0,0 @@
-<script>
-  import { viewsV2 } from "stores/builder"
-  import { admin, licensing } from "stores/portal"
-  import { Grid } from "@budibase/frontend-core"
-  import { API } from "api"
-  import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
-  import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
-  import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
-
-  $: id = $viewsV2.selected?.id
-  $: datasource = {
-    type: "viewV2",
-    id,
-    tableId: $viewsV2.selected?.tableId,
-  }
-
-  const handleGridViewUpdate = async e => {
-    viewsV2.replaceView(id, e.detail)
-  }
-</script>
-
-<Grid
-  {API}
-  {datasource}
-  allowAddRows
-  allowDeleteRows
-  showAvatars={false}
-  on:updatedatasource={handleGridViewUpdate}
-  isCloud={$admin.cloud}
-  allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
->
-  <svelte:fragment slot="filter">
-    <GridFilterButton />
-  </svelte:fragment>
-  <svelte:fragment slot="controls">
-    <GridCreateEditRowModal />
-    <GridManageAccessButton />
-  </svelte:fragment>
-</Grid>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
similarity index 94%
rename from packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
index f3a7678cf2b..706b98ae290 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
@@ -1,9 +1,9 @@
 <script>
   import { getContext } from "svelte"
   import { ActionButton, Popover, Icon, notifications } from "@budibase/bbui"
-  import { getColumnIcon } from "../lib/utils"
-  import ToggleActionButtonGroup from "./ToggleActionButtonGroup.svelte"
+  import ToggleActionButtonGroup from "components/common/ToggleActionButtonGroup.svelte"
   import { helpers } from "@budibase/shared-core"
+  import { SchemaUtils } from "@budibase/frontend-core"
 
   export let allowViewReadonlyColumns = false
 
@@ -110,7 +110,7 @@
     <div class="columns">
       {#each displayColumns as column}
         <div class="column">
-          <Icon size="S" name={getColumnIcon(column)} />
+          <Icon size="S" name={SchemaUtils.getColumnIcon(column)} />
           <div class="column-label" title={column.label}>
             {column.label}
           </div>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/SizeButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
similarity index 86%
rename from packages/builder/src/components/backend/DataTable/buttons/grid/SizeButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
index a4db68f1aa5..f9499e54b76 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/SizeButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
@@ -1,34 +1,34 @@
 <script>
   import { getContext } from "svelte"
   import { ActionButton, Popover, Label } from "@budibase/bbui"
-  import {
-    DefaultColumnWidth,
-    LargeRowHeight,
-    MediumRowHeight,
-    SmallRowHeight,
-  } from "../lib/constants"
 
-  const { columns, rowHeight, definition, fixedRowHeight, datasource } =
-    getContext("grid")
+  const {
+    Constants,
+    columns,
+    rowHeight,
+    definition,
+    fixedRowHeight,
+    datasource,
+  } = getContext("grid")
 
   // Some constants for column width options
   const smallColSize = 120
-  const mediumColSize = DefaultColumnWidth
-  const largeColSize = DefaultColumnWidth * 1.5
+  const mediumColSize = Constants.DefaultColumnWidth
+  const largeColSize = Constants.DefaultColumnWidth * 1.5
 
   // Row height sizes
   const rowSizeOptions = [
     {
       label: "Small",
-      size: SmallRowHeight,
+      size: Constants.SmallRowHeight,
     },
     {
       label: "Medium",
-      size: MediumRowHeight,
+      size: Constants.MediumRowHeight,
     },
     {
       label: "Large",
-      size: LargeRowHeight,
+      size: Constants.LargeRowHeight,
     },
   ]
 
@@ -66,7 +66,6 @@
   }
 </script>
 
-#
 <div bind:this={anchor}>
   <ActionButton
     icon="MoveUpDown"
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/SortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
similarity index 100%
rename from packages/builder/src/components/backend/DataTable/buttons/grid/SortButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
diff --git a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte b/packages/builder/src/components/common/ToggleActionButtonGroup.svelte
similarity index 82%
rename from packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte
rename to packages/builder/src/components/common/ToggleActionButtonGroup.svelte
index 497e77c2c91..8a5778534fa 100644
--- a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte
+++ b/packages/builder/src/components/common/ToggleActionButtonGroup.svelte
@@ -9,7 +9,7 @@
   export let options
 </script>
 
-<div class="permissionPicker">
+<div>
   {#each options as option}
     <AbsTooltip text={option.tooltip} type={TooltipType.Info}>
       <ActionButton
@@ -26,15 +26,14 @@
 </div>
 
 <style>
-  .permissionPicker {
+  div {
     display: flex;
     gap: var(--spacing-xs);
   }
-
-  .permissionPicker :global(.spectrum-Icon) {
+  div :global(.spectrum-Icon) {
     width: 14px;
   }
-  .permissionPicker :global(.spectrum-ActionButton) {
+  div :global(.spectrum-ActionButton) {
     width: 28px;
     height: 28px;
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index c2281710bad..fb784334812 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -1,5 +1,45 @@
 <script>
-  import ViewV2DataTable from "components/backend/DataTable/ViewV2DataTable.svelte"
+  import { viewsV2 } from "stores/builder"
+  import { admin, licensing } from "stores/portal"
+  import { Grid } from "@budibase/frontend-core"
+  import { API } from "api"
+  import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
+  import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
+  import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
+  import GridSortButton from "components/backend/DataTable/buttons/grid/GridSortButton.svelte"
+  import GridColumnsSettingButton from "components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte"
+  import GridSizeButton from "components/backend/DataTable/buttons/grid/GridSizeButton.svelte"
+  import GridCreateAutomationButton from "components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte"
+
+  $: id = $viewsV2.selected?.id
+  $: datasource = {
+    type: "viewV2",
+    id,
+    tableId: $viewsV2.selected?.tableId,
+  }
+
+  const handleGridViewUpdate = async e => {
+    viewsV2.replaceView(id, e.detail)
+  }
 </script>
 
-<ViewV2DataTable />
+<Grid
+  {API}
+  {datasource}
+  allowAddRows
+  allowDeleteRows
+  showAvatars={false}
+  on:updatedatasource={handleGridViewUpdate}
+  isCloud={$admin.cloud}
+  allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
+>
+  <svelte:fragment slot="controls">
+    <GridFilterButton />
+    <GridSortButton />
+    <GridColumnsSettingButton />
+    <GridManageAccessButton />
+    <GridSizeButton />
+    <GridCreateAutomationButton />
+  </svelte:fragment>
+  <GridCreateEditRowModal />
+</Grid>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte
deleted file mode 100644
index 39dbcb9d112..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte
+++ /dev/null
@@ -1,10 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import RelationshipDataTable from "components/backend/DataTable/RelationshipDataTable.svelte"
-</script>
-
-<RelationshipDataTable
-  tableId={$params.tableId}
-  rowId={$params.rowId}
-  fieldName={decodeURI($params.field)}
-/>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte
deleted file mode 100644
index 348ed0b5bf4..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte
+++ /dev/null
@@ -1,7 +0,0 @@
-<script>
-  import { redirect } from "@roxi/routify"
-
-  $redirect("../../")
-</script>
-
-<!-- routify:options index=false -->
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte
deleted file mode 100644
index cecec0ab532..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte
+++ /dev/null
@@ -1,7 +0,0 @@
-<script>
-  import { redirect } from "@roxi/routify"
-
-  $redirect("../")
-</script>
-
-<!-- routify:options index=false -->
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte
deleted file mode 100644
index 3d0cffc3879..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte
+++ /dev/null
@@ -1,24 +0,0 @@
-<script>
-  import { viewsV2, builderStore } from "stores/builder"
-  import { syncURLToState } from "helpers/urlStateSync"
-  import * as routify from "@roxi/routify"
-  import { onDestroy } from "svelte"
-
-  $: id = $viewsV2.selectedViewId
-  $: builderStore.selectResource(id)
-
-  const stopSyncing = syncURLToState({
-    urlParam: "viewId",
-    stateKey: "selectedViewId",
-    validate: id => $viewsV2.list?.some(view => view.id === id),
-    update: viewsV2.select,
-    fallbackUrl: "../../",
-    store: viewsV2,
-    routify,
-    decode: decodeURIComponent,
-  })
-
-  onDestroy(stopSyncing)
-</script>
-
-<slot />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte
deleted file mode 100644
index c2281710bad..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte
+++ /dev/null
@@ -1,5 +0,0 @@
-<script>
-  import ViewV2DataTable from "components/backend/DataTable/ViewV2DataTable.svelte"
-</script>
-
-<ViewV2DataTable />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte
deleted file mode 100644
index c11ca870230..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte
+++ /dev/null
@@ -1,5 +0,0 @@
-<script>
-  import { redirect } from "@roxi/routify"
-
-  $redirect("../")
-</script>
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 3b6aa5d4241..1e6e49c3549 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -3,7 +3,7 @@
   import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
   import { Icon, Menu, MenuItem, Modal } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
-  import { getColumnIcon } from "../lib/utils"
+  import { getColumnIcon } from "../../../utils/schema"
   import MigrationModal from "../controls/MigrationModal.svelte"
   import { debounce } from "../../../utils/utils"
   import { FieldType, FormulaType } from "@budibase/types"
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 86c154be952..9b9abfbabd3 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -21,15 +21,7 @@
   import KeyboardManager from "../overlays/KeyboardManager.svelte"
   import NewRow from "./NewRow.svelte"
   import { createGridWebsocket } from "../lib/websocket"
-  import {
-    MaxCellRenderOverflow,
-    GutterWidth,
-    DefaultRowHeight,
-    VPadding,
-    SmallRowHeight,
-    ControlsHeight,
-    ScrollBarSize,
-  } from "../lib/constants"
+  import * as Constants from "../lib/constants"
 
   export let API = null
   export let datasource = null
@@ -64,6 +56,7 @@
   // Build up context
   let context = {
     API: API || createAPIClient(),
+    Constants,
     gridID,
     props,
   }
@@ -112,8 +105,13 @@
 
   // Derive min height and make available in context
   const minHeight = derived(rowHeight, $height => {
-    const heightForControls = $$slots.controls ? ControlsHeight : 0
-    return VPadding + SmallRowHeight + $height + heightForControls
+    const heightForControls = $$slots.controls ? Constants.ControlsHeight : 0
+    return (
+      Constants.VPadding +
+      Constants.SmallRowHeight +
+      $height +
+      heightForControls
+    )
   })
   context = { ...context, minHeight }
 
@@ -141,7 +139,7 @@
   class:quiet
   on:mouseenter={() => gridFocused.set(true)}
   on:mouseleave={() => gridFocused.set(false)}
-  style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-overflow:{MaxCellRenderOverflow}px; --content-lines:{$contentLines}; --min-height:{$minHeight}px; --controls-height:{ControlsHeight}px; --scroll-bar-size:{ScrollBarSize}px;"
+  style="--row-height:{$rowHeight}px; --default-row-height:{Constants.DefaultRowHeight}px; --gutter-width:{Constants.GutterWidth}px; --max-cell-render-overflow:{Constants.MaxCellRenderOverflow}px; --content-lines:{$contentLines}; --min-height:{$minHeight}px; --controls-height:{Constants.ControlsHeight}px; --scroll-bar-size:{Constants.ScrollBarSize}px;"
 >
   {#if $$slots.controls}
     <div class="controls">
diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js
index 1988b66cc23..b5485abc36f 100644
--- a/packages/frontend-core/src/components/grid/lib/utils.js
+++ b/packages/frontend-core/src/components/grid/lib/utils.js
@@ -1,6 +1,3 @@
-import { helpers } from "@budibase/shared-core"
-import { TypeIconMap } from "../../../constants"
-
 // We can't use "-" as a separator as this can be present in the ID
 // or column name, so we use something very unusual to avoid this problem
 const JOINING_CHARACTER = "‽‽"
@@ -18,24 +15,6 @@ export const getCellID = (rowId, fieldName) => {
   return `${rowId}${JOINING_CHARACTER}${fieldName}`
 }
 
-export const getColumnIcon = column => {
-  if (column.schema.autocolumn) {
-    return "MagicWand"
-  }
-
-  if (helpers.schema.isDeprecatedSingleUserColumn(column.schema)) {
-    return "User"
-  }
-
-  const { type, subtype } = column.schema
-  const result =
-    typeof TypeIconMap[type] === "object" && subtype
-      ? TypeIconMap[type][subtype]
-      : TypeIconMap[type]
-
-  return result || "Text"
-}
-
 export const parseEventLocation = e => {
   return {
     x: e.clientX ?? e.touches?.[0]?.clientX,
diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js
index 9eb72060123..f4714466784 100644
--- a/packages/frontend-core/src/utils/index.js
+++ b/packages/frontend-core/src/utils/index.js
@@ -5,6 +5,7 @@ export * as RoleUtils from "./roles"
 export * as Utils from "./utils"
 export * as RowUtils from "./rows"
 export * as search from "./searchFields"
+export * as SchemaUtils from "./schema"
 export { memo, derivedMemo } from "./memo"
 export { createWebsocket } from "./websocket"
 export * from "./download"
diff --git a/packages/frontend-core/src/utils/schema.js b/packages/frontend-core/src/utils/schema.js
new file mode 100644
index 00000000000..7185b30d109
--- /dev/null
+++ b/packages/frontend-core/src/utils/schema.js
@@ -0,0 +1,20 @@
+import { helpers } from "@budibase/shared-core"
+import { TypeIconMap } from "../constants"
+
+export const getColumnIcon = column => {
+  if (column.schema.autocolumn) {
+    return "MagicWand"
+  }
+
+  if (helpers.schema.isDeprecatedSingleUserColumn(column.schema)) {
+    return "User"
+  }
+
+  const { type, subtype } = column.schema
+  const result =
+    typeof TypeIconMap[type] === "object" && subtype
+      ? TypeIconMap[type][subtype]
+      : TypeIconMap[type]
+
+  return result || "Text"
+}

From f54d1985b44a03e705b13f6ce5832852dec3eddb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 21 Aug 2024 07:46:15 +0100
Subject: [PATCH 010/455] Add new detail popover and update manage access
 button to use it

---
 packages/bbui/src/Popover/Popover.svelte      |  3 +
 .../buttons/ManageAccessButton.svelte         | 25 +++---
 .../DataTable/modals/ManageAccessModal.svelte | 82 ++++++-------------
 .../components/common/DetailPopover.svelte    | 59 +++++++++++++
 .../src/components/grid/stores/datasource.js  |  1 -
 5 files changed, 103 insertions(+), 67 deletions(-)
 create mode 100644 packages/builder/src/components/common/DetailPopover.svelte

diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte
index 696e246d28b..6c6ae6817dd 100644
--- a/packages/bbui/src/Popover/Popover.svelte
+++ b/packages/bbui/src/Popover/Popover.svelte
@@ -143,6 +143,9 @@
     border-color: var(--spectrum-global-color-gray-300);
     overflow: auto;
     transition: opacity 260ms ease-out;
+    filter: none;
+    -webkit-filter: none;
+    box-shadow: 0 1px 4px var(--drop-shadow);
   }
   .blockPointerEvents {
     pointer-events: none;
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index aa8eefe89c9..e43cc339151 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -2,22 +2,25 @@
   import { ActionButton, Modal } from "@budibase/bbui"
   import { permissions } from "stores/builder"
   import ManageAccessModal from "../modals/ManageAccessModal.svelte"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   export let resourceId
-  export let disabled = false
 
-  let modal
   let resourcePermissions
 
-  async function openModal() {
-    resourcePermissions = await permissions.forResourceDetailed(resourceId)
-    modal.show()
+  $: fetchPermissions(resourceId)
+
+  const fetchPermissions = async id => {
+    console.log("getting perms for", id)
+    resourcePermissions = await permissions.forResourceDetailed(id)
   }
 </script>
 
-<ActionButton icon="LockClosed" quiet on:click={openModal} {disabled}>
-  Access
-</ActionButton>
-<Modal bind:this={modal}>
-  <ManageAccessModal {resourceId} permissions={resourcePermissions} />
-</Modal>
+<DetailPopover title="Manage access">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton icon="LockClosed" selected={open} quiet>Access</ActionButton>
+  </svelte:fragment>
+  {#if resourcePermissions}
+    <ManageAccessModal {resourceId} permissions={resourcePermissions} />
+  {/if}
+</DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
index 48b584690ea..b80dad43218 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
@@ -1,19 +1,8 @@
 <script>
   import { PermissionSource } from "@budibase/types"
   import { roles, permissions as permissionsStore } from "stores/builder"
-  import {
-    Label,
-    Input,
-    Select,
-    notifications,
-    Body,
-    ModalContent,
-    Tags,
-    Tag,
-    Icon,
-  } from "@budibase/bbui"
+  import { Label, Input, Select, notifications, Body } from "@budibase/bbui"
   import { capitalise } from "helpers"
-  import { getFormattedPlanName } from "helpers/planTitle"
   import { get } from "svelte/store"
 
   export let resourceId
@@ -69,8 +58,6 @@
     {}
   )
 
-  $: requiresPlanToModify = permissions.requiresPlanToModify
-
   let dependantsInfoMessage
   async function loadDependantInfo() {
     const dependantsInfo = await permissionsStore.getDependantsInfo(resourceId)
@@ -94,48 +81,33 @@
   loadDependantInfo()
 </script>
 
-<ModalContent showCancelButton={false} confirmText="Done">
-  <span slot="header">
-    Manage Access
-    {#if requiresPlanToModify}
-      <span class="lock-tag">
-        <Tags>
-          <Tag icon="LockClosed"
-            >{getFormattedPlanName(requiresPlanToModify)}</Tag
-          >
-        </Tags>
-      </span>
-    {/if}
-  </span>
-  <Body size="S">Specify the minimum access level role for this data.</Body>
-  <div class="row">
-    <Label extraSmall grey>Level</Label>
-    <Label extraSmall grey>Role</Label>
-    {#each Object.keys(computedPermissions) as level}
-      <Input value={capitalise(level)} disabled />
-      <Select
-        disabled={requiresPlanToModify}
-        placeholder={false}
-        value={computedPermissions[level].selectedValue}
-        on:change={e => changePermission(level, e.detail)}
-        options={computedPermissions[level].options}
-        getOptionLabel={x => x.name}
-        getOptionValue={x => x._id}
-      />
-    {/each}
+<Body size="S">Specify the minimum access level role for this data.</Body>
+<div class="row">
+  <Label extraSmall grey>Level</Label>
+  <Label extraSmall grey>Role</Label>
+  {#each Object.keys(computedPermissions) as level}
+    <Input value={capitalise(level)} disabled />
+    <Select
+      placeholder={false}
+      value={computedPermissions[level].selectedValue}
+      on:change={e => changePermission(level, e.detail)}
+      options={computedPermissions[level].options}
+      getOptionLabel={x => x.name}
+      getOptionValue={x => x._id}
+    />
+  {/each}
+</div>
+
+{#if dependantsInfoMessage}
+  <div class="inheriting-resources">
+    <Icon name="Alert" />
+    <Body size="S">
+      <i>
+        {dependantsInfoMessage}
+      </i>
+    </Body>
   </div>
-
-  {#if dependantsInfoMessage}
-    <div class="inheriting-resources">
-      <Icon name="Alert" />
-      <Body size="S">
-        <i>
-          {dependantsInfoMessage}
-        </i>
-      </Body>
-    </div>
-  {/if}
-</ModalContent>
+{/if}
 
 <style>
   .row {
diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
new file mode 100644
index 00000000000..d2d3c431c24
--- /dev/null
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -0,0 +1,59 @@
+<script>
+  import { Popover, Icon } from "@budibase/bbui"
+
+  export let title
+  export let align = "left"
+
+  let popover
+  let anchor
+  let open
+
+  export const show = () => popover?.show()
+  export const hide = () => popover?.hide()
+</script>
+
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<div class="anchor" bind:this={anchor} on:click={show}>
+  <slot name="anchor" {open} />
+</div>
+
+<Popover bind:this={popover} bind:open {anchor} {align} minWidth={300}>
+  <div class="detail-popover">
+    <div class="detail-popover__header">
+      <div class="detail-popover__title">
+        {title}
+      </div>
+      <Icon name="Close" />
+    </div>
+    <div class="detail-popover__body">
+      <slot />
+    </div>
+  </div>
+</Popover>
+
+<style>
+  .detail-popover {
+    --padding: 12px;
+    background-color: var(--spectrum-alias-background-color-primary);
+  }
+  .detail-popover__header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    border-bottom: 1px solid var(--spectrum-global-color-gray-300);
+    padding: var(--padding);
+  }
+  .detail-popover__title {
+    font-size: 16px;
+    font-weight: 600;
+  }
+  .detail-popover__body {
+    padding: var(--padding);
+    display: flex;
+    flex-direction: column;
+    align-items: stretch;
+    gap: var(--padding);
+  }
+</style>
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 6f0a8bb1688..12a3bd5afe6 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -152,7 +152,6 @@ export const createActions = context => {
       return
     }
     schemaMutations.update($schemaMutations => {
-      console.log($schemaMutations)
       return {
         ...$schemaMutations,
         [field]: {

From 1f7b10cfd1afc2052dbef2c539b19f7f63081f3d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 21 Aug 2024 09:04:49 +0100
Subject: [PATCH 011/455] Update style of generate automation button

---
 ...utton.svelte => GridGenerateButton.svelte} | 35 ++++---------------
 .../grid/GridUsersTableButton.svelte          |  0
 .../DataTable/modals/ManageAccessModal.svelte |  9 ++++-
 .../table/[tableId]/[viewId]/index.svelte     |  8 +++--
 .../data/table/[tableId]/index.svelte         | 12 +++----
 .../src/components/grid/layout/Grid.svelte    |  1 +
 6 files changed, 27 insertions(+), 38 deletions(-)
 rename packages/builder/src/components/backend/DataTable/buttons/grid/{GridCreateAutomationButton.svelte => GridGenerateButton.svelte} (83%)
 rename packages/builder/src/components/backend/DataTable/{modals => buttons}/grid/GridUsersTableButton.svelte (100%)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
similarity index 83%
rename from packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index 148db7554ce..907213e97e6 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -1,20 +1,14 @@
 <script>
-  import {
-    ActionButton,
-    Popover,
-    Menu,
-    MenuItem,
-    notifications,
-  } from "@budibase/bbui"
+  import { ActionButton, Menu, MenuItem, notifications } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { automationStore, tables, builderStore } from "stores/builder"
   import { TriggerStepID } from "constants/backend/automations"
   import { goto } from "@roxi/routify"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { datasource } = getContext("grid")
 
   $: triggers = $automationStore.blockDefinitions.CREATABLE_TRIGGER
-
   $: table = $tables.list.find(table => table._id === $datasource.tableId)
 
   async function createAutomation(type) {
@@ -57,24 +51,12 @@
       notifications.error("Error creating automation")
     }
   }
-
-  let anchor
-  let open
 </script>
 
-<div bind:this={anchor}>
-  <ActionButton
-    icon="MagicWand"
-    quiet
-    size="M"
-    on:click={() => (open = !open)}
-    selected={open}
-  >
-    Generate
-  </ActionButton>
-</div>
-
-<Popover bind:open {anchor} align="left">
+<DetailPopover title="Generate">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton icon="MagicWand" selected={open}>Generate</ActionButton>
+  </svelte:fragment>
   <Menu>
     <MenuItem
       icon="ShareAndroid"
@@ -95,7 +77,4 @@
       Automation: when row is updated
     </MenuItem>
   </Menu>
-</Popover>
-
-<style>
-</style>
+</DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte
similarity index 100%
rename from packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte
rename to packages/builder/src/components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte
diff --git a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
index b80dad43218..d26078f47e5 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
@@ -1,7 +1,14 @@
 <script>
   import { PermissionSource } from "@budibase/types"
   import { roles, permissions as permissionsStore } from "stores/builder"
-  import { Label, Input, Select, notifications, Body } from "@budibase/bbui"
+  import {
+    Label,
+    Input,
+    Select,
+    notifications,
+    Body,
+    Icon,
+  } from "@budibase/bbui"
   import { capitalise } from "helpers"
   import { get } from "svelte/store"
 
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index fb784334812..a2f490e4eb8 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -9,7 +9,7 @@
   import GridSortButton from "components/backend/DataTable/buttons/grid/GridSortButton.svelte"
   import GridColumnsSettingButton from "components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte"
   import GridSizeButton from "components/backend/DataTable/buttons/grid/GridSizeButton.svelte"
-  import GridCreateAutomationButton from "components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte"
+  import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
 
   $: id = $viewsV2.selected?.id
   $: datasource = {
@@ -36,10 +36,12 @@
   <svelte:fragment slot="controls">
     <GridFilterButton />
     <GridSortButton />
+    <GridSizeButton />
     <GridColumnsSettingButton />
     <GridManageAccessButton />
-    <GridSizeButton />
-    <GridCreateAutomationButton />
+  </svelte:fragment>
+  <svelte:fragment slot="controls-right">
+    <GridGenerateButton />
   </svelte:fragment>
   <GridCreateEditRowModal />
 </Grid>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 5b5132255b2..72f4e87fadc 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -2,11 +2,9 @@
   import { Banner } from "@budibase/bbui"
   import { datasources, tables, integrations, appStore } from "stores/builder"
   import { themeStore, admin } from "stores/portal"
-  import EditRolesButton from "components/backend/DataTable/buttons/EditRolesButton.svelte"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
-  import GridCreateAutomationButton from "components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte"
   import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
@@ -15,7 +13,8 @@
   import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
   import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
-  import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
+  import GridUsersTableButton from "components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte"
+  import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
   import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   const userSchemaOverrides = {
@@ -109,9 +108,6 @@
         <GridUsersTableButton />
       {/if}
       <GridManageAccessButton />
-      {#if !isUsersTable}
-        <GridCreateAutomationButton />
-      {/if}
       {#if relationshipsEnabled}
         <GridRelationshipButton />
       {/if}
@@ -121,6 +117,10 @@
       <GridExportButton />
     </svelte:fragment>
 
+    <svelte:fragment slot="controls-right">
+      <GridGenerateButton />
+    </svelte:fragment>
+
     <!-- Content for editing columns -->
     <svelte:fragment slot="edit-column">
       <GridEditColumnModal />
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 9b9abfbabd3..f6d36e348e4 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -147,6 +147,7 @@
         <slot name="controls" />
       </div>
       <div class="controls-right">
+        <slot name="controls-right" />
         {#if showAvatars}
           <UserAvatars />
         {/if}

From 3aca23e19831c916d82bf13b4c95cbee2d956a4c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 21 Aug 2024 09:47:52 +0100
Subject: [PATCH 012/455] Add initial work on screens button

---
 .../buttons/grid/GridAutomationsButton.svelte | 12 ++++++++
 .../grid/GridManageAccessButton.svelte        | 10 ++-----
 .../buttons/grid/GridRowActionsButton.svelte  | 12 ++++++++
 .../buttons/grid/GridScreensButton.svelte     | 29 +++++++++++++++++++
 .../components/common/DetailPopover.svelte    |  2 +-
 .../table/[tableId]/[viewId]/index.svelte     |  6 ++++
 .../data/table/[tableId]/index.svelte         |  4 +++
 7 files changed, 66 insertions(+), 9 deletions(-)
 create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
 create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
 create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
new file mode 100644
index 00000000000..d89bab0fc29
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -0,0 +1,12 @@
+<script>
+  import { ActionButton } from "@budibase/bbui"
+  import DetailPopover from "components/common/DetailPopover.svelte"
+</script>
+
+<DetailPopover title="Automations">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton icon="JourneyVoyager" selected={open} quiet>
+      Automations
+    </ActionButton>
+  </svelte:fragment>
+</DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte
index 0cd008bab14..c5bc9663324 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte
@@ -4,14 +4,8 @@
 
   const { datasource } = getContext("grid")
 
-  $: resourceId = getResourceID($datasource)
-
-  const getResourceID = datasource => {
-    if (!datasource) {
-      return null
-    }
-    return datasource.type === "table" ? datasource.tableId : datasource.id
-  }
+  $: ds = $datasource
+  $: resourceId = ds?.type === "table" ? ds.tableId : ds?.id
 </script>
 
 <ManageAccessButton {resourceId} />
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
new file mode 100644
index 00000000000..8033f3b3d61
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -0,0 +1,12 @@
+<script>
+  import { ActionButton } from "@budibase/bbui"
+  import DetailPopover from "components/common/DetailPopover.svelte"
+</script>
+
+<DetailPopover title="Row Actions">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton icon="Engagement" selected={open} quiet>
+      Row Actions
+    </ActionButton>
+  </svelte:fragment>
+</DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
new file mode 100644
index 00000000000..ac2891a1df4
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -0,0 +1,29 @@
+<script>
+  import { ActionButton } from "@budibase/bbui"
+  import DetailPopover from "components/common/DetailPopover.svelte"
+  import { screenStore } from "stores/builder"
+  import { getContext } from "svelte"
+
+  const { datasource } = getContext("grid")
+
+  $: ds = $datasource
+  $: resourceId = ds?.type === "table" ? ds.tableId : ds?.id
+  $: connectedScreens = findConnectedScreens($screenStore.screens, resourceId)
+
+  const findConnectedScreens = (screens, resourceId) => {
+    console.log(resourceId)
+    return screens.filter(screen => {
+      return JSON.stringify(screen).includes(`"${resourceId}"`)
+    })
+  }
+
+  $: console.log(connectedScreens)
+</script>
+
+<DetailPopover title="Screens">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton icon="WebPage" selected={open} quiet>Screens</ActionButton>
+  </svelte:fragment>
+  The following screens are connected to this data:
+  {connectedScreens.map(screen => screen.routing.route)}
+</DetailPopover>
diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index d2d3c431c24..b3d856e1ea9 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -34,7 +34,7 @@
 
 <style>
   .detail-popover {
-    --padding: 12px;
+    --padding: var(--spacing-l);
     background-color: var(--spectrum-alias-background-color-primary);
   }
   .detail-popover__header {
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index a2f490e4eb8..e8adc5e0313 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -10,6 +10,9 @@
   import GridColumnsSettingButton from "components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte"
   import GridSizeButton from "components/backend/DataTable/buttons/grid/GridSizeButton.svelte"
   import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
+  import GridScreensButton from "components/backend/DataTable/buttons/grid/GridScreensButton.svelte"
+  import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
+  import GridAutomationsButton from "components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte"
 
   $: id = $viewsV2.selected?.id
   $: datasource = {
@@ -39,6 +42,9 @@
     <GridSizeButton />
     <GridColumnsSettingButton />
     <GridManageAccessButton />
+    <GridRowActionsButton />
+    <GridAutomationsButton />
+    <GridScreensButton />
   </svelte:fragment>
   <svelte:fragment slot="controls-right">
     <GridGenerateButton />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 72f4e87fadc..2526caa203a 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -15,6 +15,8 @@
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
   import GridUsersTableButton from "components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte"
   import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
+  import GridScreensButton from "components/backend/DataTable/buttons/grid/GridScreensButton.svelte"
+  import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
   import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   const userSchemaOverrides = {
@@ -111,6 +113,8 @@
       {#if relationshipsEnabled}
         <GridRelationshipButton />
       {/if}
+      <GridRowActionsButton />
+      <GridScreensButton />
       {#if !isUsersTable}
         <GridImportButton />
       {/if}

From c7c659742401c85476f0a910229e783e8b23046e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 21 Aug 2024 16:33:51 +0100
Subject: [PATCH 013/455] Add majority of frontend implementation of row
 actions

---
 packages/bbui/src/Form/Core/Switch.svelte     |   1 +
 packages/bbui/src/List/ListItem.svelte        |  75 +++++++------
 .../buttons/ManageAccessButton.svelte         |   3 +-
 .../buttons/grid/GridAutomationsButton.svelte |  66 ++++++++++-
 .../buttons/grid/GridGenerateButton.svelte    |   7 +-
 .../buttons/grid/GridRowActionsButton.svelte  | 104 +++++++++++++++++-
 .../buttons/grid/GridScreensButton.svelte     |  21 +++-
 .../components/common/DetailPopover.svelte    |  18 ++-
 .../table/[tableId]/[viewId]/index.svelte     |   1 -
 .../data/table/[tableId]/index.svelte         |  12 +-
 packages/frontend-core/src/api/index.js       |   2 +
 packages/frontend-core/src/api/rowActions.js  |  63 +++++++++++
 12 files changed, 311 insertions(+), 62 deletions(-)
 create mode 100644 packages/frontend-core/src/api/rowActions.js

diff --git a/packages/bbui/src/Form/Core/Switch.svelte b/packages/bbui/src/Form/Core/Switch.svelte
index deffc19167b..d7110a6e674 100644
--- a/packages/bbui/src/Form/Core/Switch.svelte
+++ b/packages/bbui/src/Form/Core/Switch.svelte
@@ -19,6 +19,7 @@
     {disabled}
     on:change={onChange}
     on:click
+    on:click|stopPropagation
     {id}
     type="checkbox"
     class="spectrum-Switch-input"
diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 76b242cf9c2..0c9b796f5b9 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -1,55 +1,50 @@
 <script>
-  import Body from "../Typography/Body.svelte"
-  import IconAvatar from "../Icon/IconAvatar.svelte"
-  import Label from "../Label/Label.svelte"
-  import Avatar from "../Avatar/Avatar.svelte"
+  import Icon from "../Icon/Icon.svelte"
 
   export let icon = null
-  export let iconBackground = null
   export let iconColor = null
-  export let avatar = false
   export let title = null
   export let subtitle = null
-  export let hoverable = false
-
-  $: initials = avatar ? title?.[0] : null
+  export let url = null
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
-<div class="list-item" class:hoverable on:click>
+<a href={url} class="list-item" class:hoverable={url != null} on:click>
   <div class="left">
     {#if icon}
-      <IconAvatar {icon} color={iconColor} background={iconBackground} />
-    {/if}
-    {#if avatar}
-      <Avatar {initials} />
-    {/if}
-    {#if title}
-      <Body>{title}</Body>
-    {/if}
-    {#if subtitle}
-      <Label>{subtitle}</Label>
+      <Icon name={icon} color={iconColor} />
     {/if}
-  </div>
-  {#if $$slots.default}
-    <div class="right">
-      <slot />
+    <div class="list-item__text">
+      {#if title}
+        <div class="list-item__title">
+          {title}
+        </div>
+      {/if}
+      {#if subtitle}
+        <div class="list-item__subtitle">
+          {subtitle}
+        </div>
+      {/if}
     </div>
-  {/if}
-</div>
+  </div>
+  <div class="right">
+    <slot name="right" />
+    <Icon name="ChevronRight" />
+  </div>
+</a>
 
 <style>
   .list-item {
-    padding: 0 16px;
-    height: 56px;
-    background: var(--spectrum-global-color-gray-50);
+    padding: var(--spacing-m);
+    background: var(--spectrum-global-color-gray-75);
     display: flex;
     flex-direction: row;
     justify-content: space-between;
     border: 1px solid var(--spectrum-global-color-gray-300);
     transition: background 130ms ease-out;
     gap: var(--spacing-m);
+    color: var(--spectrum-global-color-gray-800);
   }
   .list-item:not(:first-child) {
     border-top: none;
@@ -64,14 +59,15 @@
   }
   .hoverable:hover {
     cursor: pointer;
-    background: var(--spectrum-global-color-gray-75);
+    background: var(--spectrum-global-color-gray-200);
   }
+
   .left,
   .right {
     display: flex;
     flex-direction: row;
     align-items: center;
-    gap: var(--spacing-xl);
+    gap: var(--spacing-l);
   }
   .left {
     width: 0;
@@ -79,17 +75,20 @@
   }
   .right {
     flex: 0 0 auto;
+    color: var(--spectrum-global-color-gray-600);
   }
-  .list-item :global(.spectrum-Icon),
-  .list-item :global(.spectrum-Avatar) {
-    flex: 0 0 auto;
-  }
-  .list-item :global(.spectrum-Body) {
-    color: var(--spectrum-global-color-gray-900);
+
+  .list-item__text {
+    flex: 1 1 auto;
+    width: 0;
   }
-  .list-item :global(.spectrum-Body) {
+  .list-item__title,
+  .list-item__subtitle {
     white-space: nowrap;
     overflow: hidden;
     text-overflow: ellipsis;
   }
+  .list-item__subtitle {
+    color: var(--spectrum-global-color-gray-600);
+  }
 </style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index e43cc339151..de868c60e03 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { ActionButton, Modal } from "@budibase/bbui"
+  import { ActionButton } from "@budibase/bbui"
   import { permissions } from "stores/builder"
   import ManageAccessModal from "../modals/ManageAccessModal.svelte"
   import DetailPopover from "components/common/DetailPopover.svelte"
@@ -11,7 +11,6 @@
   $: fetchPermissions(resourceId)
 
   const fetchPermissions = async id => {
-    console.log("getting perms for", id)
     resourcePermissions = await permissions.forResourceDetailed(id)
   }
 </script>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index d89bab0fc29..5216fb5ebbe 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -1,12 +1,68 @@
 <script>
-  import { ActionButton } from "@budibase/bbui"
+  import { ActionButton, List, ListItem, Button } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
+  import { TriggerStepID } from "constants/backend/automations"
+  import { automationStore, appStore } from "stores/builder"
+  import { createEventDispatcher, getContext } from "svelte"
+
+  const dispatch = createEventDispatcher()
+  const { datasource } = getContext("grid")
+  const triggerTypes = [
+    TriggerStepID.ROW_SAVED,
+    TriggerStepID.ROW_UPDATED,
+    TriggerStepID.ROW_DELETED,
+  ]
+
+  let popover
+
+  $: ds = $datasource
+  $: resourceId = ds?.type === "table" ? ds.tableId : ds?.id
+  $: connectedAutomations = findConnectedAutomations(
+    $automationStore.automations,
+    resourceId
+  )
+
+  const findConnectedAutomations = (automations, resourceId) => {
+    return automations.filter(automation => {
+      if (!triggerTypes.includes(automation.definition?.trigger?.stepId)) {
+        return false
+      }
+      return automation.definition?.trigger?.inputs?.tableId === resourceId
+    })
+  }
+
+  const generateAutomation = () => {
+    popover?.hide()
+    dispatch("generate-automation")
+  }
 </script>
 
-<DetailPopover title="Automations">
+<DetailPopover title="Automations" minWidth={400} bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton icon="JourneyVoyager" selected={open} quiet>
-      Automations
-    </ActionButton>
+    <ActionButton icon="JourneyVoyager" selected={open} quiet
+      >Automations</ActionButton
+    >
   </svelte:fragment>
+  {#if !connectedAutomations.length}
+    There aren't any automations connected to this data.
+  {:else}
+    The following automations are connected to this data.
+    <List>
+      {#each connectedAutomations as automation}
+        <ListItem
+          icon={automation.disabled ? "PauseCircle" : "PlayCircle"}
+          iconColor={automation.disabled
+            ? "var(--spectrum-global-color-gray-600)"
+            : "var(--spectrum-global-color-green-600)"}
+          title={automation.name}
+          url={`/builder/app/${$appStore.appId}/automation/${automation._id}`}
+        />
+      {/each}
+    </List>
+  {/if}
+  <div>
+    <Button secondary icon="JourneyVoyager" on:click={generateAutomation}>
+      Generate new automation
+    </Button>
+  </div>
 </DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index 907213e97e6..454c70b6071 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -8,9 +8,14 @@
 
   const { datasource } = getContext("grid")
 
+  let popover
+
   $: triggers = $automationStore.blockDefinitions.CREATABLE_TRIGGER
   $: table = $tables.list.find(table => table._id === $datasource.tableId)
 
+  export const show = () => popover?.show()
+  export const hide = () => popover?.hide()
+
   async function createAutomation(type) {
     const triggerType = triggers[type]
     if (!triggerType) {
@@ -53,7 +58,7 @@
   }
 </script>
 
-<DetailPopover title="Generate">
+<DetailPopover title="Generate" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="MagicWand" selected={open}>Generate</ActionButton>
   </svelte:fragment>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 8033f3b3d61..bd50b403f4f 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -1,12 +1,110 @@
 <script>
-  import { ActionButton } from "@budibase/bbui"
+  import {
+    ActionButton,
+    List,
+    ListItem,
+    Button,
+    Toggle,
+    notifications,
+  } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
+  import { getContext } from "svelte"
+  import { appStore, automationStore } from "stores/builder"
+  import { API } from "api"
+  import { goto, url } from "@roxi/routify"
+  import { derived } from "svelte/store"
+  import { getSequentialName } from "helpers/duplicate"
+
+  const { datasource } = getContext("grid")
+
+  let rowActions = []
+
+  $: ds = $datasource
+  $: tableId = ds?.tableId
+  $: isView = ds?.type === "viewV2"
+  $: fetchRowActions(tableId)
+  $: console.log(rowActions)
+  $: activeCount = 0
+  $: suffix = isView ? activeCount : rowActions.length
+
+  const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
+    return ({ automationId }) => {
+      return $url(`/builder/app/${$appStore.appId}/automation/${automationId}`)
+    }
+  })
+
+  const fetchRowActions = async tableId => {
+    if (!tableId) {
+      rowActions = []
+      return
+    }
+    const res = await API.rowActions.fetch(tableId)
+    rowActions = Object.values(res || {})
+  }
+
+  const createRowAction = async () => {
+    try {
+      const name = getSequentialName(rowActions, "New row action ", {
+        getName: x => x.name,
+      })
+      const res = await API.rowActions.create({
+        name,
+        tableId,
+      })
+      console.log(res)
+      await automationStore.actions.fetch()
+      notifications.success("Row action created successfully")
+      $goto($rowActionUrl(res))
+    } catch (error) {
+      console.log(error)
+    }
+  }
 </script>
 
-<DetailPopover title="Row Actions">
+<DetailPopover title="Row Actions" minWidth={400} maxWidth={400}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="Engagement" selected={open} quiet>
-      Row Actions
+      Row Actions ({suffix})
     </ActionButton>
   </svelte:fragment>
+  A row action is a user-triggered automation for a chosen row.
+  {#if isView && rowActions.length}
+    <br />
+    Use the toggle to enable/disable row actions for this view.
+    <br />
+  {/if}
+  {#if !rowActions.length}
+    <br />
+    You haven't created any row actions.
+  {:else}
+    <List>
+      {#each rowActions as action}
+        <ListItem title={action.name} url={$rowActionUrl(action)}>
+          <svelte:fragment slot="right">
+            {#if isView}
+              <span>
+                <Toggle />
+              </span>
+            {/if}
+          </svelte:fragment>
+        </ListItem>
+      {/each}
+    </List>
+  {/if}
+  <div>
+    <Button secondary icon="Engagement" on:click={createRowAction}>
+      Create row action
+    </Button>
+  </div>
 </DetailPopover>
+
+<style>
+  span :global(.spectrum-Switch) {
+    min-height: 0;
+    margin-right: 0;
+  }
+  span :global(.spectrum-Switch-switch) {
+    margin-bottom: 0;
+    margin-top: 2px;
+  }
+</style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index ac2891a1df4..c2c77216644 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -1,7 +1,7 @@
 <script>
-  import { ActionButton } from "@budibase/bbui"
+  import { ActionButton, List, ListItem } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
-  import { screenStore } from "stores/builder"
+  import { screenStore, appStore } from "stores/builder"
   import { getContext } from "svelte"
 
   const { datasource } = getContext("grid")
@@ -20,10 +20,21 @@
   $: console.log(connectedScreens)
 </script>
 
-<DetailPopover title="Screens">
+<DetailPopover title="Screens" minWidth={400}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="WebPage" selected={open} quiet>Screens</ActionButton>
   </svelte:fragment>
-  The following screens are connected to this data:
-  {connectedScreens.map(screen => screen.routing.route)}
+  {#if !connectedScreens.length}
+    There aren't any screens connected to this data.
+  {:else}
+    The following screens are connected to this data.
+    <List>
+      {#each connectedScreens as screen}
+        <ListItem
+          title={screen.routing.route}
+          url={`/builder/app/${$appStore.appId}/design/${screen._id}`}
+        />
+      {/each}
+    </List>
+  {/if}
 </DetailPopover>
diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index b3d856e1ea9..223814130b5 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -3,6 +3,8 @@
 
   export let title
   export let align = "left"
+  export let minWidth
+  export let maxWidth
 
   let popover
   let anchor
@@ -18,7 +20,14 @@
   <slot name="anchor" {open} />
 </div>
 
-<Popover bind:this={popover} bind:open {anchor} {align} minWidth={300}>
+<Popover
+  bind:this={popover}
+  bind:open
+  {anchor}
+  {align}
+  minWidth={minWidth || 300}
+  {maxWidth}
+>
   <div class="detail-popover">
     <div class="detail-popover__header">
       <div class="detail-popover__title">
@@ -34,7 +43,6 @@
 
 <style>
   .detail-popover {
-    --padding: var(--spacing-l);
     background-color: var(--spectrum-alias-background-color-primary);
   }
   .detail-popover__header {
@@ -43,17 +51,17 @@
     justify-content: space-between;
     align-items: center;
     border-bottom: 1px solid var(--spectrum-global-color-gray-300);
-    padding: var(--padding);
+    padding: var(--spacing-l) var(--spacing-xl);
   }
   .detail-popover__title {
     font-size: 16px;
     font-weight: 600;
   }
   .detail-popover__body {
-    padding: var(--padding);
+    padding: var(--spacing-xl) var(--spacing-xl);
     display: flex;
     flex-direction: column;
     align-items: stretch;
-    gap: var(--padding);
+    gap: var(--spacing-xl);
   }
 </style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index e8adc5e0313..88537ccd0ef 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -43,7 +43,6 @@
     <GridColumnsSettingButton />
     <GridManageAccessButton />
     <GridRowActionsButton />
-    <GridAutomationsButton />
     <GridScreensButton />
   </svelte:fragment>
   <svelte:fragment slot="controls-right">
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 2526caa203a..6c267c27339 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -16,6 +16,7 @@
   import GridUsersTableButton from "components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte"
   import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
   import GridScreensButton from "components/backend/DataTable/buttons/grid/GridScreensButton.svelte"
+  import GridAutomationsButton from "components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte"
   import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
   import { DB_TYPE_EXTERNAL } from "constants/backend"
 
@@ -27,6 +28,8 @@
     status: { displayName: "Status", disabled: true },
   }
 
+  let generateButton
+
   $: autoColumnStatus = verifyAutocolumns($tables?.selected)
   $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
     if (status.length > 1) {
@@ -113,7 +116,12 @@
       {#if relationshipsEnabled}
         <GridRelationshipButton />
       {/if}
-      <GridRowActionsButton />
+      {#if !isUsersTable}
+        <GridRowActionsButton />
+      {/if}
+      <GridAutomationsButton
+        on:generate-automation={() => generateButton?.show()}
+      />
       <GridScreensButton />
       {#if !isUsersTable}
         <GridImportButton />
@@ -122,7 +130,7 @@
     </svelte:fragment>
 
     <svelte:fragment slot="controls-right">
-      <GridGenerateButton />
+      <GridGenerateButton bind:this={generateButton} />
     </svelte:fragment>
 
     <!-- Content for editing columns -->
diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js
index 066ab16f6e8..9558ca36cbe 100644
--- a/packages/frontend-core/src/api/index.js
+++ b/packages/frontend-core/src/api/index.js
@@ -34,6 +34,7 @@ import { buildEventEndpoints } from "./events"
 import { buildAuditLogsEndpoints } from "./auditLogs"
 import { buildLogsEndpoints } from "./logs"
 import { buildMigrationEndpoints } from "./migrations"
+import { buildRowActionEndpoints } from "./rowActions"
 
 /**
  * Random identifier to uniquely identify a session in a tab. This is
@@ -301,5 +302,6 @@ export const createAPIClient = config => {
     ...buildLogsEndpoints(API),
     ...buildMigrationEndpoints(API),
     viewV2: buildViewV2Endpoints(API),
+    rowActions: buildRowActionEndpoints(API),
   }
 }
diff --git a/packages/frontend-core/src/api/rowActions.js b/packages/frontend-core/src/api/rowActions.js
new file mode 100644
index 00000000000..cf8808f6d00
--- /dev/null
+++ b/packages/frontend-core/src/api/rowActions.js
@@ -0,0 +1,63 @@
+export const buildRowActionEndpoints = API => ({
+  /**
+   * Gets the available row actions for a table.
+   * @param tableId the ID of the table
+   */
+  fetch: async tableId => {
+    const res = await API.get({
+      url: `/api/tables/${tableId}/actions`,
+    })
+    return res?.actions || {}
+  },
+
+  /**
+   * Creates a row action.
+   * @param name the name of the row action
+   * @param tableId the ID of the table
+   */
+  create: async ({ name, tableId }) => {
+    return await API.post({
+      url: `/api/tables/${tableId}/actions`,
+      body: {
+        name,
+      },
+    })
+  },
+
+  /**
+   * Updates a row action.
+   * @param name the new name of the row action
+   * @param tableId the ID of the table
+   * @param rowActionId the ID of the row action to update
+   */
+  update: async ({ tableId, rowActionId, name }) => {
+    return await API.post({
+      url: `/api/tables/${tableId}/actions/${rowActionId}`,
+      body: {
+        name,
+      },
+    })
+  },
+
+  /**
+   * Deletes a row action.
+   * @param tableId the ID of the table
+   * @param rowActionId the ID of the row action to delete
+   */
+  delete: async ({ tableId, rowActionId }) => {
+    return await API.delete({
+      url: `/api/tables/${tableId}/actions/${rowActionId}`,
+    })
+  },
+
+  /**
+   * Triggers a row action.
+   * @param tableId the ID of the table
+   * @param rowActionId the ID of the row action to trigger
+   */
+  trigger: async ({ tableId, rowActionid }) => {
+    return await API.post({
+      url: `/api/tables/${tableId}/actions/${rowActionId}/trigger`,
+    })
+  },
+})

From 7431d1c8daf80c050241e90954be16e4c4cc5b27 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 08:06:24 +0100
Subject: [PATCH 014/455] Make detail popover icon function

---
 .../builder/src/components/common/DetailPopover.svelte    | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index 223814130b5..67d2e0a6eb0 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -33,7 +33,13 @@
       <div class="detail-popover__title">
         {title}
       </div>
-      <Icon name="Close" />
+      <Icon
+        name="Close"
+        hoverable
+        color="var(--spectrum-global-color-gray-600)"
+        hoverColor="var(--spectum-global-color-gray-900)"
+        on:click={hide}
+      />
     </div>
     <div class="detail-popover__body">
       <slot />

From acb3893730f61644424858c5db89f89f844b69be Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 08:06:38 +0100
Subject: [PATCH 015/455] Fix styles in generate button and hide generate
 button for users table

---
 .../buttons/grid/GridGenerateButton.svelte    | 51 +++++++++++--------
 .../data/table/[tableId]/index.svelte         |  5 +-
 2 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index 454c70b6071..a7f1cf57c53 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -62,24 +62,35 @@
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="MagicWand" selected={open}>Generate</ActionButton>
   </svelte:fragment>
-  <Menu>
-    <MenuItem
-      icon="ShareAndroid"
-      on:click={() => {
-        open = false
-        createAutomation(TriggerStepID.ROW_SAVED)
-      }}
-    >
-      Automation: when row is created
-    </MenuItem>
-    <MenuItem
-      icon="ShareAndroid"
-      on:click={() => {
-        open = false
-        createAutomation(TriggerStepID.ROW_UPDATED)
-      }}
-    >
-      Automation: when row is updated
-    </MenuItem>
-  </Menu>
+  <div class="menu">
+    <Menu>
+      <MenuItem
+        icon="ShareAndroid"
+        on:click={() => {
+          open = false
+          createAutomation(TriggerStepID.ROW_SAVED)
+        }}
+      >
+        Automation: when row is created
+      </MenuItem>
+      <MenuItem
+        icon="ShareAndroid"
+        on:click={() => {
+          open = false
+          createAutomation(TriggerStepID.ROW_UPDATED)
+        }}
+      >
+        Automation: when row is updated
+      </MenuItem>
+    </Menu>
+  </div>
 </DetailPopover>
+
+<style>
+  .menu {
+    margin: 0 calc(-1 * var(--spacing-xl));
+    display: flex;
+    flex-direction: column;
+    align-items: stretch;
+  }
+</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 6c267c27339..50ad390b1c7 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -128,9 +128,10 @@
       {/if}
       <GridExportButton />
     </svelte:fragment>
-
     <svelte:fragment slot="controls-right">
-      <GridGenerateButton bind:this={generateButton} />
+      {#if !isUsersTable}
+        <GridGenerateButton bind:this={generateButton} />
+      {/if}
     </svelte:fragment>
 
     <!-- Content for editing columns -->

From 8a84a796511b28e59f948bfe03b89b28037962db Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 08:38:15 +0100
Subject: [PATCH 016/455] Add role editing to access popover

---
 .../backend/DataTable/buttons/EditRolesButton.svelte | 12 +++++++-----
 .../DataTable/buttons/ManageAccessButton.svelte      |  8 +++++++-
 .../buttons/grid/GridRowActionsButton.svelte         |  5 ++---
 .../DataTable/buttons/grid/GridScreensButton.svelte  |  3 ---
 .../src/components/common/DetailPopover.svelte       |  4 +++-
 5 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
index 87ca2fa142a..3b31de170c2 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
@@ -1,13 +1,15 @@
 <script>
-  import { ActionButton, Modal } from "@budibase/bbui"
+  import { Button, Modal } from "@budibase/bbui"
   import EditRolesModal from "../modals/EditRoles.svelte"
 
   let modal
 </script>
 
-<ActionButton icon="UsersLock" quiet on:click={modal.show}>
-  Edit roles
-</ActionButton>
-<Modal bind:this={modal}>
+<div>
+  <Button secondary icon="UsersLock" on:click on:click={modal.show}>
+    Edit roles
+  </Button>
+</div>
+<Modal bind:this={modal} on:show on:hide>
   <EditRolesModal />
 </Modal>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index de868c60e03..22a55ecc03d 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -3,10 +3,12 @@
   import { permissions } from "stores/builder"
   import ManageAccessModal from "../modals/ManageAccessModal.svelte"
   import DetailPopover from "components/common/DetailPopover.svelte"
+  import EditRolesButton from "./EditRolesButton.svelte"
 
   export let resourceId
 
   let resourcePermissions
+  let showPopover = true
 
   $: fetchPermissions(resourceId)
 
@@ -15,11 +17,15 @@
   }
 </script>
 
-<DetailPopover title="Manage access">
+<DetailPopover title="Manage access" {showPopover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="LockClosed" selected={open} quiet>Access</ActionButton>
   </svelte:fragment>
   {#if resourcePermissions}
     <ManageAccessModal {resourceId} permissions={resourcePermissions} />
   {/if}
+  <EditRolesButton
+    on:show={() => (showPopover = false)}
+    on:hide={() => (showPopover = true)}
+  />
 </DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index bd50b403f4f..7753d4eb3bc 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -23,7 +23,6 @@
   $: tableId = ds?.tableId
   $: isView = ds?.type === "viewV2"
   $: fetchRowActions(tableId)
-  $: console.log(rowActions)
   $: activeCount = 0
   $: suffix = isView ? activeCount : rowActions.length
 
@@ -51,12 +50,12 @@
         name,
         tableId,
       })
-      console.log(res)
       await automationStore.actions.fetch()
       notifications.success("Row action created successfully")
       $goto($rowActionUrl(res))
     } catch (error) {
-      console.log(error)
+      console.error(error)
+      notifications.error("Error creating row action")
     }
   }
 </script>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index c2c77216644..48c9bdc96c7 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -11,13 +11,10 @@
   $: connectedScreens = findConnectedScreens($screenStore.screens, resourceId)
 
   const findConnectedScreens = (screens, resourceId) => {
-    console.log(resourceId)
     return screens.filter(screen => {
       return JSON.stringify(screen).includes(`"${resourceId}"`)
     })
   }
-
-  $: console.log(connectedScreens)
 </script>
 
 <DetailPopover title="Screens" minWidth={400}>
diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index 67d2e0a6eb0..a399bae10fb 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -5,6 +5,7 @@
   export let align = "left"
   export let minWidth
   export let maxWidth
+  export let showPopover
 
   let popover
   let anchor
@@ -23,10 +24,11 @@
 <Popover
   bind:this={popover}
   bind:open
+  minWidth={minWidth || 300}
   {anchor}
   {align}
-  minWidth={minWidth || 300}
   {maxWidth}
+  {showPopover}
 >
   <div class="detail-popover">
     <div class="detail-popover__header">

From 787d93e317595acba9efd2eeeb3a47902678f001 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 08:45:21 +0100
Subject: [PATCH 017/455] Make role options reactive

---
 .../DataTable/modals/ManageAccessModal.svelte | 61 ++++++++-----------
 1 file changed, 27 insertions(+), 34 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
index d26078f47e5..bc5c437c572 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
@@ -17,6 +17,32 @@
 
   const inheritedRoleId = "inherited"
 
+  let dependantsInfoMessage
+
+  $: loadDependantInfo(resourceId)
+  $: computedPermissions = Object.entries(permissions.permissions).reduce(
+    (p, [level, roleInfo]) => {
+      p[level] = {
+        selectedValue:
+          roleInfo.permissionType === PermissionSource.INHERITED
+            ? inheritedRoleId
+            : roleInfo.role,
+        options: [...$roles],
+      }
+      if (roleInfo.inheritablePermission) {
+        p[level].inheritOption = roleInfo.inheritablePermission
+        p[level].options.unshift({
+          _id: inheritedRoleId,
+          name: `Inherit (${
+            get(roles).find(x => x._id === roleInfo.inheritablePermission).name
+          })`,
+        })
+      }
+      return p
+    },
+    {}
+  )
+
   async function changePermission(level, role) {
     try {
       if (role === inheritedRoleId) {
@@ -41,36 +67,9 @@
     }
   }
 
-  $: computedPermissions = Object.entries(permissions.permissions).reduce(
-    (p, [level, roleInfo]) => {
-      p[level] = {
-        selectedValue:
-          roleInfo.permissionType === PermissionSource.INHERITED
-            ? inheritedRoleId
-            : roleInfo.role,
-        options: [...get(roles)],
-      }
-
-      if (roleInfo.inheritablePermission) {
-        p[level].inheritOption = roleInfo.inheritablePermission
-        p[level].options.unshift({
-          _id: inheritedRoleId,
-          name: `Inherit (${
-            get(roles).find(x => x._id === roleInfo.inheritablePermission).name
-          })`,
-        })
-      }
-      return p
-    },
-    {}
-  )
-
-  let dependantsInfoMessage
-  async function loadDependantInfo() {
+  async function loadDependantInfo(resourceId) {
     const dependantsInfo = await permissionsStore.getDependantsInfo(resourceId)
-
     const resourceByType = dependantsInfo?.resourceByType
-
     if (resourceByType) {
       const total = Object.values(resourceByType).reduce((p, c) => p + c, 0)
       let resourceDisplay =
@@ -85,7 +84,6 @@
       }
     }
   }
-  loadDependantInfo()
 </script>
 
 <Body size="S">Specify the minimum access level role for this data.</Body>
@@ -122,11 +120,6 @@
     grid-template-columns: 1fr 1fr;
     grid-gap: var(--spacing-s);
   }
-
-  .lock-tag {
-    padding-left: var(--spacing-s);
-  }
-
   .inheriting-resources {
     display: flex;
     gap: var(--spacing-s);

From 49628e6235f528d2cac03ecf412c300cb54e0ea1 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 22 Aug 2024 09:26:08 +0100
Subject: [PATCH 018/455] Merge commit

---
 .../builder/src/stores/builder/components.js  |  35 +++-
 packages/frontend-core/src/utils/utils.js     |  97 +++++++++++
 packages/shared-core/src/filters.ts           | 154 +++++++++++++++++-
 3 files changed, 282 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index c281c73dfe3..dae23010687 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -29,7 +29,7 @@ import {
   DB_TYPE_EXTERNAL,
 } from "constants/backend"
 import BudiStore from "../BudiStore"
-import { Utils } from "@budibase/frontend-core"
+import { Utils, Constants } from "@budibase/frontend-core"
 import { FieldType } from "@budibase/types"
 
 export const INITIAL_COMPONENTS_STATE = {
@@ -196,6 +196,26 @@ export class ComponentStore extends BudiStore {
       }
     }
 
+    if (!enrichedComponent?._component) {
+      return migrated
+    }
+
+    const def = this.getDefinition(enrichedComponent?._component)
+    const filterableTypes = def.settings.filter(setting =>
+      setting?.type?.startsWith("filter")
+    )
+
+    // Map this?
+    for (let setting of filterableTypes) {
+      const updateFilter = Utils.migrateSearchFilters(
+        enrichedComponent[setting.key]
+      )
+      // DEAN - switch to real value when complete
+      if (updateFilter) {
+        enrichedComponent[setting.key + "_x"] = updateFilter
+      }
+    }
+
     return migrated
   }
 
@@ -405,7 +425,13 @@ export class ComponentStore extends BudiStore {
       screen: get(selectedScreen),
       useDefaultValues: true,
     })
-    this.migrateSettings(instance)
+
+    try {
+      this.migrateSettings(instance)
+    } catch (e) {
+      console.error(e)
+      throw e
+    }
 
     // Custom post processing for creation only
     let extras = {}
@@ -555,7 +581,10 @@ export class ComponentStore extends BudiStore {
       const patchResult = patchFn(component, screen)
 
       // Post processing
-      const migrated = this.migrateSettings(component)
+      // DEAN - SKIP ON SAVE FOR THE MOMENT
+      const migrated = null
+
+      this.migrateSettings(component)
 
       // Returning an explicit false signifies that we should skip this
       // update. If we migrated something, ensure we never skip.
diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js
index b8cfa5ad376..60752a35fe5 100644
--- a/packages/frontend-core/src/utils/utils.js
+++ b/packages/frontend-core/src/utils/utils.js
@@ -1,5 +1,6 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { Helpers } from "@budibase/bbui"
+import * as Constants from "../constants"
 
 export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
@@ -350,3 +351,99 @@ export const buildMultiStepFormBlockDefaultProps = props => {
     title,
   }
 }
+
+/**
+ * Processes the filter config. Filters are migrated
+ * SearchFilter[] to SearchFilterGroup
+ *
+ *  This is supposed to be in shared-core
+ * @param {Object[]} filter
+ */
+export const migrateSearchFilters = filters => {
+  const defaultCfg = {
+    logicalOperator: Constants.FilterOperator.ALL,
+    groups: [],
+  }
+
+  const filterWhitelistKeys = [
+    "field",
+    "operator",
+    "valueType", // bb
+    "value",
+    "type",
+    "noValue", // bb
+  ]
+
+  /**
+   * Review these
+   * externalType, formulaType, subtype
+   */
+
+  if (Array.isArray(filters)) {
+    const baseGroup = {
+      filters: [],
+      logicalOperator: Constants.FilterOperator.ALL,
+    }
+
+    const migratedSetting = filters.reduce((acc, filter) => {
+      // Sort the properties for easier debugging
+      // Remove unset values
+      const filterEntries = Object.entries(filter)
+        .sort((a, b) => {
+          return a[0].localeCompare(b[0])
+        })
+        .filter(x => x[1] ?? false)
+
+      // Scrub invalid filters
+      const { operator, onEmptyFilter, field, valueType } = filter
+      if (!field || !valueType) {
+        // THIS SCRUBS THE 2 GLOBALS
+        // return acc
+      }
+
+      if (filterEntries.length == 1) {
+        console.log("### one entry ")
+        const [key, value] = filterEntries[0]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise, seems to be the default
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = Constants.FilterOperator.ANY
+        }
+
+        return acc
+      }
+
+      // Process settings??
+      const whiteListedFilterSettings = filterEntries.reduce((acc, entry) => {
+        const [key, value] = entry
+
+        if (filterWhitelistKeys.includes(key)) {
+          if (key === "field") {
+            acc.push([key, value.replace(/^[0-9]+:/, "")])
+          } else {
+            acc.push([key, value])
+          }
+        }
+        return acc
+      }, [])
+
+      const migratedFilter = Object.fromEntries(whiteListedFilterSettings)
+
+      baseGroup.filters.push(migratedFilter)
+
+      if (!acc.groups.length) {
+        // init the base group
+        acc.groups.push(baseGroup)
+      }
+
+      return acc
+    }, defaultCfg)
+
+    console.log("MIGRATED ", migratedSetting)
+    return migratedSetting
+  }
+  return false
+}
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 2dd485a8a0e..d695859979c 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -19,12 +19,15 @@ import {
   RangeOperator,
   LogicalOperator,
   isLogicalSearchOperator,
+  SearchFilterGroup,
+  FilterGroupLogicalOperator,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
 import { decodeNonAscii } from "./helpers/schema"
+// import { Constants } from "@budibase/frontend-core"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
 
@@ -263,11 +266,134 @@ export class ColumnSplitter {
   }
 }
 
+/*
+  separate into buildQuery and build filter?
+*/
+
 /**
  * Builds a JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
  */
+
+const builderFilter = (expression: SearchFilter) => {
+  // Filter body
+  let query: SearchFilters = {
+    // string: {},
+    // fuzzy: {},
+    // range: {},
+    // equal: {},
+    // notEqual: {},
+    // empty: {},
+    // notEmpty: {},
+    // contains: {},
+    // notContains: {},
+    // oneOf: {},
+    // containsAny: {},
+  }
+
+  // DEAN -
+  // This is the chattiest service we have, pruning the requests
+  // of bloat should have meaningful impact
+  // Further validation in this area is a must
+
+  let { operator, field, type, value, externalType, onEmptyFilter } = expression
+  const queryOperator = operator as SearchFilterOperator
+  const isHbs =
+    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+  // Parse all values into correct types
+  if (operator === "allOr") {
+    query.allOr = true
+    return
+  }
+  if (onEmptyFilter) {
+    query.onEmptyFilter = onEmptyFilter
+    return
+  }
+  if (
+    type === "datetime" &&
+    !isHbs &&
+    queryOperator !== "empty" &&
+    queryOperator !== "notEmpty"
+  ) {
+    // Ensure date value is a valid date and parse into correct format
+    if (!value) {
+      return
+    }
+    try {
+      value = new Date(value).toISOString()
+    } catch (error) {
+      return
+    }
+  }
+  if (type === "number" && typeof value === "string" && !isHbs) {
+    if (queryOperator === "oneOf") {
+      value = value.split(",").map(item => parseFloat(item))
+    } else {
+      value = parseFloat(value)
+    }
+  }
+  if (type === "boolean") {
+    value = `${value}`?.toLowerCase() === "true"
+  }
+  if (
+    ["contains", "notContains", "containsAny"].includes(operator) &&
+    type === "array" &&
+    typeof value === "string"
+  ) {
+    value = value.split(",")
+  }
+  if (operator.startsWith("range") && query.range) {
+    const minint =
+      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
+        ?.min || Number.MIN_SAFE_INTEGER
+    const maxint =
+      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
+        ?.max || Number.MAX_SAFE_INTEGER
+    if (!query.range[field]) {
+      query.range[field] = {
+        low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+        high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+      }
+    }
+    if (operator === "rangeLow" && value != null && value !== "") {
+      query.range[field] = {
+        ...query.range[field],
+        low: value,
+      }
+    } else if (operator === "rangeHigh" && value != null && value !== "") {
+      query.range[field] = {
+        ...query.range[field],
+        high: value,
+      }
+    }
+  } else if (isLogicalSearchOperator(queryOperator)) {
+    // TODO
+  } else if (query[queryOperator] && operator !== "onEmptyFilter") {
+    if (type === "boolean") {
+      // Transform boolean filters to cope with null.
+      // "equals false" needs to be "not equals true"
+      // "not equals false" needs to be "equals true"
+      if (queryOperator === "equal" && value === false) {
+        query.notEqual = query.notEqual || {}
+        query.notEqual[field] = true
+      } else if (queryOperator === "notEqual" && value === false) {
+        query.equal = query.equal || {}
+        query.equal[field] = true
+      } else {
+        query[queryOperator] ??= {}
+        query[queryOperator]![field] = value
+      }
+    } else {
+      query[queryOperator] ??= {}
+      query[queryOperator]![field] = value
+    }
+  }
+
+  return query
+}
+
 export const buildQuery = (filter: SearchFilter[]) => {
+  //
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -383,10 +509,36 @@ export const buildQuery = (filter: SearchFilter[]) => {
       }
     }
   })
-
   return query
 }
 
+/**
+ * FOR TESTING - this still isnt type safe
+ */
+export const buildQueryX = (filter: SearchFilterGroup) => {
+  const operatorMap = {
+    [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
+    [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
+  }
+
+  const globalOnEmpty = filter.onEmptyFilter ? filter.onEmptyFilter : null
+  const globalOperator = operatorMap[filter.logicalOperator]
+
+  const coreRequest: SearchFilters = {
+    ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
+    [globalOperator]: {
+      conditions: filter.groups?.map((group: SearchFilterGroup) => {
+        return {
+          [operatorMap[group.logicalOperator]]: {
+            conditions: group.filters?.map(x => builderFilter(x)), //buildFilters
+          },
+        }
+      }),
+    },
+  }
+  return coreRequest
+}
+
 // The frontend can send single values for array fields sometimes, so to handle
 // this we convert them to arrays at the controller level so that nothing below
 // this has to worry about the non-array values.

From 8805b7f67f7bf4adf14b053b8340d08f92868810 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 11:36:29 +0100
Subject: [PATCH 019/455] Update row actions button

---
 .../DataTable/buttons/grid/GridRowActionsButton.svelte        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 7753d4eb3bc..52eee33453a 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -60,10 +60,10 @@
   }
 </script>
 
-<DetailPopover title="Row Actions" minWidth={400} maxWidth={400}>
+<DetailPopover title="Row actions" minWidth={400} maxWidth={400}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="Engagement" selected={open} quiet>
-      Row Actions ({suffix})
+      Row actions ({suffix})
     </ActionButton>
   </svelte:fragment>
   A row action is a user-triggered automation for a chosen row.

From bd4a804a969b4217120bd39cdeb85ceeeea7adb0 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 12:07:12 +0100
Subject: [PATCH 020/455] Add accent colors to all buttons

---
 .../bbui/src/ActionButton/ActionButton.svelte | 22 +++++++++++++++++++
 packages/bbui/src/helpers.js                  | 10 +++++++++
 .../buttons/TableFilterButton.svelte          |  9 +++++++-
 .../buttons/grid/GridAutomationsButton.svelte |  9 ++++++--
 .../grid/GridColumnsSettingButton.svelte      |  1 +
 .../buttons/grid/GridRowActionsButton.svelte  | 13 +++++++----
 .../buttons/grid/GridScreensButton.svelte     |  9 +++++++-
 7 files changed, 65 insertions(+), 8 deletions(-)

diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte
index 7019ad7d2c1..8b2d11232c8 100644
--- a/packages/bbui/src/ActionButton/ActionButton.svelte
+++ b/packages/bbui/src/ActionButton/ActionButton.svelte
@@ -3,6 +3,7 @@
   import { createEventDispatcher } from "svelte"
   import Tooltip from "../Tooltip/Tooltip.svelte"
   import { fade } from "svelte/transition"
+  import { hexToRGBA } from "../helpers"
 
   const dispatch = createEventDispatcher()
 
@@ -17,9 +18,22 @@
   export let fullWidth = false
   export let noPadding = false
   export let tooltip = ""
+  export let accentColor = null
 
   let showTooltip = false
 
+  $: accentStyle = getAccentStyle(accentColor)
+
+  const getAccentStyle = color => {
+    if (!color) {
+      return ""
+    }
+    let style = ""
+    style += `--accent-bg-color:${hexToRGBA(color, 0.15)};`
+    style += `--accent-border-color:${hexToRGBA(color, 0.35)};`
+    return style
+  }
+
   function longPress(element) {
     if (!longPressable) return
     let timer
@@ -47,6 +61,7 @@
   on:mouseover={() => (showTooltip = true)}
   on:mouseleave={() => (showTooltip = false)}
   on:focus={() => (showTooltip = true)}
+  style={accentStyle}
 >
   <button
     use:longPress
@@ -58,6 +73,7 @@
     class="spectrum-ActionButton spectrum-ActionButton--size{size}"
     class:active
     class:disabled
+    class:accent={accentColor != null}
     {disabled}
     on:longPress
     on:click|preventDefault
@@ -138,4 +154,10 @@
     text-align: center;
     z-index: 1;
   }
+
+  button.accent.is-selected,
+  button:active.accent {
+    border: 1px solid var(--accent-border-color);
+    background: var(--accent-bg-color);
+  }
 </style>
diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js
index 0f912a71614..246587af44e 100644
--- a/packages/bbui/src/helpers.js
+++ b/packages/bbui/src/helpers.js
@@ -228,3 +228,13 @@ export const getDateDisplayValue = (
     return value.format(`${localeDateFormat} HH:mm`)
   }
 }
+
+export const hexToRGBA = (color, opacity) => {
+  if (color.includes("#")) {
+    color = color.replace("#", "")
+  }
+  const r = parseInt(color.substring(0, 2), 16)
+  const g = parseInt(color.substring(2, 4), 16)
+  const b = parseInt(color.substring(4, 6), 16)
+  return `rgba(${r}, ${g}, ${b}, ${opacity})`
+}
diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index 91ec9a9b016..c4bdc653c32 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -44,7 +44,14 @@
   }
 </script>
 
-<ActionButton icon="Filter" quiet {disabled} on:click={drawer.show} {selected}>
+<ActionButton
+  icon="Filter"
+  quiet
+  {disabled}
+  on:click={drawer.show}
+  {selected}
+  accentColor="#004EA6"
+>
   {text}
 </ActionButton>
 
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index 5216fb5ebbe..a19dbe46969 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -39,9 +39,14 @@
 
 <DetailPopover title="Automations" minWidth={400} bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton icon="JourneyVoyager" selected={open} quiet
-      >Automations</ActionButton
+    <ActionButton
+      icon="JourneyVoyager"
+      selected={open || connectedAutomations.length}
+      quiet
+      accentColor="#5610AD"
     >
+      Automations
+    </ActionButton>
   </svelte:fragment>
   {#if !connectedAutomations.length}
     There aren't any automations connected to this data.
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
index 8c66d9ecfc4..3544a94d74b 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
@@ -22,6 +22,7 @@
     on:click={() => (open = !open)}
     selected={open || anyRestricted}
     disabled={!$columns.length}
+    accentColor="#674D00"
   >
     {text}
   </ActionButton>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 52eee33453a..90e446ce918 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -23,8 +23,8 @@
   $: tableId = ds?.tableId
   $: isView = ds?.type === "viewV2"
   $: fetchRowActions(tableId)
-  $: activeCount = 0
-  $: suffix = isView ? activeCount : rowActions.length
+  $: viewActiveCount = 0
+  $: activeCount = isView ? viewActiveCount : rowActions.length
 
   const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
     return ({ automationId }) => {
@@ -62,8 +62,13 @@
 
 <DetailPopover title="Row actions" minWidth={400} maxWidth={400}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton icon="Engagement" selected={open} quiet>
-      Row actions ({suffix})
+    <ActionButton
+      icon="Engagement"
+      selected={open || activeCount}
+      quiet
+      accentColor="#A24400"
+    >
+      Row actions ({activeCount})
     </ActionButton>
   </svelte:fragment>
   A row action is a user-triggered automation for a chosen row.
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index 48c9bdc96c7..9baeccb353f 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -19,7 +19,14 @@
 
 <DetailPopover title="Screens" minWidth={400}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton icon="WebPage" selected={open} quiet>Screens</ActionButton>
+    <ActionButton
+      icon="WebPage"
+      selected={open || connectedScreens.length}
+      quiet
+      accentColor="#364800"
+    >
+      Screens
+    </ActionButton>
   </svelte:fragment>
   {#if !connectedScreens.length}
     There aren't any screens connected to this data.

From 5153b684faaf71e0ff5a8dd28adac80c8d000953 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 12:10:47 +0100
Subject: [PATCH 021/455] Standardise grid button suffixes

---
 .../DataTable/buttons/grid/GridAutomationsButton.svelte     | 5 +++--
 .../DataTable/buttons/grid/GridColumnsSettingButton.svelte  | 2 +-
 .../DataTable/buttons/grid/GridRowActionsButton.svelte      | 6 +++---
 .../backend/DataTable/buttons/grid/GridScreensButton.svelte | 5 +++--
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index a19dbe46969..9420ca563a4 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -21,6 +21,7 @@
     $automationStore.automations,
     resourceId
   )
+  $: automationCount = connectedAutomations.length
 
   const findConnectedAutomations = (automations, resourceId) => {
     return automations.filter(automation => {
@@ -41,11 +42,11 @@
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="JourneyVoyager"
-      selected={open || connectedAutomations.length}
+      selected={open || automationCount}
       quiet
       accentColor="#5610AD"
     >
-      Automations
+      Automations{automationCount ? `: ${automationCount}` : ""}
     </ActionButton>
   </svelte:fragment>
   {#if !connectedAutomations.length}
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
index 3544a94d74b..11109c0f78a 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
@@ -11,7 +11,7 @@
   let anchor
 
   $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
-  $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns"
+  $: text = anyRestricted ? `Columns: ${anyRestricted} restricted` : "Columns"
 </script>
 
 <div bind:this={anchor}>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 90e446ce918..36ead100fc8 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -24,7 +24,7 @@
   $: isView = ds?.type === "viewV2"
   $: fetchRowActions(tableId)
   $: viewActiveCount = 0
-  $: activeCount = isView ? viewActiveCount : rowActions.length
+  $: actionCount = isView ? viewActiveCount : rowActions.length
 
   const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
     return ({ automationId }) => {
@@ -64,11 +64,11 @@
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="Engagement"
-      selected={open || activeCount}
+      selected={open || actionCount}
       quiet
       accentColor="#A24400"
     >
-      Row actions ({activeCount})
+      Row actions{actionCount ? `: ${actionCount}` : ""}
     </ActionButton>
   </svelte:fragment>
   A row action is a user-triggered automation for a chosen row.
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index 9baeccb353f..3fe6defffaa 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -9,6 +9,7 @@
   $: ds = $datasource
   $: resourceId = ds?.type === "table" ? ds.tableId : ds?.id
   $: connectedScreens = findConnectedScreens($screenStore.screens, resourceId)
+  $: screenCount = connectedScreens.length
 
   const findConnectedScreens = (screens, resourceId) => {
     return screens.filter(screen => {
@@ -21,11 +22,11 @@
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="WebPage"
-      selected={open || connectedScreens.length}
+      selected={open || screenCount}
       quiet
       accentColor="#364800"
     >
-      Screens
+      Screens{screenCount ? `: ${screenCount}` : ""}
     </ActionButton>
   </svelte:fragment>
   {#if !connectedScreens.length}

From dc5c66927158fd519dc6b9d6fbb705047edb861b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 14:39:54 +0100
Subject: [PATCH 022/455] Add multi coloured generate icon

---
 .../buttons/grid/GridGenerateButton.svelte          | 13 ++++++++++++-
 .../backend/DataTable/buttons/grid/magic-wand.svg   |  6 ++++++
 2 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index a7f1cf57c53..7a904d2eab8 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -5,6 +5,7 @@
   import { TriggerStepID } from "constants/backend/automations"
   import { goto } from "@roxi/routify"
   import DetailPopover from "components/common/DetailPopover.svelte"
+  import MagicWand from "./magic-wand.svg"
 
   const { datasource } = getContext("grid")
 
@@ -60,7 +61,12 @@
 
 <DetailPopover title="Generate" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton icon="MagicWand" selected={open}>Generate</ActionButton>
+    <ActionButton selected={open}>
+      <div class="center">
+        <img height={16} alt="magic wand" src={MagicWand} />
+        Generate
+      </div>
+    </ActionButton>
   </svelte:fragment>
   <div class="menu">
     <Menu>
@@ -93,4 +99,9 @@
     flex-direction: column;
     align-items: stretch;
   }
+  .center {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+  }
 </style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg b/packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg
new file mode 100644
index 00000000000..1702c9470a7
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg
@@ -0,0 +1,6 @@
+<svg width="13" height="12" viewBox="0 0 13 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M9.4179 4.13222C9.4179 3.73121 9.26166 3.35428 8.97913 3.07175C8.41342 2.50538 7.4239 2.50408 6.85753 3.07175L5.64342 4.28586C5.6291 4.30018 5.61543 4.3158 5.60305 4.33143C5.58678 4.3438 5.5718 4.35747 5.55683 4.37244L0.491426 9.43785C0.208245 9.72103 0.052002 10.098 0.052002 10.4983C0.052002 10.8987 0.208245 11.2756 0.491426 11.5588C0.774607 11.842 1.15153 11.9982 1.5519 11.9982C1.95227 11.9982 2.32919 11.842 2.61238 11.5588L8.97848 5.1927C9.26166 4.90952 9.4179 4.53259 9.4179 4.13222ZM1.90539 10.8518C1.7166 11.0406 1.3872 11.0406 1.1984 10.8518C1.10401 10.7574 1.05193 10.6318 1.05193 10.4983C1.05193 10.3649 1.104 10.2392 1.1984 10.1448L5.99821 5.34503L6.70845 6.04875L1.90539 10.8518ZM8.2715 4.48571L7.41544 5.34178L6.7052 4.63805L7.56452 3.77873C7.7533 3.58995 8.08271 3.58929 8.2715 3.77939C8.36589 3.87313 8.41798 3.99877 8.41798 4.13223C8.41798 4.26569 8.3659 4.39132 8.2715 4.48571Z" fill="#C8C8C8"/>
+<path d="M11.8552 6.55146L11.0144 6.21913L10.879 5.32449C10.8356 5.03919 10.3737 4.98776 10.2686 5.255L9.93606 6.09642L9.04143 6.23085C8.89951 6.25216 8.78884 6.36658 8.77257 6.50947C8.75629 6.65253 8.83783 6.78826 8.97193 6.84148L9.81335 7.17464L9.94794 8.06862C9.9691 8.21053 10.0835 8.32121 10.2266 8.33748C10.3695 8.35375 10.5052 8.27221 10.5586 8.13811L10.8914 7.29751L11.7855 7.1621C11.9283 7.1403 12.0381 7.02637 12.0544 6.88348C12.0707 6.74058 11.9887 6.60403 11.8552 6.55146Z" fill="#F9634C"/>
+<path d="M8.94215 1.76145L9.78356 2.0946L9.91815 2.9885C9.93931 3.13049 10.0539 3.24117 10.1968 3.25744C10.3398 3.27371 10.4756 3.19218 10.5288 3.05807L10.8618 2.21739L11.7559 2.08207C11.8985 2.06034 12.0085 1.94633 12.0248 1.80344C12.0411 1.66054 11.959 1.524 11.8254 1.47143L10.9847 1.13909L10.8494 0.244456C10.806 -0.0409246 10.3439 -0.0922745 10.2388 0.174881L9.90643 1.0163L9.0118 1.15089C8.86972 1.17213 8.75905 1.28654 8.74278 1.42952C8.72651 1.57249 8.80804 1.70823 8.94215 1.76145Z" fill="#8488FD"/>
+<path d="M3.2379 2.46066L3.92063 2.73091L4.02984 3.45637C4.04709 3.57151 4.14002 3.66135 4.25606 3.67453C4.37194 3.6878 4.48212 3.62163 4.52541 3.51276L4.79557 2.83059L5.52094 2.72074C5.63682 2.70316 5.72601 2.61072 5.73936 2.49468C5.75254 2.37864 5.68597 2.26797 5.57758 2.22533L4.89533 1.95565L4.78548 1.22963C4.75016 0.998038 4.37535 0.956375 4.29007 1.17315L4.0204 1.85597L3.29437 1.96517C3.17915 1.98235 3.08931 2.07527 3.07613 2.19131C3.06294 2.30727 3.12902 2.41737 3.2379 2.46066Z" fill="#F7D804"/>
+</svg>
\ No newline at end of file

From c810f023def5d135ca15507210ce542829a2f6ce Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 14:43:39 +0100
Subject: [PATCH 023/455] Improve styles of automation group headers

---
 .../automation/AutomationPanel/AutomationPanel.svelte  | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
index 58eebfdd3ef..2d58c7227d5 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
@@ -88,16 +88,22 @@
 
 <style>
   .nav-group {
-    padding-top: var(--spacing-l);
+    padding-top: 24px;
+  }
+  .nav-group:first-child {
+    padding-top: var(--spacing-s);
   }
   .nav-group-header {
     color: var(--spectrum-global-color-gray-600);
     padding: 0px calc(var(--spacing-l) + 4px);
-    padding-bottom: var(--spacing-l);
+    padding-bottom: var(--spacing-m);
     white-space: nowrap;
     overflow: hidden;
     text-overflow: ellipsis;
+    font-size: 12px;
+    font-weight: 600;
   }
+
   .side-bar {
     flex: 0 0 260px;
     display: flex;

From c7a4fddd11c9ebdd6e958e59d536a5df3dd14f12 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 16:44:57 +0100
Subject: [PATCH 024/455] Update automation grouping to group by datasource
 first

---
 .../AutomationPanel/AutomationPanel.svelte    | 58 +++++++++++++++----
 .../src/constants/backend/automations.js      |  1 +
 2 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
index 2d58c7227d5..6b96c4ebf5b 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
@@ -3,13 +3,21 @@
   import { Modal, notifications, Layout } from "@budibase/bbui"
   import NavHeader from "components/common/NavHeader.svelte"
   import { onMount } from "svelte"
-  import { automationStore } from "stores/builder"
+  import { automationStore, tables } from "stores/builder"
   import AutomationNavItem from "./AutomationNavItem.svelte"
+  import { TriggerStepID } from "constants/backend/automations"
 
   export let modal
   export let webhookModal
   let searchString
 
+  const dsTriggers = [
+    TriggerStepID.ROW_SAVED,
+    TriggerStepID.ROW_UPDATED,
+    TriggerStepID.ROW_DELETED,
+    TriggerStepID.ROW_ACTION,
+  ]
+
   $: filteredAutomations = $automationStore.automations
     .filter(automation => {
       return (
@@ -29,19 +37,47 @@
       return lowerA > lowerB ? 1 : -1
     })
 
-  $: groupedAutomations = filteredAutomations.reduce((acc, auto) => {
-    const catName = auto.definition?.trigger?.event || "No Trigger"
-    acc[catName] ??= {
-      icon: auto.definition?.trigger?.icon || "AlertCircle",
-      name: (auto.definition?.trigger?.name || "No Trigger").toUpperCase(),
-      entries: [],
-    }
-    acc[catName].entries.push(auto)
-    return acc
-  }, {})
+  $: groupedAutomations = groupAutomations(filteredAutomations)
 
   $: showNoResults = searchString && !filteredAutomations.length
 
+  const groupAutomations = automations => {
+    let groups = {}
+
+    for (let auto of automations) {
+      let category = null
+      let dataTrigger = false
+
+      // Group by datasource if possible
+      if (dsTriggers.includes(auto.definition?.trigger?.stepId)) {
+        if (auto.definition.trigger.inputs?.tableId) {
+          const tableId = auto.definition.trigger.inputs?.tableId
+          category = $tables.list.find(x => x._id === tableId)?.name
+        }
+      }
+      // Otherwise group by trigger
+      if (!category) {
+        category = auto.definition?.trigger?.name || "No Trigger"
+      } else {
+        dataTrigger = true
+      }
+      groups[category] ??= {
+        icon: auto.definition?.trigger?.icon || "AlertCircle",
+        name: category.toUpperCase(),
+        entries: [],
+        dataTrigger,
+      }
+      groups[category].entries.push(auto)
+    }
+
+    return Object.values(groups).sort((a, b) => {
+      if (a.dataTrigger === b.dataTrigger) {
+        return a.name < b.name ? -1 : 1
+      }
+      return a.dataTrigger ? -1 : 1
+    })
+  }
+
   onMount(async () => {
     try {
       await automationStore.actions.fetch()
diff --git a/packages/builder/src/constants/backend/automations.js b/packages/builder/src/constants/backend/automations.js
index 7c3e17e2254..d119cc6d552 100644
--- a/packages/builder/src/constants/backend/automations.js
+++ b/packages/builder/src/constants/backend/automations.js
@@ -5,6 +5,7 @@ export const TriggerStepID = {
   WEBHOOK: "WEBHOOK",
   APP: "APP",
   CRON: "CRON",
+  ROW_ACTION: "ROW_ACTION",
 }
 
 export const ActionStepID = {

From 2a67c0e09e8994225408ae01930168963c418f3e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 16:58:12 +0100
Subject: [PATCH 025/455] Allow deleting row actions from automations

---
 .../AutomationPanel/AutomationNavItem.svelte  | 74 +++++++++----------
 .../builder/src/stores/builder/automations.js | 17 ++++-
 2 files changed, 50 insertions(+), 41 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index 6e4d7c00992..ee5bf1af467 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -22,6 +22,7 @@
       await automationStore.actions.delete(automation)
       notifications.success("Automation deleted successfully")
     } catch (error) {
+      console.error(error)
       notifications.error("Error deleting automation")
     }
   }
@@ -37,41 +38,7 @@
 
   const getContextMenuItems = () => {
     const isRowAction = sdk.automations.isRowAction(automation)
-    const result = []
-    if (!isRowAction) {
-      result.push(
-        ...[
-          {
-            icon: "Delete",
-            name: "Delete",
-            keyBind: null,
-            visible: true,
-            disabled: false,
-            callback: confirmDeleteDialog.show,
-          },
-          {
-            icon: "Edit",
-            name: "Edit",
-            keyBind: null,
-            visible: true,
-            disabled: !automation.definition.trigger,
-            callback: updateAutomationDialog.show,
-          },
-          {
-            icon: "Duplicate",
-            name: "Duplicate",
-            keyBind: null,
-            visible: true,
-            disabled:
-              !automation.definition.trigger ||
-              automation.definition.trigger?.name === "Webhook",
-            callback: duplicateAutomation,
-          },
-        ]
-      )
-    }
-
-    result.push({
+    const pause = {
       icon: automation.disabled ? "CheckmarkCircle" : "Cancel",
       name: automation.disabled ? "Activate" : "Pause",
       keyBind: null,
@@ -83,8 +50,41 @@
           automation.disabled
         )
       },
-    })
-    return result
+    }
+    const del = {
+      icon: "Delete",
+      name: "Delete",
+      keyBind: null,
+      visible: true,
+      disabled: false,
+      callback: confirmDeleteDialog.show,
+    }
+    if (!isRowAction) {
+      return [
+        {
+          icon: "Edit",
+          name: "Edit",
+          keyBind: null,
+          visible: true,
+          disabled: !automation.definition.trigger,
+          callback: updateAutomationDialog.show,
+        },
+        {
+          icon: "Duplicate",
+          name: "Duplicate",
+          keyBind: null,
+          visible: true,
+          disabled:
+            !automation.definition.trigger ||
+            automation.definition.trigger?.name === "Webhook",
+          callback: duplicateAutomation,
+        },
+        pause,
+        del,
+      ]
+    } else {
+      return [pause, del]
+    }
   }
 
   const openContextMenu = e => {
diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js
index fdb09919112..69a340d3401 100644
--- a/packages/builder/src/stores/builder/automations.js
+++ b/packages/builder/src/stores/builder/automations.js
@@ -6,6 +6,7 @@ import { createHistoryStore } from "stores/builder/history"
 import { notifications } from "@budibase/bbui"
 import { updateReferencesInObject } from "dataBinding"
 import { AutomationTriggerStepId } from "@budibase/types"
+import { sdk } from "@budibase/shared-core"
 
 const initialAutomationState = {
   automations: [],
@@ -123,10 +124,18 @@ const automationActions = store => ({
     return response.automation
   },
   delete: async automation => {
-    await API.deleteAutomation({
-      automationId: automation?._id,
-      automationRev: automation?._rev,
-    })
+    const isRowAction = sdk.automations.isRowAction(automation)
+    if (isRowAction) {
+      await API.rowActions.delete({
+        tableId: automation.definition.trigger.inputs.tableId,
+        rowActionId: automation.definition.trigger.inputs.rowActionId,
+      })
+    } else {
+      await API.deleteAutomation({
+        automationId: automation?._id,
+        automationRev: automation?._rev,
+      })
+    }
 
     store.update(state => {
       // Remove the automation

From cc20e7dd52cf3980a9757f4bac657971088db0cb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 17:09:37 +0100
Subject: [PATCH 026/455] Fix icon colours in automation list

---
 .../AutomationPanel/AutomationNavItem.svelte    | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index ee5bf1af467..1e471ba50ce 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -99,7 +99,9 @@
 <NavItem
   on:contextmenu={openContextMenu}
   {icon}
-  iconColor={"var(--spectrum-global-color-gray-900)"}
+  iconColor={automation.disabled
+    ? "var(--spectrum-global-color-gray-600)"
+    : "var(--spectrum-global-color-gray-900)"}
   text={automation.displayName}
   selected={automation._id === $selectedAutomation?._id}
   hovering={automation._id === $contextMenuStore.id}
@@ -107,9 +109,7 @@
   selectedBy={$userSelectedResourceMap[automation._id]}
   disabled={automation.disabled}
 >
-  <div class="icon">
-    <Icon on:click={openContextMenu} size="S" hoverable name="MoreSmallList" />
-  </div>
+  <Icon on:click={openContextMenu} size="S" hoverable name="MoreSmallList" />
 </NavItem>
 
 <ConfirmDialog
@@ -123,12 +123,3 @@
   This action cannot be undone.
 </ConfirmDialog>
 <UpdateAutomationModal {automation} bind:this={updateAutomationDialog} />
-
-<style>
-  div.icon {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-end;
-    align-items: center;
-  }
-</style>

From 1adddba3c3187af4eb23b957db2af229ec852bda Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 18:40:29 +0100
Subject: [PATCH 027/455] Prevent disabling row actions

---
 .../FlowChart/FlowChart.svelte                | 28 +++++++++++--------
 .../AutomationPanel/AutomationNavItem.svelte  |  2 +-
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
index c263468f3be..c155b9bb1e9 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
@@ -12,13 +12,17 @@
   import { Icon, notifications, Modal, Toggle } from "@budibase/bbui"
   import { ActionStepID } from "constants/backend/automations"
   import UndoRedoControl from "components/common/UndoRedoControl.svelte"
+  import { sdk } from "@budibase/shared-core"
 
   export let automation
 
   let testDataModal
   let confirmDeleteDialog
   let scrolling = false
+
   $: blocks = getBlocks(automation).filter(x => x.stepId !== ActionStepID.LOOP)
+  $: isRowAction = sdk.automations.isRowAction(automation)
+
   const getBlocks = automation => {
     let blocks = []
     if (automation.definition.trigger) {
@@ -74,17 +78,19 @@
         Test details
       </div>
     </div>
-    <div class="setting-spacing">
-      <Toggle
-        text={automation.disabled ? "Paused" : "Activated"}
-        on:change={automationStore.actions.toggleDisabled(
-          automation._id,
-          automation.disabled
-        )}
-        disabled={!$selectedAutomation?.definition?.trigger}
-        value={!automation.disabled}
-      />
-    </div>
+    {#if !isRowAction}
+      <div class="setting-spacing">
+        <Toggle
+          text={automation.disabled ? "Paused" : "Activated"}
+          on:change={automationStore.actions.toggleDisabled(
+            automation._id,
+            automation.disabled
+          )}
+          disabled={!$selectedAutomation?.definition?.trigger}
+          value={!automation.disabled}
+        />
+      </div>
+    {/if}
   </div>
 </div>
 <div class="canvas" on:scroll={handleScroll}>
diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index 1e471ba50ce..8ab9851ff07 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -83,7 +83,7 @@
         del,
       ]
     } else {
-      return [pause, del]
+      return [del]
     }
   }
 

From c12a7ab19eb1be3db3a9bbb838a21b84cc274c65 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 22 Aug 2024 18:56:17 +0100
Subject: [PATCH 028/455] Remove deprecated unused props from action buttons

---
 .../bbui/src/ActionButton/ActionButton.svelte | 119 ++++++------------
 .../bbui/src/Notification/Notification.svelte |   6 +-
 2 files changed, 42 insertions(+), 83 deletions(-)

diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte
index 8b2d11232c8..b8030b80311 100644
--- a/packages/bbui/src/ActionButton/ActionButton.svelte
+++ b/packages/bbui/src/ActionButton/ActionButton.svelte
@@ -1,16 +1,11 @@
 <script>
   import "@spectrum-css/actionbutton/dist/index-vars.css"
-  import { createEventDispatcher } from "svelte"
   import Tooltip from "../Tooltip/Tooltip.svelte"
   import { fade } from "svelte/transition"
   import { hexToRGBA } from "../helpers"
 
-  const dispatch = createEventDispatcher()
-
   export let quiet = false
-  export let emphasized = false
   export let selected = false
-  export let longPressable = false
   export let disabled = false
   export let icon = ""
   export let size = "M"
@@ -33,82 +28,50 @@
     style += `--accent-border-color:${hexToRGBA(color, 0.35)};`
     return style
   }
-
-  function longPress(element) {
-    if (!longPressable) return
-    let timer
-
-    const listener = () => {
-      timer = setTimeout(() => {
-        dispatch("longpress")
-      }, 700)
-    }
-
-    element.addEventListener("pointerdown", listener)
-
-    return {
-      destroy() {
-        clearTimeout(timer)
-        element.removeEventListener("pointerdown", longPress)
-      },
-    }
-  }
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
-<span
-  class="btn-wrap"
+<button
+  class="spectrum-ActionButton spectrum-ActionButton--size{size}"
+  class:spectrum-ActionButton--quiet={quiet}
+  class:is-selected={selected}
+  class:noPadding
+  class:fullWidth
+  class:active
+  class:disabled
+  class:accent={accentColor != null}
+  on:click|preventDefault
   on:mouseover={() => (showTooltip = true)}
   on:mouseleave={() => (showTooltip = false)}
   on:focus={() => (showTooltip = true)}
+  {disabled}
   style={accentStyle}
 >
-  <button
-    use:longPress
-    class:spectrum-ActionButton--quiet={quiet}
-    class:spectrum-ActionButton--emphasized={emphasized}
-    class:is-selected={selected}
-    class:noPadding
-    class:fullWidth
-    class="spectrum-ActionButton spectrum-ActionButton--size{size}"
-    class:active
-    class:disabled
-    class:accent={accentColor != null}
-    {disabled}
-    on:longPress
-    on:click|preventDefault
-  >
-    {#if longPressable}
-      <svg
-        class="spectrum-Icon spectrum-UIIcon-CornerTriangle100 spectrum-ActionButton-hold"
-        focusable="false"
-        aria-hidden="true"
-      >
-        <use xlink:href="#spectrum-css-icon-CornerTriangle100" />
-      </svg>
-    {/if}
-    {#if icon}
-      <svg
-        class="spectrum-Icon spectrum-Icon--sizeS"
-        focusable="false"
-        aria-hidden="true"
-        aria-label={icon}
-      >
-        <use xlink:href="#spectrum-icon-18-{icon}" />
-      </svg>
-    {/if}
-    {#if $$slots}
-      <span class="spectrum-ActionButton-label"><slot /></span>
-    {/if}
-    {#if tooltip && showTooltip}
-      <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
-        <Tooltip textWrapping direction="bottom" text={tooltip} />
-      </div>
-    {/if}
-  </button>
-</span>
+  {#if icon}
+    <svg
+      class="spectrum-Icon spectrum-Icon--sizeS"
+      focusable="false"
+      aria-hidden="true"
+      aria-label={icon}
+    >
+      <use xlink:href="#spectrum-icon-18-{icon}" />
+    </svg>
+  {/if}
+  {#if $$slots}
+    <span class="spectrum-ActionButton-label"><slot /></span>
+  {/if}
+  {#if tooltip && showTooltip}
+    <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
+      <Tooltip textWrapping direction="bottom" text={tooltip} />
+    </div>
+  {/if}
+</button>
 
 <style>
+  button {
+    transition: filter 130ms ease-out, background 130ms ease-out,
+      border 130ms ease-out, color 130ms ease-out;
+  }
   .fullWidth {
     width: 100%;
   }
@@ -120,9 +83,7 @@
     margin-left: 0;
     transition: color ease-out 130ms;
   }
-  .is-selected:not(.spectrum-ActionButton--emphasized):not(
-      .spectrum-ActionButton--quiet
-    ) {
+  .is-selected:not(.spectrum-ActionButton--quiet) {
     background: var(--spectrum-global-color-gray-300);
     border-color: var(--spectrum-global-color-gray-500);
   }
@@ -137,7 +98,7 @@
     padding: 0;
     min-width: 0;
   }
-  .is-selected:not(.emphasized) .spectrum-Icon {
+  .is-selected .spectrum-Icon {
     color: var(--spectrum-global-color-gray-900);
   }
   .is-selected.disabled .spectrum-Icon {
@@ -154,10 +115,12 @@
     text-align: center;
     z-index: 1;
   }
-
-  button.accent.is-selected,
-  button:active.accent {
+  .accent.is-selected,
+  .accent:active {
     border: 1px solid var(--accent-border-color);
     background: var(--accent-bg-color);
   }
+  .accent:hover {
+    filter: brightness(1.2);
+  }
 </style>
diff --git a/packages/bbui/src/Notification/Notification.svelte b/packages/bbui/src/Notification/Notification.svelte
index 26e60ed3662..fbc7197e36e 100644
--- a/packages/bbui/src/Notification/Notification.svelte
+++ b/packages/bbui/src/Notification/Notification.svelte
@@ -27,11 +27,7 @@
   <div class="spectrum-Toast-body" class:actionBody={!!action}>
     <div class="wrap spectrum-Toast-content">{message || ""}</div>
     {#if action}
-      <ActionButton
-        quiet
-        emphasized
-        on:click={() => action(() => dispatch("dismiss"))}
-      >
+      <ActionButton quiet on:click={() => action(() => dispatch("dismiss"))}>
         <div style="color: white; font-weight: 600;">{actionMessage}</div>
       </ActionButton>
     {/if}

From a29cd7fefc1f90556655b37630718c94f9989b3d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 09:27:14 +0100
Subject: [PATCH 029/455] Update grid border to match data section

---
 packages/frontend-core/src/components/grid/layout/Grid.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index f6d36e348e4..b330f4f0332 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -277,7 +277,7 @@
     flex-direction: row;
     justify-content: space-between;
     align-items: center;
-    border-bottom: var(--cell-border);
+    border-bottom: var(--border-light);
     padding: var(--cell-padding);
     gap: var(--cell-spacing);
     background: var(--grid-background-alt);

From 495c01c1a2a22d8f480dbec4cdf99bc15ecfbabc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 11:19:56 +0100
Subject: [PATCH 030/455] Allow generating screens from the data section

---
 .../buttons/grid/GridGenerateButton.svelte    | 42 ++++++++++++++++++-
 packages/builder/src/constants/index.js       |  6 +++
 .../NewScreen/CreateScreenModal.svelte        | 41 +++++++++++++-----
 .../NewScreen/DatasourceModal.svelte          | 38 ++++-------------
 .../_components/NewScreen/TypeModal.svelte    |  2 +
 .../design/_components/NewScreen/utils.js     | 17 ++++++++
 6 files changed, 104 insertions(+), 42 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/utils.js

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index 7a904d2eab8..a54b4d51e1b 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -1,15 +1,24 @@
 <script>
   import { ActionButton, Menu, MenuItem, notifications } from "@budibase/bbui"
   import { getContext } from "svelte"
-  import { automationStore, tables, builderStore } from "stores/builder"
+  import {
+    automationStore,
+    tables,
+    builderStore,
+    viewsV2,
+  } from "stores/builder"
   import { TriggerStepID } from "constants/backend/automations"
   import { goto } from "@roxi/routify"
   import DetailPopover from "components/common/DetailPopover.svelte"
   import MagicWand from "./magic-wand.svg"
+  import { AutoScreenTypes } from "constants"
+  import CreateScreenModal from "pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte"
 
   const { datasource } = getContext("grid")
 
   let popover
+  let createScreenModal
+  let isCreatingScreen
 
   $: triggers = $automationStore.blockDefinitions.CREATABLE_TRIGGER
   $: table = $tables.list.find(table => table._id === $datasource.tableId)
@@ -57,6 +66,17 @@
       notifications.error("Error creating automation")
     }
   }
+
+  const startScreenWizard = autoScreenType => {
+    popover.hide()
+    let preSelected
+    if ($datasource.type === "table") {
+      preSelected = $tables.list.find(x => x._id === $datasource.tableId)
+    } else {
+      preSelected = $viewsV2.list.find(x => x.id === $datasource.id)
+    }
+    createScreenModal.show(autoScreenType, preSelected)
+  }
 </script>
 
 <DetailPopover title="Generate" bind:this={popover}>
@@ -88,10 +108,30 @@
       >
         Automation: when row is updated
       </MenuItem>
+      <MenuItem
+        icon="WebPage"
+        on:click={() => {
+          open = false
+          startScreenWizard(AutoScreenTypes.TABLE)
+        }}
+      >
+        CRUD app
+      </MenuItem>
+      <MenuItem
+        icon="WebPage"
+        on:click={() => {
+          open = false
+          startScreenWizard(AutoScreenTypes.FORM)
+        }}
+      >
+        Form
+      </MenuItem>
     </Menu>
   </div>
 </DetailPopover>
 
+<CreateScreenModal bind:this={createScreenModal} />
+
 <style>
   .menu {
     margin: 0 calc(-1 * var(--spacing-xl));
diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.js
index 44c71f2e3b7..0d51f57c3e8 100644
--- a/packages/builder/src/constants/index.js
+++ b/packages/builder/src/constants/index.js
@@ -72,3 +72,9 @@ export const PlanModel = {
 }
 
 export const ChangelogURL = "https://docs.budibase.com/changelog"
+
+export const AutoScreenTypes = {
+  BLANK: "blank",
+  TABLE: "table",
+  FORM: "form",
+}
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
index 554a2c80f58..51e82a6fe2e 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
@@ -10,12 +10,16 @@
     navigationStore,
     permissions as permissionsStore,
     builderStore,
+    datasources,
+    appStore,
   } from "stores/builder"
   import { auth } from "stores/portal"
   import { goto } from "@roxi/routify"
   import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
   import * as screenTemplating from "templates/screenTemplating"
   import { Roles } from "constants/backend"
+  import { AutoScreenTypes } from "constants"
+  import { makeTableOption, makeViewOption } from "./utils"
 
   let mode
 
@@ -23,20 +27,33 @@
   let datasourceModal
   let formTypeModal
   let tableTypeModal
-
   let selectedTablesAndViews = []
   let permissions = {}
+  let hasPreselectedDatasource = false
 
   $: screens = $screenStore.screens
 
-  export const show = newMode => {
+  export const show = (newMode, preselectedDatasource) => {
     mode = newMode
     selectedTablesAndViews = []
     permissions = {}
-
-    if (mode === "table" || mode === "form") {
-      datasourceModal.show()
-    } else if (mode === "blank") {
+    hasPreselectedDatasource = preselectedDatasource != null
+
+    if (mode === AutoScreenTypes.TABLE || mode === AutoScreenTypes.FORM) {
+      if (preselectedDatasource) {
+        // If preselecting a datasource, skip a step
+        const isTable = preselectedDatasource.type === "table"
+        const tableOrView = isTable
+          ? makeTableOption(preselectedDatasource, $datasources.list)
+          : makeViewOption(preselectedDatasource)
+        fetchPermission(tableOrView.id)
+        selectedTablesAndViews.push(tableOrView)
+        onSelectDatasources()
+      } else {
+        // Otherwise choose a datasource
+        datasourceModal.show()
+      }
+    } else if (mode === AutoScreenTypes.BLANK) {
       screenDetailsModal.show()
     } else {
       throw new Error("Invalid mode provided")
@@ -77,9 +94,9 @@
   }
 
   const onSelectDatasources = async () => {
-    if (mode === "form") {
+    if (mode === AutoScreenTypes.FORM) {
       formTypeModal.show()
-    } else if (mode === "table") {
+    } else if (mode === AutoScreenTypes.TABLE) {
       tableTypeModal.show()
     }
   }
@@ -136,9 +153,11 @@
     if (screen?.props?._children.length) {
       // Focus on the main component for the screen type
       const mainComponent = screen?.props?._children?.[0]._id
-      $goto(`./${screen._id}/${mainComponent}`)
+      $goto(
+        `/builder/app/${$appStore.appId}/design/${screen._id}/${mainComponent}`
+      )
     } else {
-      $goto(`./${screen._id}`)
+      $goto(`/builder/app/${$appStore.appId}/design/${screen._id}`)
     }
 
     screenStore.select(screen._id)
@@ -214,6 +233,7 @@
       tableTypeModal.hide()
       datasourceModal.show()
     }}
+    showCancelButton={!hasPreselectedDatasource}
   />
 </Modal>
 
@@ -230,5 +250,6 @@
       formTypeModal.hide()
       datasourceModal.show()
     }}
+    showCancelButton={!hasPreselectedDatasource}
   />
 </Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
index d036d6a9054..d32e8e3fc7a 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte
@@ -1,11 +1,11 @@
 <script>
-  import { Body, ModalContent, Layout, notifications } from "@budibase/bbui"
+  import { Body, ModalContent, Layout } from "@budibase/bbui"
   import { datasources as datasourcesStore } from "stores/builder"
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { IntegrationNames } from "constants"
-  import { createEventDispatcher, onMount } from "svelte"
+  import { createEventDispatcher } from "svelte"
   import TableOrViewOption from "./TableOrViewOption.svelte"
-  import * as format from "helpers/data/format"
+  import { makeTableOption, makeViewOption } from "./utils"
 
   export let onConfirm
   export let selectedTablesAndViews
@@ -16,17 +16,10 @@
     const views = Object.values(table.views || {}).filter(
       view => view.version === 2
     )
-
-    return views.map(view => ({
-      icon: "Remove",
-      name: view.name,
-      id: view.id,
-      tableSelectFormat: format.tableSelect.viewV2(view),
-      datasourceSelectFormat: format.datasourceSelect.viewV2(view),
-    }))
+    return views.map(makeViewOption)
   }
 
-  const getTablesAndViews = datasource => {
+  const getTablesAndViews = (datasource, datasources) => {
     let tablesAndViews = []
     const tables = Array.isArray(datasource.entities)
       ? datasource.entities
@@ -37,16 +30,7 @@
         continue
       }
 
-      const formattedTable = {
-        icon: "Table",
-        name: table.name,
-        id: table._id,
-        tableSelectFormat: format.tableSelect.table(table),
-        datasourceSelectFormat: format.datasourceSelect.table(
-          table,
-          $datasourcesStore.list
-        ),
-      }
+      const formattedTable = makeTableOption(table, datasources)
 
       tablesAndViews = tablesAndViews.concat([
         formattedTable,
@@ -71,7 +55,7 @@
       const datasource = {
         name: rawDatasource.name,
         iconComponent: ICONS[rawDatasource.source],
-        tablesAndViews: getTablesAndViews(rawDatasource),
+        tablesAndViews: getTablesAndViews(rawDatasource, rawDatasources),
       }
 
       datasources.push(datasource)
@@ -85,14 +69,6 @@
   const toggleSelection = tableOrView => {
     dispatch("toggle", tableOrView)
   }
-
-  onMount(async () => {
-    try {
-      await datasourcesStore.fetch()
-    } catch (error) {
-      notifications.error("Error fetching datasources")
-    }
-  })
 </script>
 
 <ModalContent
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte
index 045e3000e7a..07a24291331 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte
@@ -7,6 +7,7 @@
   export let types
   export let onCancel = () => {}
   export let onConfirm = () => {}
+  export let showCancelButton = true
 </script>
 
 <ModalContent
@@ -17,6 +18,7 @@
   {onCancel}
   disabled={!selectedType}
   size="L"
+  {showCancelButton}
 >
   <!-- svelte-ignore a11y-click-events-have-key-events -->
   <!-- svelte-ignore a11y-no-static-element-interactions -->
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/utils.js b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/utils.js
new file mode 100644
index 00000000000..1f2461d2cb3
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/utils.js
@@ -0,0 +1,17 @@
+import * as format from "helpers/data/format"
+
+export const makeViewOption = view => ({
+  icon: "Remove",
+  name: view.name,
+  id: view.id,
+  tableSelectFormat: format.tableSelect.viewV2(view),
+  datasourceSelectFormat: format.datasourceSelect.viewV2(view),
+})
+
+export const makeTableOption = (table, datasources) => ({
+  icon: "Table",
+  name: table.name,
+  id: table._id,
+  tableSelectFormat: format.tableSelect.table(table),
+  datasourceSelectFormat: format.datasourceSelect.table(table, datasources),
+})

From bf998f6e6c35e16d0695af61d915451e3855d708 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 12:00:19 +0100
Subject: [PATCH 031/455] Add complete generation popover with both automation
 and screen generation

---
 packages/bbui/src/List/ListItem.svelte        |  13 +-
 .../buttons/grid/GridAutomationsButton.svelte |   3 +-
 .../buttons/grid/GridGenerateButton.svelte    | 152 +++++++++++-------
 .../buttons/grid/GridRowActionsButton.svelte  |   2 +-
 .../buttons/grid/GridScreensButton.svelte     |  20 ++-
 .../table/[tableId]/[viewId]/index.svelte     |   7 +-
 .../data/table/[tableId]/index.svelte         |   4 +-
 7 files changed, 135 insertions(+), 66 deletions(-)

diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 0c9b796f5b9..341c53a190c 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -6,11 +6,18 @@
   export let title = null
   export let subtitle = null
   export let url = null
+  export let hoverable = false
+  export let showArrow = false
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
-<a href={url} class="list-item" class:hoverable={url != null} on:click>
+<a
+  href={url}
+  class="list-item"
+  class:hoverable={hoverable || url != null}
+  on:click
+>
   <div class="left">
     {#if icon}
       <Icon name={icon} color={iconColor} />
@@ -30,7 +37,9 @@
   </div>
   <div class="right">
     <slot name="right" />
-    <Icon name="ChevronRight" />
+    {#if showArrow}
+      <Icon name="ChevronRight" />
+    {/if}
   </div>
 </a>
 
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index 9420ca563a4..3cf6da7c246 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -34,7 +34,7 @@
 
   const generateAutomation = () => {
     popover?.hide()
-    dispatch("generate-automation")
+    dispatch("request-generate")
   }
 </script>
 
@@ -62,6 +62,7 @@
             : "var(--spectrum-global-color-green-600)"}
           title={automation.name}
           url={`/builder/app/${$appStore.appId}/automation/${automation._id}`}
+          showArrow
         />
       {/each}
     </List>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index a54b4d51e1b..84ef6102ca0 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { ActionButton, Menu, MenuItem, notifications } from "@budibase/bbui"
+  import { ActionButton, ListItem, notifications } from "@budibase/bbui"
   import { getContext } from "svelte"
   import {
     automationStore,
@@ -13,12 +13,12 @@
   import MagicWand from "./magic-wand.svg"
   import { AutoScreenTypes } from "constants"
   import CreateScreenModal from "pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte"
+  import { getSequentialName } from "helpers/duplicate"
 
   const { datasource } = getContext("grid")
 
   let popover
   let createScreenModal
-  let isCreatingScreen
 
   $: triggers = $automationStore.blockDefinitions.CREATABLE_TRIGGER
   $: table = $tables.list.find(table => table._id === $datasource.tableId)
@@ -39,9 +39,19 @@
       return
     }
 
-    const automationName = `${table.name} : Row ${
-      type === TriggerStepID.ROW_SAVED ? "created" : "updated"
-    }`
+    const suffixMap = {
+      [TriggerStepID.ROW_SAVED]: "created",
+      [TriggerStepID.ROW_UPDATED]: "updated",
+      [TriggerStepID.ROW_DELETED]: "deleted",
+    }
+    const namePrefix = `Row ${suffixMap[type]} `
+    const automationName = getSequentialName(
+      $automationStore.automations,
+      namePrefix,
+      {
+        getName: x => x.name,
+      }
+    )
     const triggerBlock = automationStore.actions.constructBlock(
       "TRIGGER",
       triggerType.stepId,
@@ -59,10 +69,10 @@
         "/builder/app/:application/data",
         window.location.pathname
       )
-      $goto(`/builder/app/${response.appId}/automation/${response.id}`)
-      notifications.success(`Automation created`)
+      $goto(`/builder/app/${response.appId}/automation/${response._id}`)
+      notifications.success(`Automation created successfully`)
     } catch (e) {
-      console.error("Error creating automation", e)
+      console.error(e)
       notifications.error("Error creating automation")
     }
   }
@@ -79,7 +89,7 @@
   }
 </script>
 
-<DetailPopover title="Generate" bind:this={popover}>
+<DetailPopover title="Generate" bind:this={popover} minWidth={400}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton selected={open}>
       <div class="center">
@@ -88,60 +98,94 @@
       </div>
     </ActionButton>
   </svelte:fragment>
-  <div class="menu">
-    <Menu>
-      <MenuItem
-        icon="ShareAndroid"
-        on:click={() => {
-          open = false
-          createAutomation(TriggerStepID.ROW_SAVED)
-        }}
-      >
-        Automation: when row is created
-      </MenuItem>
-      <MenuItem
-        icon="ShareAndroid"
-        on:click={() => {
-          open = false
-          createAutomation(TriggerStepID.ROW_UPDATED)
-        }}
-      >
-        Automation: when row is updated
-      </MenuItem>
-      <MenuItem
-        icon="WebPage"
-        on:click={() => {
-          open = false
-          startScreenWizard(AutoScreenTypes.TABLE)
-        }}
-      >
-        CRUD app
-      </MenuItem>
-      <MenuItem
-        icon="WebPage"
-        on:click={() => {
-          open = false
-          startScreenWizard(AutoScreenTypes.FORM)
-        }}
-      >
-        Form
-      </MenuItem>
-    </Menu>
+
+  {#if $datasource.type === "table"}
+    Generate a new app screen or automation from this data.
+  {:else}
+    Generate a new app screen from this data.
+  {/if}
+
+  <div class="generate-section">
+    <div class="generate-section__title">App screens</div>
+    <div class="generate-section__options">
+      <div>
+        <ListItem
+          title="CRUD app"
+          icon="TableEdit"
+          hoverable
+          on:click={() => startScreenWizard(AutoScreenTypes.TABLE)}
+          iconColor="var(--spectrum-global-color-gray-600)"
+        />
+      </div>
+      <div>
+        <ListItem
+          title="Form"
+          icon="Form"
+          hoverable
+          on:click={() => startScreenWizard(AutoScreenTypes.FORM)}
+          iconColor="var(--spectrum-global-color-gray-600)"
+        />
+      </div>
+    </div>
   </div>
+
+  {#if $datasource.type === "table"}
+    <div class="generate-section">
+      <div class="generate-section__title">Automation triggers (When a...)</div>
+      <div class="generate-section__options">
+        <div>
+          <ListItem
+            title="Row is created"
+            icon="TableRowAddBottom"
+            hoverable
+            on:click={() => createAutomation(TriggerStepID.ROW_SAVED)}
+            iconColor="var(--spectrum-global-color-gray-600)"
+          />
+        </div>
+        <div>
+          <ListItem
+            title="Row is updated"
+            icon="Refresh"
+            hoverable
+            on:click={() => createAutomation(TriggerStepID.ROW_UPDATED)}
+            iconColor="var(--spectrum-global-color-gray-600)"
+          />
+        </div>
+        <div>
+          <ListItem
+            title="Row is deleted"
+            icon="TableRowRemoveCenter"
+            hoverable
+            on:click={() => createAutomation(TriggerStepID.ROW_DELETED)}
+            iconColor="var(--spectrum-global-color-gray-600)"
+          />
+        </div>
+      </div>
+    </div>
+  {/if}
 </DetailPopover>
 
 <CreateScreenModal bind:this={createScreenModal} />
 
 <style>
-  .menu {
-    margin: 0 calc(-1 * var(--spacing-xl));
-    display: flex;
-    flex-direction: column;
-    align-items: stretch;
-  }
   .center {
     display: flex;
     align-items: center;
     gap: 8px;
   }
+
+  .generate-section {
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+  }
+  .generate-section__title {
+    color: var(--spectrum-global-color-gray-600);
+  }
+  .generate-section__options {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    grid-column-gap: 16px;
+    grid-row-gap: 8px;
+  }
 </style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 36ead100fc8..7b52c72fc73 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -83,7 +83,7 @@
   {:else}
     <List>
       {#each rowActions as action}
-        <ListItem title={action.name} url={$rowActionUrl(action)}>
+        <ListItem title={action.name} url={$rowActionUrl(action)} showArrow>
           <svelte:fragment slot="right">
             {#if isView}
               <span>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index 3fe6defffaa..c1927306706 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -1,10 +1,13 @@
 <script>
-  import { ActionButton, List, ListItem } from "@budibase/bbui"
+  import { ActionButton, List, ListItem, Button } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
   import { screenStore, appStore } from "stores/builder"
-  import { getContext } from "svelte"
+  import { getContext, createEventDispatcher } from "svelte"
 
   const { datasource } = getContext("grid")
+  const dispatch = createEventDispatcher()
+
+  let popover
 
   $: ds = $datasource
   $: resourceId = ds?.type === "table" ? ds.tableId : ds?.id
@@ -16,9 +19,14 @@
       return JSON.stringify(screen).includes(`"${resourceId}"`)
     })
   }
+
+  const generateScreen = () => {
+    popover?.hide()
+    dispatch("request-generate")
+  }
 </script>
 
-<DetailPopover title="Screens" minWidth={400}>
+<DetailPopover title="Screens" minWidth={400} bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="WebPage"
@@ -38,8 +46,14 @@
         <ListItem
           title={screen.routing.route}
           url={`/builder/app/${$appStore.appId}/design/${screen._id}`}
+          showArrow
         />
       {/each}
     </List>
   {/if}
+  <div>
+    <Button secondary icon="WebPage" on:click={generateScreen}>
+      Generate app screen
+    </Button>
+  </div>
 </DetailPopover>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 88537ccd0ef..798d1e0b51e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -12,7 +12,8 @@
   import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
   import GridScreensButton from "components/backend/DataTable/buttons/grid/GridScreensButton.svelte"
   import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
-  import GridAutomationsButton from "components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte"
+
+  let generateButton
 
   $: id = $viewsV2.selected?.id
   $: datasource = {
@@ -43,10 +44,10 @@
     <GridColumnsSettingButton />
     <GridManageAccessButton />
     <GridRowActionsButton />
-    <GridScreensButton />
+    <GridScreensButton on:request-generate={() => generateButton?.show()} />
   </svelte:fragment>
   <svelte:fragment slot="controls-right">
-    <GridGenerateButton />
+    <GridGenerateButton bind:this={generateButton} />
   </svelte:fragment>
   <GridCreateEditRowModal />
 </Grid>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 50ad390b1c7..81752387793 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -120,9 +120,9 @@
         <GridRowActionsButton />
       {/if}
       <GridAutomationsButton
-        on:generate-automation={() => generateButton?.show()}
+        on:request-generate={() => generateButton?.show()}
       />
-      <GridScreensButton />
+      <GridScreensButton on:request-generate={() => generateButton?.show()} />
       {#if !isUsersTable}
         <GridImportButton />
       {/if}

From 51decfa58d40cf6a77c3716317e5722eec95f5fe Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 13:00:12 +0100
Subject: [PATCH 032/455] Update wording

---
 .../backend/DataTable/buttons/grid/GridAutomationsButton.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index 3cf6da7c246..39d894d2fde 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -69,7 +69,7 @@
   {/if}
   <div>
     <Button secondary icon="JourneyVoyager" on:click={generateAutomation}>
-      Generate new automation
+      Generate automation
     </Button>
   </div>
 </DetailPopover>

From 0dd432f286bfe8897a20f668049346f5640c2929 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 13:34:36 +0100
Subject: [PATCH 033/455] Add support for old views in new view nav bar

---
 .../[tableId]/_components/ViewNavBar.svelte   | 46 +++++++++++++++---
 .../[tableId]}/v1/[viewName]/_layout.svelte   |  0
 .../[tableId]}/v1/[viewName]/index.svelte     | 47 ++++++++++---------
 .../{view => table/[tableId]}/v1/index.svelte |  0
 .../app/[application]/data/view/index.svelte  | 19 --------
 packages/client/src/constants.js              |  3 +-
 6 files changed, 68 insertions(+), 47 deletions(-)
 rename packages/builder/src/pages/builder/app/[application]/data/{view => table/[tableId]}/v1/[viewName]/_layout.svelte (100%)
 rename packages/builder/src/pages/builder/app/[application]/data/{view => table/[tableId]}/v1/[viewName]/index.svelte (75%)
 rename packages/builder/src/pages/builder/app/[application]/data/{view => table/[tableId]}/v1/index.svelte (100%)
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/index.svelte

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index cc62eb78d54..5d3f3ac5b42 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -48,12 +48,18 @@
   $: datasource = $datasources.list.find(ds => ds._id === table?.sourceId)
   $: tableSelectedBy = $userSelectedResourceMap[table?._id]
   $: tableEditable = table?._id !== TableNames.USERS
-  $: activeId = $params.viewId ?? $params.tableId
+  $: activeId = $params.viewName ?? $params.viewId ?? $params.tableId
   $: views = Object.values(table?.views || {})
     .filter(x => x.version === 2)
     .slice()
     .sort(alphabetical)
+  $: v1Views = Object.values(table?.views || {})
+    .filter(x => x.version !== 2)
+    .slice()
+    .sort(alphabetical)
   $: setUpObserver(views)
+  $: hasViews = v1Views.length || views.length
+
   $: overflowedViews = views.filter(view => !viewVisibiltyMap[view.id])
   $: viewHidden = viewVisibiltyMap[activeId] === false
 
@@ -61,6 +67,10 @@
     return $url(`../${$params.tableId}/${encodeURIComponent(viewId)}`)
   })
 
+  const viewV1Url = derived([url, params], ([$url, $params]) => viewName => {
+    return $url(`../${$params.tableId}/v1/${encodeURIComponent(viewName)}`)
+  })
+
   const tableUrl = derived(url, $url => tableId => $url(`../${tableId}`))
 
   const openTableContextMenu = e => {
@@ -176,8 +186,8 @@
 
 <div class="nav">
   <IntegrationIcon
-    integrationType={datasource.source}
-    schema={datasource.schema}
+    integrationType={datasource?.source}
+    schema={datasource?.schema}
     size="24"
   />
   <a
@@ -202,8 +212,32 @@
       />
     {/if}
   </a>
-  {#if views.length}
+  {#if hasViews}
     <div class="nav__views" bind:this={viewContainer}>
+      {#each v1Views as view (view.name)}
+        {@const selectedBy = $userSelectedResourceMap[view.name]}
+        <a
+          href={$viewV1Url(view.name)}
+          class="nav-item"
+          class:active={view.name === activeId}
+          on:contextmenu={e => openViewContextMenu(e, view)}
+          data-id={view.name}
+        >
+          <div class="nav-item__title">
+            {view.name}
+          </div>
+          {#if selectedBy}
+            <UserAvatars size="XS" users={selectedBy} />
+          {/if}
+          <Icon
+            on:click={e => openViewContextMenu(e, view)}
+            hoverable
+            name="MoreSmallList"
+            color="var(--spectrum-global-color-gray-600)"
+            hoverColor="var(--spectrum-global-color-gray-900)"
+          />
+        </a>
+      {/each}
       {#each views as view (view.id)}
         {@const selectedBy = $userSelectedResourceMap[view.id]}
         <a
@@ -231,7 +265,7 @@
       {/each}
     </div>
   {/if}
-  {#if !views.length && tableEditable}
+  {#if !hasViews && tableEditable}
     <Button cta on:click={createViewModal?.show}>Create a view</Button>
     <span>
       To create subsets of data, control access and more, create a view.
@@ -274,7 +308,7 @@
       {/each}
     </ActionMenu>
   {/if}
-  {#if views.length}
+  {#if hasViews}
     <Icon
       name="Add"
       size="L"
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/_layout.svelte
similarity index 100%
rename from packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/_layout.svelte
rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/_layout.svelte
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
similarity index 75%
rename from packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
index d96a9239ba7..0084d9b5da1 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
@@ -57,29 +57,34 @@
   }
 </script>
 
-{#if view}
-  <Table
-    title={decodeURI(name)}
-    {schema}
-    tableId={view.tableId}
-    {data}
-    {loading}
-    rowCount={10}
-    allowEditing={false}
-    bind:hideAutocolumns
-  >
-    <ViewFilterButton {view} />
-    <CalculateButton {view} />
-    {#if view.calculation}
-      <GroupByButton {view} />
-    {/if}
-    <ManageAccessButton resourceId={decodeURI(name)} />
-    <HideAutocolumnButton bind:hideAutocolumns />
-    <ExportButton view={view.name} formats={supportedFormats} />
-  </Table>
-{:else}<i>Create your first table to start building</i>{/if}
+<div class="view-v1">
+  {#if view}
+    <Table
+      title={decodeURI(name)}
+      {schema}
+      tableId={view.tableId}
+      {data}
+      {loading}
+      rowCount={10}
+      allowEditing={false}
+      bind:hideAutocolumns
+    >
+      <ViewFilterButton {view} />
+      <CalculateButton {view} />
+      {#if view.calculation}
+        <GroupByButton {view} />
+      {/if}
+      <ManageAccessButton resourceId={decodeURI(name)} />
+      <HideAutocolumnButton bind:hideAutocolumns />
+      <ExportButton view={view.name} formats={supportedFormats} />
+    </Table>
+  {:else}<i>Create your first table to start building</i>{/if}
+</div>
 
 <style>
+  .view-v1 {
+    padding: var(--spacing-xl);
+  }
   i {
     font-size: var(--font-size-m);
     color: var(--grey-4);
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/index.svelte
similarity index 100%
rename from packages/builder/src/pages/builder/app/[application]/data/view/v1/index.svelte
rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/index.svelte
diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/index.svelte
deleted file mode 100644
index 623cd224dbc..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/view/index.svelte
+++ /dev/null
@@ -1,19 +0,0 @@
-<script>
-  import { onMount } from "svelte"
-  import { views, viewsV2 } from "stores/builder"
-  import { redirect } from "@roxi/routify"
-
-  onMount(async () => {
-    if ($viewsV2.selected) {
-      $redirect(`./v2/${$viewsV2.selected.id}`)
-    } else if ($viewsV2.list?.length) {
-      $redirect(`./v2/${$viewsV2.list[0].id}`)
-    } else if ($views.selected) {
-      $redirect(`./${encodeURIComponent($views.selected?.name)}`)
-    } else if ($views.list?.length) {
-      $redirect(`./${encodeURIComponent($views.list[0].name)}`)
-    } else {
-      $redirect("../")
-    }
-  })
-</script>
diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js
index f7e3e86d40c..bfa2d08dc59 100644
--- a/packages/client/src/constants.js
+++ b/packages/client/src/constants.js
@@ -3,7 +3,8 @@ export { FieldType as FieldTypes } from "@budibase/types"
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   UpdateFieldValue: "UpdateFieldValue",
-  RefreshDatasource: "RefreshDatasource",
+  RefreshDatasource:
+     "RefreshDatasource",
   AddDataProviderQueryExtension: "AddDataProviderQueryExtension",
   RemoveDataProviderQueryExtension: "RemoveDataProviderQueryExtension",
   SetDataProviderSorting: "SetDataProviderSorting",

From 8d87395a980209db22dff999d1e54ecb7ead238b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 13:41:35 +0100
Subject: [PATCH 034/455] Fix a few edge cases with v1 views

---
 .../data/table/[tableId]/_components/ViewNavBar.svelte    | 4 +++-
 .../data/table/[tableId]/v1/[viewName]/index.svelte       | 8 ++++----
 packages/builder/src/stores/builder/views.js              | 1 +
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 5d3f3ac5b42..b1fffebbfeb 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -48,7 +48,9 @@
   $: datasource = $datasources.list.find(ds => ds._id === table?.sourceId)
   $: tableSelectedBy = $userSelectedResourceMap[table?._id]
   $: tableEditable = table?._id !== TableNames.USERS
-  $: activeId = $params.viewName ?? $params.viewId ?? $params.tableId
+  $: activeId = decodeURIComponent(
+    $params.viewName ?? $params.viewId ?? $params.tableId
+  )
   $: views = Object.values(table?.views || {})
     .filter(x => x.version === 2)
     .slice()
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
index 0084d9b5da1..e6aaa647de3 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
@@ -17,9 +17,9 @@
   let loading = false
 
   $: view = $views.selected
-  $: name = view.name
-  $: schema = view.schema
-  $: calculation = view.calculation
+  $: name = view?.name
+  $: schema = view?.schema
+  $: calculation = view?.calculation
   $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {
     if (calculation && key === ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA) {
       return false
@@ -28,7 +28,7 @@
   })
 
   // Fetch rows for specified view
-  $: fetchViewData(name, view.field, view.groupBy, view.calculation)
+  $: fetchViewData(name, view?.field, view?.groupBy, view?.calculation)
 
   async function fetchViewData(name, field, groupBy, calculation) {
     loading = true
diff --git a/packages/builder/src/stores/builder/views.js b/packages/builder/src/stores/builder/views.js
index b9986889db6..1d2463e2e27 100644
--- a/packages/builder/src/stores/builder/views.js
+++ b/packages/builder/src/stores/builder/views.js
@@ -41,6 +41,7 @@ export function createViewsStore() {
 
   const save = async view => {
     const savedView = await API.saveView(view)
+    select(view.name)
 
     // Update tables
     tables.update(state => {

From 57da6d8cac8dfa50c2bcfffc3580a95804ab53f3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:10:23 +0100
Subject: [PATCH 035/455] Update create view button to be a popover instead of
 modal

---
 .../buttons/grid/GridAutomationsButton.svelte |   2 +-
 .../buttons/grid/GridCreateViewButton.svelte  |  29 -----
 .../buttons/grid/GridGenerateButton.svelte    |   2 +-
 .../buttons/grid/GridRowActionsButton.svelte  |   2 +-
 .../buttons/grid/GridScreensButton.svelte     |   2 +-
 .../components/common/DetailPopover.svelte    |   8 +-
 .../_components/CreateViewButton.svelte       | 103 ++++++++++++++++++
 .../_components/CreateViewModal.svelte        |  62 -----------
 .../_components/EditViewModal.svelte          |   4 +-
 .../[tableId]/_components/ViewNavBar.svelte   |  17 +--
 10 files changed, 117 insertions(+), 114 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index 39d894d2fde..9670bb2f358 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -38,7 +38,7 @@
   }
 </script>
 
-<DetailPopover title="Automations" minWidth={400} bind:this={popover}>
+<DetailPopover title="Automations" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="JourneyVoyager"
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte
deleted file mode 100644
index d4f4bcd1b1a..00000000000
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte
+++ /dev/null
@@ -1,29 +0,0 @@
-<script>
-  import { getContext } from "svelte"
-  import { Modal, ActionButton, TooltipType, TempTooltip } from "@budibase/bbui"
-  import GridCreateViewModal from "../../modals/grid/GridCreateViewModal.svelte"
-
-  const { filter } = getContext("grid")
-
-  let modal
-  let firstFilterUsage = false
-
-  $: {
-    if ($filter?.length && !firstFilterUsage) {
-      firstFilterUsage = true
-    }
-  }
-</script>
-
-<TempTooltip
-  text="Create a view to save your filters"
-  type={TooltipType.Info}
-  condition={firstFilterUsage}
->
-  <ActionButton icon="CollectionAdd" quiet on:click={modal.show}>
-    Create view
-  </ActionButton>
-</TempTooltip>
-<Modal bind:this={modal}>
-  <GridCreateViewModal />
-</Modal>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index 84ef6102ca0..cb2afbc0008 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -89,7 +89,7 @@
   }
 </script>
 
-<DetailPopover title="Generate" bind:this={popover} minWidth={400}>
+<DetailPopover title="Generate" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton selected={open}>
       <div class="center">
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 7b52c72fc73..a6a7c930f55 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -60,7 +60,7 @@
   }
 </script>
 
-<DetailPopover title="Row actions" minWidth={400} maxWidth={400}>
+<DetailPopover title="Row actions">
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="Engagement"
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index c1927306706..701a286112a 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -26,7 +26,7 @@
   }
 </script>
 
-<DetailPopover title="Screens" minWidth={400} bind:this={popover}>
+<DetailPopover title="Screens" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="WebPage"
diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index a399bae10fb..bc891bfe421 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -3,8 +3,6 @@
 
   export let title
   export let align = "left"
-  export let minWidth
-  export let maxWidth
   export let showPopover
 
   let popover
@@ -24,11 +22,13 @@
 <Popover
   bind:this={popover}
   bind:open
-  minWidth={minWidth || 300}
+  minWidth={400}
+  maxWidth={400}
   {anchor}
   {align}
-  {maxWidth}
   {showPopover}
+  on:open
+  on:close
 >
   <div class="detail-popover">
     <div class="detail-popover__header">
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
new file mode 100644
index 00000000000..78f1b7edeea
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -0,0 +1,103 @@
+<script>
+  import DetailPopover from "components/common/DetailPopover.svelte"
+  import { Input, notifications, Button, Icon } from "@budibase/bbui"
+  import { goto } from "@roxi/routify"
+  import { viewsV2 } from "stores/builder"
+
+  export let table
+  export let firstView = false
+
+  let name
+  let popover
+
+  $: views = Object.keys(table?.views || {}).map(x => x.toLowerCase())
+  $: trimmedName = name?.trim()
+  $: nameExists = views.includes(trimmedName?.toLowerCase())
+  $: nameValid = trimmedName?.length && !nameExists
+
+  export const show = () => {
+    popover?.show()
+  }
+
+  const enrichSchema = schema => {
+    // We need to sure that "visible" is set to true for any fields which have
+    // not yet been saved with grid metadata attached
+    const cloned = { ...schema }
+    Object.entries(cloned).forEach(([field, fieldSchema]) => {
+      if (fieldSchema.visible == null) {
+        cloned[field] = { ...cloned[field], visible: true }
+      }
+    })
+    return cloned
+  }
+
+  const saveView = async () => {
+    popover.hide()
+    try {
+      const newView = await viewsV2.create({
+        name: trimmedName,
+        tableId: table._id,
+        schema: enrichSchema(table.schema),
+        primaryDisplay: table.primaryDisplay,
+      })
+      notifications.success(`View ${name} created`)
+      $goto(`./${newView.id}`)
+    } catch (error) {
+      notifications.error("Error creating view")
+    }
+  }
+</script>
+
+<DetailPopover
+  title="Create view"
+  bind:this={popover}
+  on:open={() => (name = null)}
+>
+  <svelte:fragment slot="anchor" let:open>
+    {#if firstView}
+      <Button cta>Create a view</Button>
+    {:else}
+      <div class="icon" class:open>
+        <Icon
+          name="Add"
+          size="L"
+          color="var(--spectrum-global-color-gray-600)"
+        />
+      </div>
+    {/if}
+  </svelte:fragment>
+  <Input
+    label="Name"
+    thin
+    bind:value={name}
+    error={nameExists ? "A view already exists with that name" : null}
+    autofocus
+  />
+  <div>
+    <Button cta on:click={saveView} disabled={nameExists || !nameValid}>
+      Create view
+    </Button>
+  </div>
+</DetailPopover>
+
+<style>
+  .icon {
+    height: 32px;
+    padding: 0 8px;
+    border-radius: 4px;
+    display: grid;
+    place-items: center;
+    transition: background 130ms ease-out;
+  }
+  .icon:hover,
+  .icon:active,
+  .icon.open {
+    background: var(--spectrum-global-color-gray-300);
+    cursor: pointer;
+  }
+  .icon:hover :global(.spectrum-Icon),
+  .icon:active :global(.spectrum-Icon),
+  .icon.open :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
index a37b5ccf828..49f5fcd1ff6 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
@@ -1,64 +1,2 @@
 <script>
-  import { Modal, Input, notifications, ModalContent } from "@budibase/bbui"
-  import { goto } from "@roxi/routify"
-  import { viewsV2 } from "stores/builder"
-
-  export let table
-
-  let name
-  let modal
-
-  $: views = Object.keys(table?.views || {}).map(x => x.toLowerCase())
-  $: trimmedName = name?.trim()
-  $: nameExists = views.includes(trimmedName?.toLowerCase())
-  $: nameValid = trimmedName?.length && !nameExists
-
-  export const show = () => {
-    name = null
-    modal?.show()
-  }
-
-  const enrichSchema = schema => {
-    // We need to sure that "visible" is set to true for any fields which have
-    // not yet been saved with grid metadata attached
-    const cloned = { ...schema }
-    Object.entries(cloned).forEach(([field, fieldSchema]) => {
-      if (fieldSchema.visible == null) {
-        cloned[field] = { ...cloned[field], visible: true }
-      }
-    })
-    return cloned
-  }
-
-  const saveView = async () => {
-    try {
-      const newView = await viewsV2.create({
-        name: trimmedName,
-        tableId: table._id,
-        schema: enrichSchema(table.schema),
-        primaryDisplay: table.primaryDisplay,
-      })
-      notifications.success(`View ${name} created`)
-      name = null
-      $goto(`./${newView.id}`)
-    } catch (error) {
-      notifications.error("Error creating view")
-    }
-  }
 </script>
-
-<Modal bind:this={modal}>
-  <ModalContent
-    title="Create view"
-    confirmText="Create view"
-    onConfirm={saveView}
-    disabled={!nameValid}
-  >
-    <Input
-      label="View name"
-      thin
-      bind:value={name}
-      error={nameExists ? "A view already exists with that name" : null}
-    />
-  </ModalContent>
-</Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte
index 0809d558849..0f39fa063d3 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte
@@ -39,7 +39,7 @@
 </script>
 
 <Modal bind:this={editorModal} on:show={initForm}>
-  <ModalContent title="Edit View" onConfirm={save} confirmText="Save">
-    <Input label="View Name" thin bind:value={updatedName} />
+  <ModalContent title="Edit view" onConfirm={save} confirmText="Save">
+    <Input label="Name" thin bind:value={updatedName} />
   </ModalContent>
 </Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index b1fffebbfeb..4d48dd6a9bd 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -26,6 +26,7 @@
   import CreateViewModal from "./CreateViewModal.svelte"
   import { onDestroy } from "svelte"
   import { derived } from "svelte/store"
+  import CreateViewButton from "./CreateViewButton.svelte"
 
   // View overflow
   let observer
@@ -34,7 +35,6 @@
   let overflowMenu
 
   // Editing table
-  let createViewModal
   let editTableModal
   let deleteTableModal
 
@@ -268,7 +268,7 @@
     </div>
   {/if}
   {#if !hasViews && tableEditable}
-    <Button cta on:click={createViewModal?.show}>Create a view</Button>
+    <CreateViewButton firstView {table} />
     <span>
       To create subsets of data, control access and more, create a view.
     </span>
@@ -311,19 +311,11 @@
     </ActionMenu>
   {/if}
   {#if hasViews}
-    <Icon
-      name="Add"
-      size="L"
-      hoverable
-      color="var(--spectrum-global-color-gray-600)"
-      hoverColor="var(--spectrum-global-color-gray-900)"
-      on:click={createViewModal?.show}
-    />
+    <CreateViewButton firstView={false} {table} />
   {/if}
 </div>
 
 {#if table && tableEditable}
-  <CreateViewModal {table} bind:this={createViewModal} />
   <EditTableModal {table} bind:this={editTableModal} />
   <DeleteTableModal {table} bind:this={deleteTableModal} />
 {/if}
@@ -343,7 +335,7 @@
     justify-content: flex-start;
     align-items: center;
     padding: 0 var(--spacing-xl);
-    gap: 12px;
+    gap: 8px;
   }
   .nav__views {
     flex: 0 1 auto;
@@ -353,7 +345,6 @@
     align-items: center;
     overflow: hidden;
     gap: 8px;
-    margin-left: -4px;
   }
 
   /* Table and view items */

From 3da0a04fa5f326dfb6f10fe6b07851b3aa2c3d5c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:13:41 +0100
Subject: [PATCH 036/455] Add comments

---
 packages/bbui/src/Popover/Popover.svelte | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte
index 6c6ae6817dd..cccd83f8712 100644
--- a/packages/bbui/src/Popover/Popover.svelte
+++ b/packages/bbui/src/Popover/Popover.svelte
@@ -35,6 +35,9 @@
 
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
   $: {
+    // Disable pointer events for the initial part of the animation, because we
+    // fly from top to bottom and initially can be positioned under the cursor,
+    // causing a flashing hover state in the content
     if (open && animate) {
       blockPointerEvents = true
       clearTimeout(timeout)

From 304c6fb58afb97f06ccde72786a603e6c293f2ac Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:30:22 +0100
Subject: [PATCH 037/455] Lint

---
 packages/bbui/src/ActionButton/ActionButton.svelte       | 1 -
 packages/bbui/src/ActionMenu/ActionMenu.svelte           | 2 +-
 packages/bbui/src/List/ListItem.svelte                   | 2 --
 .../data/table/[tableId]/_components/ViewNavBar.svelte   | 9 +--------
 .../data/table/[tableId]/v1/[viewName]/index.svelte      | 3 +--
 packages/frontend-core/src/api/rowActions.js             | 2 +-
 6 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte
index b8030b80311..2401354fbb1 100644
--- a/packages/bbui/src/ActionButton/ActionButton.svelte
+++ b/packages/bbui/src/ActionButton/ActionButton.svelte
@@ -30,7 +30,6 @@
   }
 </script>
 
-<!-- svelte-ignore a11y-no-static-element-interactions -->
 <button
   class="spectrum-ActionButton spectrum-ActionButton--size{size}"
   class:spectrum-ActionButton--quiet={quiet}
diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte
index e3de70e3ee6..f27854bc045 100644
--- a/packages/bbui/src/ActionMenu/ActionMenu.svelte
+++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte
@@ -22,7 +22,7 @@
     anchor = node.firstChild
   }
 
-  export const show = e => {
+  export const show = () => {
     cancelHide()
     dropdown.show()
   }
diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 341c53a190c..2b1f6b030fc 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -10,8 +10,6 @@
   export let showArrow = false
 </script>
 
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<!-- svelte-ignore a11y-click-events-have-key-events -->
 <a
   href={url}
   class="list-item"
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 4d48dd6a9bd..022e3c202a7 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -6,13 +6,7 @@
     contextMenuStore,
   } from "stores/builder"
   import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
-  import {
-    Icon,
-    Button,
-    ActionButton,
-    ActionMenu,
-    MenuItem,
-  } from "@budibase/bbui"
+  import { Icon, ActionButton, ActionMenu, MenuItem } from "@budibase/bbui"
   import { params, url } from "@roxi/routify"
   import EditViewModal from "./EditViewModal.svelte"
   import DeleteViewModal from "./DeleteViewModal.svelte"
@@ -23,7 +17,6 @@
   import { DB_TYPE_EXTERNAL } from "constants/backend"
   import { TableNames } from "constants"
   import { alphabetical } from "components/backend/TableNavigator/utils"
-  import CreateViewModal from "./CreateViewModal.svelte"
   import { onDestroy } from "svelte"
   import { derived } from "svelte/store"
   import CreateViewButton from "./CreateViewButton.svelte"
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
index e6aaa647de3..16bdfabb41e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
@@ -1,7 +1,6 @@
 <script>
-  import { views } from "stores/builder"
+  import { views, tables } from "stores/builder"
   import { API } from "api"
-  import { tables } from "stores/builder"
   import Table from "components/backend/DataTable/Table.svelte"
   import CalculateButton from "components/backend/DataTable/buttons/CalculateButton.svelte"
   import GroupByButton from "components/backend/DataTable/buttons/GroupByButton.svelte"
diff --git a/packages/frontend-core/src/api/rowActions.js b/packages/frontend-core/src/api/rowActions.js
index cf8808f6d00..5a1692cf453 100644
--- a/packages/frontend-core/src/api/rowActions.js
+++ b/packages/frontend-core/src/api/rowActions.js
@@ -55,7 +55,7 @@ export const buildRowActionEndpoints = API => ({
    * @param tableId the ID of the table
    * @param rowActionId the ID of the row action to trigger
    */
-  trigger: async ({ tableId, rowActionid }) => {
+  trigger: async ({ tableId, rowActionId }) => {
     return await API.post({
       url: `/api/tables/${tableId}/actions/${rowActionId}/trigger`,
     })

From c694b07ff4f331de4f8b46a8230733320de537d9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:38:08 +0100
Subject: [PATCH 038/455] Standardise hover and selected states between action
 buttons and icons in view nav bar

---
 packages/bbui/src/ActionMenu/ActionMenu.svelte           | 4 +++-
 packages/bbui/src/Icon/Icon.svelte                       | 5 +++--
 .../table/[tableId]/_components/CreateViewButton.svelte  | 9 ++++-----
 .../data/table/[tableId]/_components/ViewNavBar.svelte   | 4 ++--
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte
index f27854bc045..c8d0dde8f9e 100644
--- a/packages/bbui/src/ActionMenu/ActionMenu.svelte
+++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte
@@ -15,6 +15,7 @@
   let anchor
   let dropdown
   let timeout
+  let open
 
   // This is needed because display: contents is considered "invisible".
   // It should only ever be an action button, so should be fine.
@@ -63,10 +64,11 @@
   on:mouseenter={openOnHover ? show : null}
   on:mouseleave={openOnHover ? queueHide : null}
 >
-  <slot name="control" />
+  <slot name="control" {open} />
 </div>
 <Popover
   bind:this={dropdown}
+  bind:open
   {anchor}
   {align}
   {portalTarget}
diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte
index 6ae1f4ca67d..73ad8edd109 100644
--- a/packages/bbui/src/Icon/Icon.svelte
+++ b/packages/bbui/src/Icon/Icon.svelte
@@ -60,10 +60,11 @@
   .newStyles {
     color: var(--spectrum-global-color-gray-700);
   }
-
+  svg {
+    transition: color var(--spectrum-global-animation-duration-100, 130ms);
+  }
   svg.hoverable {
     pointer-events: all;
-    transition: color var(--spectrum-global-animation-duration-100, 130ms);
   }
   svg.hoverable:hover {
     color: var(--hover-color) !important;
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index 78f1b7edeea..8af5007022b 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -88,16 +88,15 @@
     display: grid;
     place-items: center;
     transition: background 130ms ease-out;
+    cursor: pointer;
   }
-  .icon:hover,
   .icon:active,
   .icon.open {
     background: var(--spectrum-global-color-gray-300);
-    cursor: pointer;
   }
-  .icon:hover :global(.spectrum-Icon),
-  .icon:active :global(.spectrum-Icon),
-  .icon.open :global(.spectrum-Icon) {
+  .icon:hover :global(svg),
+  .icon:active :global(svg),
+  .icon.open :global(svg) {
     color: var(--spectrum-global-color-gray-900) !important;
   }
 </style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 022e3c202a7..648f3dd6cd3 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -268,8 +268,8 @@
   {/if}
   {#if overflowedViews.length}
     <ActionMenu align="right" bind:this={overflowMenu}>
-      <div slot="control">
-        <ActionButton icon="ChevronDown" quiet selected={viewHidden}>
+      <div slot="control" let:open>
+        <ActionButton icon="ChevronDown" quiet selected={open || viewHidden}>
           {overflowedViews.length} more
         </ActionButton>
       </div>

From a11e2082f9f292f62ff5ae1798a6d8dacd268976 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:39:00 +0100
Subject: [PATCH 039/455] Lint

---
 .../data/table/[tableId]/_components/ViewNavBar.svelte         | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 648f3dd6cd3..e403e0bcd7e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -13,11 +13,10 @@
   import EditTableModal from "components/backend/TableNavigator/TableNavItem/EditModal.svelte"
   import DeleteTableModal from "components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte"
   import { UserAvatars } from "@budibase/frontend-core"
-  import { tick } from "svelte"
   import { DB_TYPE_EXTERNAL } from "constants/backend"
   import { TableNames } from "constants"
   import { alphabetical } from "components/backend/TableNavigator/utils"
-  import { onDestroy } from "svelte"
+  import { tick, onDestroy } from "svelte"
   import { derived } from "svelte/store"
   import CreateViewButton from "./CreateViewButton.svelte"
 

From a4fa619dda6734a0537557979f89494eebd437fe Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:41:05 +0100
Subject: [PATCH 040/455] Fix issue with text input autofocus and autofocus
 when creating views

---
 packages/bbui/src/Form/Core/TextField.svelte | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/bbui/src/Form/Core/TextField.svelte b/packages/bbui/src/Form/Core/TextField.svelte
index 3335d3567b5..917bb2a452e 100644
--- a/packages/bbui/src/Form/Core/TextField.svelte
+++ b/packages/bbui/src/Form/Core/TextField.svelte
@@ -1,6 +1,6 @@
 <script>
   import "@spectrum-css/textfield/dist/index-vars.css"
-  import { createEventDispatcher, onMount } from "svelte"
+  import { createEventDispatcher, onMount, tick } from "svelte"
 
   export let value = null
   export let placeholder = null
@@ -68,10 +68,13 @@
     return type === "number" ? "decimal" : "text"
   }
 
-  onMount(() => {
+  onMount(async () => {
     if (disabled) return
     focus = autofocus
-    if (focus) field.focus()
+    if (focus) {
+      await tick()
+      field.focus()
+    }
   })
 </script>
 

From 81e11382758b29d4357fb3ce3ff4168d89764ecc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:45:45 +0100
Subject: [PATCH 041/455] Lint

---
 packages/client/src/constants.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js
index bfa2d08dc59..f7e3e86d40c 100644
--- a/packages/client/src/constants.js
+++ b/packages/client/src/constants.js
@@ -3,8 +3,7 @@ export { FieldType as FieldTypes } from "@budibase/types"
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   UpdateFieldValue: "UpdateFieldValue",
-  RefreshDatasource:
-     "RefreshDatasource",
+  RefreshDatasource: "RefreshDatasource",
   AddDataProviderQueryExtension: "AddDataProviderQueryExtension",
   RemoveDataProviderQueryExtension: "RemoveDataProviderQueryExtension",
   SetDataProviderSorting: "SetDataProviderSorting",

From 42aa987ce30b6a1ff14776de2029a91e07b66335 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:48:26 +0100
Subject: [PATCH 042/455] Try to undo submodule changes, please stop failing

---
 packages/account-portal | 2 +-
 packages/pro            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index 32b8fa4643b..516b27b74cb 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 32b8fa4643b4f0f74ee89760deffe431ab347ad9
+Subproject commit 516b27b74cbcb7069a25f5e738dc91c22d7c4538
diff --git a/packages/pro b/packages/pro
index 94747fd5bb6..14c89a5b20e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 94747fd5bb67c218244bb60b9540f3a6f1c3f6f1
+Subproject commit 14c89a5b20ee4de07723063458a2437b0dd4f2c9

From 2ee3d60fe65e06ec48eddc731a98e158a71a8e9a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 14:49:19 +0100
Subject: [PATCH 043/455] Undo spectrum icon version bump

---
 packages/bbui/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index 8e640556b6e..0830f8ab6ff 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -34,7 +34,7 @@
     "dist"
   ],
   "dependencies": {
-    "@adobe/spectrum-css-workflow-icons": "3.0.0",
+    "@adobe/spectrum-css-workflow-icons": "1.2.1",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@spectrum-css/accordion": "3.0.24",

From 9fc837b04aa2dd99af27360b311673d39a03fd5a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 23 Aug 2024 15:11:25 +0100
Subject: [PATCH 044/455] Reduce spacing in new screen modal

---
 .../[application]/design/_components/NewScreen/TypeModal.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte
index 07a24291331..84ad4d9fe6b 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/TypeModal.svelte
@@ -67,9 +67,9 @@
     box-sizing: border-box;
     padding: var(--spacing-m) var(--spacing-xl);
     flex-grow: 1;
-    gap: var(--spacing-s);
     display: flex;
     flex-direction: column;
+    justify-content: center;
   }
 
   .image {

From c5e27b860ffcfc395e0127b9e88599865654f428 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 2 Sep 2024 08:45:49 +0100
Subject: [PATCH 045/455] Add row actions to tables in data section

---
 .../bbui/src/ActionMenu/ActionMenu.svelte     |  6 +--
 .../bbui/src/Actions/position_dropdown.js     |  2 +-
 .../data/table/[tableId]/index.svelte         | 54 ++++++++++++++++++-
 .../src/components/app/GridBlock.svelte       |  6 ++-
 packages/frontend-core/src/api/rowActions.js  |  5 +-
 .../grid/layout/ButtonColumn.svelte           | 49 ++++++++++-------
 6 files changed, 93 insertions(+), 29 deletions(-)

diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte
index c8d0dde8f9e..e3de70e3ee6 100644
--- a/packages/bbui/src/ActionMenu/ActionMenu.svelte
+++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte
@@ -15,7 +15,6 @@
   let anchor
   let dropdown
   let timeout
-  let open
 
   // This is needed because display: contents is considered "invisible".
   // It should only ever be an action button, so should be fine.
@@ -23,7 +22,7 @@
     anchor = node.firstChild
   }
 
-  export const show = () => {
+  export const show = e => {
     cancelHide()
     dropdown.show()
   }
@@ -64,11 +63,10 @@
   on:mouseenter={openOnHover ? show : null}
   on:mouseleave={openOnHover ? queueHide : null}
 >
-  <slot name="control" {open} />
+  <slot name="control" />
 </div>
 <Popover
   bind:this={dropdown}
-  bind:open
   {anchor}
   {align}
   {portalTarget}
diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js
index c3675cfbe6a..7a852f1b46a 100644
--- a/packages/bbui/src/Actions/position_dropdown.js
+++ b/packages/bbui/src/Actions/position_dropdown.js
@@ -169,7 +169,7 @@ export default function positionDropdown(element, opts) {
         align === "left-context-menu"
       ) {
         applyYStrategy(Strategies.StartToStart)
-        styles.top -= 4 // Manual adjustment for action menu padding
+        styles.top -= 5 // Manual adjustment for action menu padding
       } else {
         applyYStrategy(Strategies.StartToEnd)
       }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 81752387793..c71f20c9a8b 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Banner } from "@budibase/bbui"
+  import { Banner, Menu, MenuItem, Popover } from "@budibase/bbui"
   import { datasources, tables, integrations, appStore } from "stores/builder"
   import { themeStore, admin } from "stores/portal"
   import { TableNames } from "constants"
@@ -28,7 +28,12 @@
     status: { displayName: "Status", disabled: true },
   }
 
+  let rowActions = []
   let generateButton
+  let rowActionPopover
+  let rowActionRow
+  let rowActionAnchor
+  let refreshRow
 
   $: autoColumnStatus = verifyAutocolumns($tables?.selected)
   $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
@@ -53,6 +58,20 @@
   $: relationshipsEnabled = relationshipSupport(tableDatasource)
   $: currentTheme = $themeStore?.theme
   $: darkMode = !currentTheme.includes("light")
+  $: buttons = [
+    {
+      text: "Actions",
+      type: "cta",
+      icon: "ChevronDown",
+      onClick: async (e, row, refresh) => {
+        rowActionRow = row
+        rowActionAnchor = e.currentTarget
+        rowActionPopover.show()
+        refreshRow = refresh
+      },
+    },
+  ]
+  $: fetchRowActions(id)
 
   const relationshipSupport = datasource => {
     const integration = $integrations[datasource?.source]
@@ -82,6 +101,25 @@
       return acc
     }, {})
   }
+
+  const fetchRowActions = async tableId => {
+    if (!tableId) {
+      rowActions = []
+      return
+    }
+    const res = await API.rowActions.fetch(tableId)
+    rowActions = Object.values(res || {})
+  }
+
+  const runRowAction = async action => {
+    await API.rowActions.trigger({
+      rowActionId: action.id,
+      tableId: id,
+      rowId: rowActionRow._id,
+    })
+    await refreshRow()
+    rowActionPopover.hide()
+  }
 </script>
 
 {#if $tables?.selected?.name}
@@ -105,6 +143,7 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
+    buttons={rowActions.length ? buttons : null}
     on:updatedatasource={handleGridTableUpdate}
   >
     <!-- Controls -->
@@ -153,6 +192,19 @@
   <i>Create your first table to start building</i>
 {/if}
 
+<Popover
+  bind:this={rowActionPopover}
+  align="right"
+  anchor={rowActionAnchor}
+  offset={5}
+>
+  <Menu>
+    {#each rowActions as action}
+      <MenuItem on:click={() => runRowAction(action)}>{action.name}</MenuItem>
+    {/each}
+  </Menu>
+</Popover>
+
 <style>
   i {
     font-size: var(--font-size-m);
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 30a35b07136..17d0f4e4cc3 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -115,13 +115,15 @@
       text: settings.text,
       type: settings.type,
       icon: settings.icon,
-      onClick: async row => {
+      onClick: async (_, row, refresh) => {
         // Create a fake, ephemeral context to run the buttons actions with
         const id = get(component).id
         const gridContext = createContextStore(context)
         gridContext.actions.provideData(id, row)
         const fn = enrichButtonActions(settings.onClick, get(gridContext))
-        return await fn?.({ row })
+        const res = await fn?.({ row })
+        await refresh()
+        return res
       },
     }))
   }
diff --git a/packages/frontend-core/src/api/rowActions.js b/packages/frontend-core/src/api/rowActions.js
index 5a1692cf453..85ae4f6b96e 100644
--- a/packages/frontend-core/src/api/rowActions.js
+++ b/packages/frontend-core/src/api/rowActions.js
@@ -55,9 +55,12 @@ export const buildRowActionEndpoints = API => ({
    * @param tableId the ID of the table
    * @param rowActionId the ID of the row action to trigger
    */
-  trigger: async ({ tableId, rowActionId }) => {
+  trigger: async ({ tableId, rowActionId, rowId }) => {
     return await API.post({
       url: `/api/tables/${tableId}/actions/${rowActionId}/trigger`,
+      body: {
+        rowId,
+      },
     })
   },
 })
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index f448460a963..8851b8ff1c2 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -18,6 +18,7 @@
     isDragging,
     buttonColumnWidth,
     showVScrollbar,
+    showHScrollbar,
     dispatch,
   } = getContext("grid")
 
@@ -32,10 +33,12 @@
   $: gridEnd = $width - $buttonColumnWidth - 1
   $: left = Math.min(columnEnd, gridEnd)
 
-  const handleClick = async (button, row) => {
-    await button.onClick?.(rows.actions.cleanRow(row))
-    // Refresh the row in case it changed
-    await rows.actions.refreshRow(row._id)
+  const handleClick = async (e, button, row) => {
+    await button.onClick?.(
+      e,
+      rows.actions.cleanRow(row),
+      async () => await rows.actions.refreshRow(row._id)
+    )
   }
 
   onMount(() => {
@@ -72,23 +75,29 @@
             highlighted={rowHovered || rowFocused}
             metadata={row.__metadata?.row}
           >
-            <div class="buttons" class:offset={$showVScrollbar}>
+            <div
+              class="buttons"
+              class:offset={$showVScrollbar && $showHScrollbar}
+            >
               {#each buttons as button}
-                <Button
-                  newStyles
-                  size="S"
-                  cta={button.type === "cta"}
-                  primary={button.type === "primary"}
-                  secondary={button.type === "secondary"}
-                  warning={button.type === "warning"}
-                  overBackground={button.type === "overBackground"}
-                  on:click={() => handleClick(button, row)}
-                >
-                  {#if button.icon}
-                    <i class="{button.icon} S" />
-                  {/if}
-                  {button.text || "Button"}
-                </Button>
+                <!-- svelte-ignore a11y-click-events-have-key-events -->
+                <span on:click={e => handleClick(e, button, row)}>
+                  <Button
+                    newStyles
+                    size="S"
+                    icon={button.icon}
+                    cta={button.type === "cta"}
+                    primary={button.type === "primary"}
+                    secondary={button.type === "secondary"}
+                    warning={button.type === "warning"}
+                    overBackground={button.type === "overBackground"}
+                  >
+                    {#if button.icon}
+                      <i class="{button.icon} S" />
+                    {/if}
+                    {button.text || "Button"}
+                  </Button>
+                </span>
               {/each}
             </div>
           </GridCell>

From 2d52eff96f57255460a1f7d38ba221dc9a7db7b2 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 2 Sep 2024 08:47:15 +0100
Subject: [PATCH 046/455] Update button column to account for ability to add
 rows

---
 .../grid/layout/ButtonColumn.svelte           | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 8851b8ff1c2..bb8dd30b93b 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -20,6 +20,7 @@
     showVScrollbar,
     showHScrollbar,
     dispatch,
+    config,
   } = getContext("grid")
 
   let container
@@ -103,17 +104,21 @@
           </GridCell>
         </div>
       {/each}
-      <div
-        class="row blank"
-        on:mouseenter={$isDragging ? null : () => ($hoveredRowId = BlankRowID)}
-        on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-      >
-        <GridCell
-          width={$buttonColumnWidth}
-          highlighted={$hoveredRowId === BlankRowID}
-          on:click={() => dispatch("add-row-inline")}
-        />
-      </div>
+      {#if $config.canAddRows}
+        <div
+          class="row blank"
+          on:mouseenter={$isDragging
+            ? null
+            : () => ($hoveredRowId = BlankRowID)}
+          on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+        >
+          <GridCell
+            width={$buttonColumnWidth}
+            highlighted={$hoveredRowId === BlankRowID}
+            on:click={() => dispatch("add-row-inline")}
+          />
+        </div>
+      {/if}
     </GridScrollWrapper>
   </div>
 </div>

From df038417b54c9368eb346efdd801ffe70619a3a9 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 2 Sep 2024 09:02:28 +0100
Subject: [PATCH 047/455] Merge commit

---
 packages/frontend-core/src/constants.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index 22e5e8583a3..05f501cfbae 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -161,3 +161,13 @@ export const TypeIconMap = {
 export const OptionColours = [...new Array(12).keys()].map(idx => {
   return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, 0.3)`
 })
+
+export const FilterOperator = {
+  ANY: "any",
+  ALL: "all",
+}
+
+export const OnEmptyFilter = {
+  RETURN_ALL: "all",
+  RETURN_NONE: "none",
+}

From fae897f4685947872d3cc8d2ad2d77842bea6e32 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 2 Sep 2024 09:31:19 +0100
Subject: [PATCH 048/455] Allow button groups to be collapsed

---
 .../bbui/src/Actions/position_dropdown.js     |   2 +-
 packages/bbui/src/Button/Button.svelte        |   2 +
 .../data/table/[tableId]/index.svelte         |   2 +-
 packages/client/manifest.json                 |  48 +++++++-
 .../src/components/app/ButtonGroup.svelte     | 108 ++++++++++++------
 .../src/components/grid/cells/GridCell.svelte |   7 +-
 .../grid/layout/ButtonColumn.svelte           |   4 +-
 7 files changed, 129 insertions(+), 44 deletions(-)

diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js
index 7a852f1b46a..e95c7dd1b62 100644
--- a/packages/bbui/src/Actions/position_dropdown.js
+++ b/packages/bbui/src/Actions/position_dropdown.js
@@ -246,7 +246,7 @@ export default function positionDropdown(element, opts) {
   }
 
   // Apply initial styles which don't need to change
-  element.style.position = "absolute"
+  element.style.position = "fixed"
   element.style.zIndex = "9999"
 
   // Set up a scroll listener
diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte
index 9e49d84d443..7971c0ea8d8 100644
--- a/packages/bbui/src/Button/Button.svelte
+++ b/packages/bbui/src/Button/Button.svelte
@@ -17,6 +17,7 @@
   export let tooltip = undefined
   export let newStyles = true
   export let id
+  export let ref
 
   const dispatch = createEventDispatcher()
 </script>
@@ -25,6 +26,7 @@
   <button
     {id}
     {type}
+    bind:this={ref}
     class:spectrum-Button--cta={cta}
     class:spectrum-Button--primary={primary}
     class:spectrum-Button--secondary={secondary}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index c71f20c9a8b..a157aef463e 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -60,7 +60,7 @@
   $: darkMode = !currentTheme.includes("light")
   $: buttons = [
     {
-      text: "Actions",
+      text: "Action",
       type: "cta",
       icon: "ChevronDown",
       onClick: async (e, row, refresh) => {
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index b1552a974ae..70479530c4c 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -447,6 +447,18 @@
         "section": true,
         "name": "Layout",
         "settings": [
+          {
+            "type": "boolean",
+            "label": "Collapse",
+            "key": "collapsed"
+          },
+          {
+            "type": "text",
+            "label": "Collapsed text",
+            "key": "collapsedText",
+            "dependsOn": "collapsed",
+            "placeholder": "Action"
+          },
           {
             "type": "select",
             "label": "Direction",
@@ -467,7 +479,11 @@
                 "barTitle": "Row layout"
               }
             ],
-            "defaultValue": "row"
+            "defaultValue": "row",
+            "dependsOn": {
+              "setting": "collapsed",
+              "invert": true
+            }
           },
           {
             "type": "select",
@@ -501,7 +517,11 @@
                 "barTitle": "Align stretched horizontally"
               }
             ],
-            "defaultValue": "left"
+            "defaultValue": "left",
+            "dependsOn": {
+              "setting": "collapsed",
+              "invert": true
+            }
           },
           {
             "type": "select",
@@ -535,7 +555,11 @@
                 "barTitle": "Align stretched vertically"
               }
             ],
-            "defaultValue": "top"
+            "defaultValue": "top",
+            "dependsOn": {
+              "setting": "collapsed",
+              "invert": true
+            }
           },
           {
             "type": "select",
@@ -557,7 +581,11 @@
                 "barTitle": "Grow container"
               }
             ],
-            "defaultValue": "shrink"
+            "defaultValue": "shrink",
+            "dependsOn": {
+              "setting": "collapsed",
+              "invert": true
+            }
           },
           {
             "type": "select",
@@ -583,7 +611,11 @@
                 "value": "L"
               }
             ],
-            "defaultValue": "M"
+            "defaultValue": "M",
+            "dependsOn": {
+              "setting": "collapsed",
+              "invert": true
+            }
           },
           {
             "type": "boolean",
@@ -591,7 +623,11 @@
             "key": "wrap",
             "showInBar": true,
             "barIcon": "ModernGridView",
-            "barTitle": "Wrap"
+            "barTitle": "Wrap",
+            "dependsOn": {
+              "setting": "collapsed",
+              "invert": true
+            }
           }
         ]
       }
diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
index b3523cdd21f..2917d1524b1 100644
--- a/packages/client/src/components/app/ButtonGroup.svelte
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -1,44 +1,86 @@
 <script>
   import BlockComponent from "../BlockComponent.svelte"
   import Block from "../Block.svelte"
+  import { Button, Popover, Menu, MenuItem } from "@budibase/bbui"
+  import { getContext } from "svelte"
 
   export let buttons = []
   export let direction = "row"
   export let hAlign = "left"
   export let vAlign = "top"
   export let gap = "S"
+  export let collapsed = false
+  export let collapsedText = "Action"
+
+  const { styleable, enrichButtonActions } = getContext("sdk")
+  const component = getContext("component")
+  const context = getContext("context")
+
+  let popover
+  let anchor
+
+  const handleCollapsedClick = async button => {
+    const fn = enrichButtonActions(button.onClick, $context)
+    await fn?.()
+    popover.hide()
+  }
 </script>
 
-<Block>
-  <BlockComponent
-    type="container"
-    props={{
-      direction,
-      hAlign,
-      vAlign,
-      gap,
-      wrap: true,
-    }}
-    styles={{
-      normal: {
-        height: "100%",
-      },
-    }}
-  >
-    {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }}
-      <BlockComponent
-        type="button"
-        props={{
-          text: text || "Button",
-          onClick,
-          type,
-          quiet,
-          disabled,
-          icon,
-          gap,
-          size: size || "M",
-        }}
-      />
-    {/each}
-  </BlockComponent>
-</Block>
+{#if collapsed}
+  <div use:styleable={$component.styles}>
+    <Button
+      bind:ref={anchor}
+      on:click={() => popover?.show()}
+      icon="ChevronDown"
+      cta
+    >
+      {collapsedText || "Action"}
+    </Button>
+  </div>
+  <Popover bind:this={popover} align="left" {anchor}>
+    <Menu>
+      {#each buttons as button}
+        <MenuItem
+          on:click={() => handleCollapsedClick(button)}
+          disabled={button.disabled}
+        >
+          {button.text || "Button"}
+        </MenuItem>
+      {/each}
+    </Menu>
+  </Popover>
+{:else}
+  <Block>
+    <BlockComponent
+      type="container"
+      props={{
+        direction,
+        hAlign,
+        vAlign,
+        gap,
+        wrap: true,
+      }}
+      styles={{
+        normal: {
+          height: "100%",
+        },
+      }}
+    >
+      {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }}
+        <BlockComponent
+          type="button"
+          props={{
+            text: text || "Button",
+            onClick,
+            type,
+            quiet,
+            disabled,
+            icon,
+            gap,
+            size: size || "M",
+          }}
+        />
+      {/each}
+    </BlockComponent>
+  </Block>
+{/if}
diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte
index 7b5deccb35d..68f98648b41 100644
--- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte
@@ -16,7 +16,12 @@
   $: style = getStyle(width, selectedUser, metadata)
 
   const getStyle = (width, selectedUser, metadata) => {
-    let style = width === "auto" ? "width: auto;" : `flex: 0 0 ${width}px;`
+    let style
+    if (width === "auto" || width === "100%") {
+      style = `width: ${width};`
+    } else {
+      style = `flex: 0 0 ${width}px;`
+    }
     if (selectedUser) {
       style += `--user-color :${selectedUser.color};`
     }
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index bb8dd30b93b..22a7e0d6883 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -45,7 +45,7 @@
   onMount(() => {
     const observer = new ResizeObserver(entries => {
       const width = entries?.[0]?.contentRect?.width ?? 0
-      buttonColumnWidth.set(Math.floor(width) - 1)
+      buttonColumnWidth.set(width)
     })
     observer.observe(container)
   })
@@ -113,7 +113,7 @@
           on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
         >
           <GridCell
-            width={$buttonColumnWidth}
+            width="100%"
             highlighted={$hoveredRowId === BlankRowID}
             on:click={() => dispatch("add-row-inline")}
           />

From e6f6760c2a17815707d3ddeb2989a7f555ad1d67 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 2 Sep 2024 09:38:34 +0100
Subject: [PATCH 049/455] Add collapsable buttons to form block

---
 packages/client/manifest.json                        | 12 ++++++++++++
 .../src/components/app/blocks/form/FormBlock.svelte  |  4 ++++
 .../components/app/blocks/form/InnerFormBlock.svelte |  6 ++++++
 3 files changed, 22 insertions(+)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 70479530c4c..f6911614261 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -7188,6 +7188,18 @@
             "key": "buttons",
             "nested": true,
             "resetOn": ["actionType", "dataSource"]
+          },
+          {
+            "type": "boolean",
+            "label": "Collapse",
+            "key": "buttonsCollapsed"
+          },
+          {
+            "type": "text",
+            "label": "Collapsed text",
+            "key": "buttonsCollapsedText",
+            "dependsOn": "buttonsCollapsed",
+            "placeholder": "Action"
           }
         ]
       },
diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte
index e3aa20ffa6b..656aa5933bc 100644
--- a/packages/client/src/components/app/blocks/form/FormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte
@@ -18,6 +18,8 @@
   export let actionUrl
   export let noRowsMessage
   export let notificationOverride
+  export let buttonsCollapsed
+  export let buttonsCollapsedText
 
   // Legacy
   export let showDeleteButton
@@ -118,5 +120,7 @@
     {notificationOverride}
     buttons={buttonsOrDefault}
     buttonPosition={buttons ? buttonPosition : "top"}
+    {buttonsCollapsed}
+    {buttonsCollapsedText}
   />
 </FormBlockWrapper>
diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
index 0227107dd27..b5b65398b65 100644
--- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
@@ -13,6 +13,8 @@
   export let description
   export let buttons
   export let buttonPosition = "bottom"
+  export let buttonsCollapsed
+  export let buttonsCollapsedText
   export let schema
 
   const context = getContext("context")
@@ -81,6 +83,8 @@
                 type="buttongroup"
                 props={{
                   buttons,
+                  collapsed: buttonsCollapsed,
+                  collapsedText: buttonsCollapsedText,
                 }}
                 order={0}
               />
@@ -104,6 +108,8 @@
         type="buttongroup"
         props={{
           buttons,
+          collapsed: buttonsCollapsed,
+          collapsedText: buttonsCollapsedText,
         }}
         styles={{
           normal: {

From 51cbced119e28a3195470398c742dc195d18ec24 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 2 Sep 2024 11:44:49 +0100
Subject: [PATCH 050/455] Add common collapsed button group and use it for row
 actions and client button groups. Add collapsed settings to grids and form
 blocks

---
 .../ButtonGroup/CollapsedButtonGroup.svelte   | 39 +++++++++++++
 packages/bbui/src/index.js                    |  1 +
 .../data/table/[tableId]/index.svelte         | 57 ++++++-------------
 packages/client/manifest.json                 | 12 ++++
 .../src/components/app/ButtonGroup.svelte     | 47 +++++++--------
 .../src/components/app/GridBlock.svelte       | 10 ++--
 .../grid/layout/ButtonColumn.svelte           | 36 ++++++++----
 .../src/components/grid/layout/Grid.svelte    |  6 +-
 8 files changed, 123 insertions(+), 85 deletions(-)
 create mode 100644 packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte

diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
new file mode 100644
index 00000000000..f1860ecaeaa
--- /dev/null
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -0,0 +1,39 @@
+<script>
+  import Button from "../Button/Button.svelte"
+  import Popover from "../Popover/Popover.svelte"
+  import Menu from "../Menu/Menu.svelte"
+  import MenuItem from "../Menu/Item.svelte"
+
+  export let buttons
+  export let text = "Action"
+  export let size = "M"
+  export let align = "left"
+  export let offset
+
+  let anchor
+  let popover
+
+  const handleClick = async button => {
+    popover.hide()
+    await button.onClick?.()
+  }
+</script>
+
+<Button
+  bind:ref={anchor}
+  {size}
+  icon="ChevronDown"
+  cta
+  on:click={() => popover?.show()}
+>
+  {text || "Action"}
+</Button>
+<Popover bind:this={popover} {align} {anchor} {offset}>
+  <Menu>
+    {#each buttons as button}
+      <MenuItem on:click={() => handleClick(button)} disabled={button.disabled}>
+        {button.text || "Button"}
+      </MenuItem>
+    {/each}
+  </Menu>
+</Popover>
diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js
index 7a4ff3f0814..083b4ecfd50 100644
--- a/packages/bbui/src/index.js
+++ b/packages/bbui/src/index.js
@@ -39,6 +39,7 @@ export { default as ActionGroup } from "./ActionGroup/ActionGroup.svelte"
 export { default as ActionMenu } from "./ActionMenu/ActionMenu.svelte"
 export { default as Button } from "./Button/Button.svelte"
 export { default as ButtonGroup } from "./ButtonGroup/ButtonGroup.svelte"
+export { default as CollapsedButtonGroup } from "./ButtonGroup/CollapsedButtonGroup.svelte"
 export { default as ClearButton } from "./ClearButton/ClearButton.svelte"
 export { default as Icon } from "./Icon/Icon.svelte"
 export { default as IconAvatar } from "./Icon/IconAvatar.svelte"
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index a157aef463e..b89becaa030 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -30,10 +30,6 @@
 
   let rowActions = []
   let generateButton
-  let rowActionPopover
-  let rowActionRow
-  let rowActionAnchor
-  let refreshRow
 
   $: autoColumnStatus = verifyAutocolumns($tables?.selected)
   $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
@@ -58,21 +54,22 @@
   $: relationshipsEnabled = relationshipSupport(tableDatasource)
   $: currentTheme = $themeStore?.theme
   $: darkMode = !currentTheme.includes("light")
-  $: buttons = [
-    {
-      text: "Action",
-      type: "cta",
-      icon: "ChevronDown",
-      onClick: async (e, row, refresh) => {
-        rowActionRow = row
-        rowActionAnchor = e.currentTarget
-        rowActionPopover.show()
-        refreshRow = refresh
-      },
-    },
-  ]
+  $: buttons = makeRowActionButtons(rowActions)
   $: fetchRowActions(id)
 
+  const makeRowActionButtons = rowActions => {
+    return rowActions.map(action => ({
+      text: action.name,
+      onClick: async row => {
+        await API.rowActions.trigger({
+          rowActionId: action.id,
+          tableId: id,
+          rowId: row._id,
+        })
+      },
+    }))
+  }
+
   const relationshipSupport = datasource => {
     const integration = $integrations[datasource?.source]
     return !isInternal && integration?.relationships !== false
@@ -110,16 +107,6 @@
     const res = await API.rowActions.fetch(tableId)
     rowActions = Object.values(res || {})
   }
-
-  const runRowAction = async action => {
-    await API.rowActions.trigger({
-      rowActionId: action.id,
-      tableId: id,
-      rowId: rowActionRow._id,
-    })
-    await refreshRow()
-    rowActionPopover.hide()
-  }
 </script>
 
 {#if $tables?.selected?.name}
@@ -143,7 +130,8 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
-    buttons={rowActions.length ? buttons : null}
+    {buttons}
+    buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}
   >
     <!-- Controls -->
@@ -192,19 +180,6 @@
   <i>Create your first table to start building</i>
 {/if}
 
-<Popover
-  bind:this={rowActionPopover}
-  align="right"
-  anchor={rowActionAnchor}
-  offset={5}
->
-  <Menu>
-    {#each rowActions as action}
-      <MenuItem on:click={() => runRowAction(action)}>{action.name}</MenuItem>
-    {/each}
-  </Menu>
-</Popover>
-
 <style>
   i {
     font-size: var(--font-size-m);
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index f6911614261..de70e98357f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -7591,6 +7591,18 @@
                 "key": "row"
               }
             ]
+          },
+          {
+            "type": "boolean",
+            "label": "Collapse",
+            "key": "buttonsCollapsed"
+          },
+          {
+            "type": "text",
+            "label": "Collapsed text",
+            "key": "buttonsCollapsedText",
+            "dependsOn": "buttonsCollapsed",
+            "placeholder": "Action"
           }
         ]
       }
diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
index 2917d1524b1..edcf5284323 100644
--- a/packages/client/src/components/app/ButtonGroup.svelte
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -1,7 +1,13 @@
 <script>
   import BlockComponent from "../BlockComponent.svelte"
   import Block from "../Block.svelte"
-  import { Button, Popover, Menu, MenuItem } from "@budibase/bbui"
+  import {
+    Button,
+    Popover,
+    Menu,
+    MenuItem,
+    CollapsedButtonGroup,
+  } from "@budibase/bbui"
   import { getContext } from "svelte"
 
   export let buttons = []
@@ -16,39 +22,26 @@
   const component = getContext("component")
   const context = getContext("context")
 
-  let popover
-  let anchor
+  $: collapsedButtons = collapsed ? makeCollapsed(buttons) : null
 
-  const handleCollapsedClick = async button => {
-    const fn = enrichButtonActions(button.onClick, $context)
-    await fn?.()
-    popover.hide()
+  const makeCollapsed = buttons => {
+    return buttons.map(button => ({
+      ...button,
+      onClick: async () => {
+        const fn = enrichButtonActions(button.onClick, $context)
+        await fn?.()
+      },
+    }))
   }
 </script>
 
 {#if collapsed}
   <div use:styleable={$component.styles}>
-    <Button
-      bind:ref={anchor}
-      on:click={() => popover?.show()}
-      icon="ChevronDown"
-      cta
-    >
-      {collapsedText || "Action"}
-    </Button>
+    <CollapsedButtonGroup
+      text={collapsedText || "Action"}
+      buttons={collapsedButtons}
+    />
   </div>
-  <Popover bind:this={popover} align="left" {anchor}>
-    <Menu>
-      {#each buttons as button}
-        <MenuItem
-          on:click={() => handleCollapsedClick(button)}
-          disabled={button.disabled}
-        >
-          {button.text || "Button"}
-        </MenuItem>
-      {/each}
-    </Menu>
-  </Popover>
 {:else}
   <Block>
     <BlockComponent
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 17d0f4e4cc3..543bade3827 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -19,6 +19,8 @@
   export let columns = null
   export let onRowClick = null
   export let buttons = null
+  export let buttonsCollapsed = false
+  export let buttonsCollapsedText = null
 
   const context = getContext("context")
   const component = getContext("component")
@@ -115,15 +117,13 @@
       text: settings.text,
       type: settings.type,
       icon: settings.icon,
-      onClick: async (_, row, refresh) => {
+      onClick: async row => {
         // Create a fake, ephemeral context to run the buttons actions with
         const id = get(component).id
         const gridContext = createContextStore(context)
         gridContext.actions.provideData(id, row)
         const fn = enrichButtonActions(settings.onClick, get(gridContext))
-        const res = await fn?.({ row })
-        await refresh()
-        return res
+        return await fn?.({ row })
       },
     }))
   }
@@ -183,6 +183,8 @@
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
     buttons={enrichedButtons}
+    {buttonsCollapsed}
+    {buttonsCollapsedText}
     isCloud={$environmentStore.cloud}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 22a7e0d6883..7b5eb58fbe8 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -4,6 +4,7 @@
   import GridCell from "../cells/GridCell.svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import { BlankRowID } from "../lib/constants"
+  import CollapsedButtonGroup from "../../../../../bbui/src/ButtonGroup/CollapsedButtonGroup.svelte"
 
   const {
     renderedRows,
@@ -34,12 +35,16 @@
   $: gridEnd = $width - $buttonColumnWidth - 1
   $: left = Math.min(columnEnd, gridEnd)
 
-  const handleClick = async (e, button, row) => {
-    await button.onClick?.(
-      e,
-      rows.actions.cleanRow(row),
-      async () => await rows.actions.refreshRow(row._id)
-    )
+  const handleClick = async (button, row) => {
+    await button.onClick?.(rows.actions.cleanRow(row))
+    await rows.actions.refreshRow(row._id)
+  }
+
+  const makeCollapsedButtons = (buttons, row) => {
+    return buttons.map(button => ({
+      ...button,
+      onClick: () => handleClick(button, row),
+    }))
   }
 
   onMount(() => {
@@ -80,26 +85,33 @@
               class="buttons"
               class:offset={$showVScrollbar && $showHScrollbar}
             >
-              {#each buttons as button}
-                <!-- svelte-ignore a11y-click-events-have-key-events -->
-                <span on:click={e => handleClick(e, button, row)}>
+              {#if $props.buttonsCollapsed}
+                <CollapsedButtonGroup
+                  buttons={makeCollapsedButtons(buttons, row)}
+                  text={$props.buttonsCollapsedText || "Action"}
+                  align="right"
+                  offset={5}
+                  size="S"
+                />
+              {:else}
+                {#each buttons as button}
                   <Button
                     newStyles
                     size="S"
-                    icon={button.icon}
                     cta={button.type === "cta"}
                     primary={button.type === "primary"}
                     secondary={button.type === "secondary"}
                     warning={button.type === "warning"}
                     overBackground={button.type === "overBackground"}
+                    on:click={() => handleClick(button, row)}
                   >
                     {#if button.icon}
                       <i class="{button.icon} S" />
                     {/if}
                     {button.text || "Button"}
                   </Button>
-                </span>
-              {/each}
+                {/each}
+              {/if}
             </div>
           </GridCell>
         </div>
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index b330f4f0332..7e28f249da4 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -43,7 +43,9 @@
   export let notifySuccess = null
   export let notifyError = null
   export let buttons = null
-  export let darkMode
+  export let buttonsCollapsed = false
+  export let buttonsCollapsedText = null
+  export let darkMode = false
   export let isCloud = null
   export let rowConditions = null
 
@@ -98,6 +100,8 @@
     notifySuccess,
     notifyError,
     buttons,
+    buttonsCollapsed,
+    buttonsCollapsedText,
     darkMode,
     isCloud,
     rowConditions,

From 3b827c35a26e14bd73aca2295e2aed71a71c366b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 2 Sep 2024 16:30:14 +0100
Subject: [PATCH 051/455] Add collapsed button option to multistep form block

---
 .../bbui/src/ButtonGroup/CollapsedButtonGroup.svelte |  2 +-
 .../settings/controls/FormStepConfiguration.svelte   |  2 ++
 packages/client/manifest.json                        | 12 ++++++++++++
 .../components/app/blocks/MultiStepFormblock.svelte  | 10 +++++++---
 4 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
index f1860ecaeaa..e8362efc4b9 100644
--- a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -28,7 +28,7 @@
 >
   {text || "Action"}
 </Button>
-<Popover bind:this={popover} {align} {anchor} {offset}>
+<Popover bind:this={popover} {align} {anchor} {offset} resizable={false}>
   <Menu>
     {#each buttons as button}
       <MenuItem on:click={() => handleClick(button)} disabled={button.disabled}>
diff --git a/packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte
index d9257545c9f..6e7297a2458 100644
--- a/packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte
@@ -54,6 +54,8 @@
     _instanceName: `Step ${currentStep + 1}`,
     title: savedInstance.title ?? defaults?.title,
     buttons: savedInstance.buttons || defaults?.buttons,
+    buttonsCollapsed: savedInstance.buttonsCollapsed,
+    buttonsCollapsedText: savedInstance.buttonsCollapsedText,
     fields: savedInstance.fields,
     desc: savedInstance.desc,
 
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index de70e98357f..c84495ac0cb 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -7076,6 +7076,18 @@
         "key": "buttons",
         "wide": true,
         "nested": true
+      },
+      {
+        "type": "boolean",
+        "label": "Collapse",
+        "key": "buttonsCollapsed"
+      },
+      {
+        "type": "text",
+        "label": "Collapsed text",
+        "key": "buttonsCollapsedText",
+        "dependsOn": "buttonsCollapsed",
+        "placeholder": "Action"
       }
     ]
   },
diff --git a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte
index bcc62b52295..6cecf57a2c9 100644
--- a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte
+++ b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte
@@ -77,7 +77,7 @@
   const enrichSteps = (steps, schema, id) => {
     const safeSteps = steps?.length ? steps : [{}]
     return safeSteps.map((step, idx) => {
-      const { title, desc, fields, buttons } = step
+      const { title, fields, buttons } = step
       const defaultProps = Utils.buildMultiStepFormBlockDefaultProps({
         _id: id,
         stepCount: safeSteps.length,
@@ -86,10 +86,10 @@
         dataSource,
       })
       return {
+        ...step,
         _stepId: Helpers.uuid(),
         fields: getDefaultFields(fields || [], schema),
         title: title ?? defaultProps.title,
-        desc,
         buttons: buttons || defaultProps.buttons,
       }
     })
@@ -172,7 +172,11 @@
           {#if buttonPosition === "bottom"}
             <BlockComponent
               type="buttongroup"
-              props={{ buttons: step.buttons }}
+              props={{
+                buttons: step.buttons,
+                collapsed: step.buttonsCollapsed,
+                collapsedText: step.buttonsCollapsedText,
+              }}
               order={3}
             />
           {/if}

From b37a4cb0c96048eefe110746cc59edae6fbdc886 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 08:49:49 +0100
Subject: [PATCH 052/455] Lint

---
 packages/bbui/src/ActionMenu/ActionMenu.svelte            | 2 +-
 .../app/[application]/data/table/[tableId]/index.svelte   | 2 +-
 packages/client/src/components/app/ButtonGroup.svelte     | 8 +-------
 .../src/components/grid/layout/ButtonColumn.svelte        | 3 +--
 4 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte
index e3de70e3ee6..f27854bc045 100644
--- a/packages/bbui/src/ActionMenu/ActionMenu.svelte
+++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte
@@ -22,7 +22,7 @@
     anchor = node.firstChild
   }
 
-  export const show = e => {
+  export const show = () => {
     cancelHide()
     dropdown.show()
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index b89becaa030..e9b2b4853f6 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Banner, Menu, MenuItem, Popover } from "@budibase/bbui"
+  import { Banner } from "@budibase/bbui"
   import { datasources, tables, integrations, appStore } from "stores/builder"
   import { themeStore, admin } from "stores/portal"
   import { TableNames } from "constants"
diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
index edcf5284323..158bfb26b9b 100644
--- a/packages/client/src/components/app/ButtonGroup.svelte
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -1,13 +1,7 @@
 <script>
   import BlockComponent from "../BlockComponent.svelte"
   import Block from "../Block.svelte"
-  import {
-    Button,
-    Popover,
-    Menu,
-    MenuItem,
-    CollapsedButtonGroup,
-  } from "@budibase/bbui"
+  import { CollapsedButtonGroup } from "@budibase/bbui"
   import { getContext } from "svelte"
 
   export let buttons = []
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 7b5eb58fbe8..25a594dbb52 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -1,10 +1,9 @@
 <script>
   import { getContext, onMount } from "svelte"
-  import { Button } from "@budibase/bbui"
+  import { Button, CollapsedButtonGroup } from "@budibase/bbui"
   import GridCell from "../cells/GridCell.svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import { BlankRowID } from "../lib/constants"
-  import CollapsedButtonGroup from "../../../../../bbui/src/ButtonGroup/CollapsedButtonGroup.svelte"
 
   const {
     renderedRows,

From 4b367bf505966e8cfa9b870b26bb26c9233a45c6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 09:06:12 +0100
Subject: [PATCH 053/455] Add row action button actions

---
 .../actions/RowAction.svelte                  | 118 ++++++++++++++++++
 .../ButtonActionEditor/actions/index.js       |   1 +
 .../controls/ButtonActionEditor/manifest.json |   5 +
 packages/client/src/utils/buttonActions.js    |  10 ++
 4 files changed, 134 insertions(+)
 create mode 100644 packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte

diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
new file mode 100644
index 00000000000..866dcd2972a
--- /dev/null
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
@@ -0,0 +1,118 @@
+<script>
+  import { Select, Label, Checkbox, Body } from "@budibase/bbui"
+  import { tables, viewsV2 } from "stores/builder"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+  import { API } from "api"
+
+  export let parameters
+  export let bindings = []
+
+  let rowActions = []
+
+  $: tableOptions = $tables.list.map(table => ({
+    label: table.name,
+    resourceId: table._id,
+  }))
+  $: viewOptions = $viewsV2.list.map(view => ({
+    label: view.name,
+    resourceId: view.id,
+  }))
+  $: datasourceOptions = [...(tableOptions || []), ...(viewOptions || [])]
+  $: fetchRowActions(parameters.resourceId)
+  $: rowActionOptions = rowActions.map(action => ({
+    label: action.name,
+    value: action.id,
+  }))
+
+  const fetchRowActions = async resourceId => {
+    if (!resourceId) {
+      rowActions = []
+      return
+    }
+    try {
+      const res = await API.rowActions.fetch(resourceId)
+      rowActions = Object.values(res || {})
+    } catch (err) {
+      console.error(err)
+      rowActions = []
+    }
+  }
+</script>
+
+<div class="root">
+  <div class="params">
+    <Label>Table or view</Label>
+    <Select
+      bind:value={parameters.resourceId}
+      options={datasourceOptions}
+      getOptionLabel={x => x.label}
+      getOptionValue={x => x.resourceId}
+    />
+
+    <Label small>Row ID</Label>
+    <DrawerBindableInput
+      {bindings}
+      title="Row ID"
+      value={parameters.rowId}
+      on:change={value => (parameters.rowId = value.detail)}
+    />
+
+    <Label small>Row action</Label>
+    <Select bind:value={parameters.rowActionId} options={rowActionOptions} />
+
+    <br />
+    <Checkbox text="Require confirmation" bind:value={parameters.confirm} />
+
+    {#if parameters.confirm}
+      <Label small>Title</Label>
+      <DrawerBindableInput
+        placeholder="Prompt User"
+        value={parameters.customTitleText}
+        on:change={e => (parameters.customTitleText = e.detail)}
+        {bindings}
+      />
+      <Label small>Text</Label>
+      <DrawerBindableInput
+        placeholder="Are you sure you want to continue?"
+        value={parameters.confirmText}
+        on:change={e => (parameters.confirmText = e.detail)}
+        {bindings}
+      />
+      <Label small>Confirm Text</Label>
+      <DrawerBindableInput
+        placeholder="Confirm"
+        value={parameters.confirmButtonText}
+        on:change={e => (parameters.confirmButtonText = e.detail)}
+        {bindings}
+      />
+      <Label small>Cancel Text</Label>
+      <DrawerBindableInput
+        placeholder="Cancel"
+        value={parameters.cancelButtonText}
+        on:change={e => (parameters.cancelButtonText = e.detail)}
+        {bindings}
+      />
+    {/if}
+  </div>
+</div>
+
+<style>
+  .root {
+    width: 100%;
+    max-width: 800px;
+    margin: 0 auto;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: var(--spacing-xl);
+  }
+
+  .params {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+  }
+</style>
diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js
index 606ee41d021..b171b341112 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js
@@ -25,3 +25,4 @@ export { default as OpenModal } from "./OpenModal.svelte"
 export { default as CloseModal } from "./CloseModal.svelte"
 export { default as ClearRowSelection } from "./ClearRowSelection.svelte"
 export { default as DownloadFile } from "./DownloadFile.svelte"
+export { default as RowAction } from "./RowAction.svelte"
diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json
index 4022926e7f9..631e3119e88 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json
@@ -178,6 +178,11 @@
       "name": "Download File",
       "type": "data",
       "component": "DownloadFile"
+    },
+    {
+      "name": "Row Action",
+      "type": "data",
+      "component": "RowAction"
     }
   ]
 }
diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js
index 8f0cb575a7f..4dae0ad2739 100644
--- a/packages/client/src/utils/buttonActions.js
+++ b/packages/client/src/utils/buttonActions.js
@@ -493,6 +493,15 @@ const downloadFileHandler = async action => {
   }
 }
 
+const rowActionHandler = async action => {
+  const { resourceId, rowId, rowActionId } = action.parameters
+  await API.rowActions.trigger({
+    rowActionId,
+    tableId: resourceId,
+    rowId,
+  })
+}
+
 const handlerMap = {
   ["Fetch Row"]: fetchRowHandler,
   ["Save Row"]: saveRowHandler,
@@ -514,6 +523,7 @@ const handlerMap = {
   ["Open Modal"]: openModalHandler,
   ["Close Modal"]: closeModalHandler,
   ["Download File"]: downloadFileHandler,
+  ["Row Action"]: rowActionHandler,
 }
 
 const confirmTextMap = {

From 232a2829d2495a60ac108743bb391fbb5f5478bc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 10:09:07 +0100
Subject: [PATCH 054/455] Add UI for controlling row actions on views, and add
 support for view row actions in button actions

---
 .../buttons/grid/GridRowActionsButton.svelte  | 27 ++++++++++++++++--
 .../actions/RowAction.svelte                  | 16 +++++++++--
 .../data/table/[tableId]/index.svelte         |  2 +-
 packages/client/src/utils/buttonActions.js    |  2 +-
 packages/frontend-core/src/api/rowActions.js  | 28 +++++++++++++++++--
 5 files changed, 66 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index a6a7c930f55..c44c2d0f1ca 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -38,7 +38,10 @@
       return
     }
     const res = await API.rowActions.fetch(tableId)
-    rowActions = Object.values(res || {})
+    rowActions = Object.values(res || {}).map(action => ({
+      ...action,
+      enabled: !isView || action.allowedViews?.includes(ds.id),
+    }))
   }
 
   const createRowAction = async () => {
@@ -58,6 +61,23 @@
       notifications.error("Error creating row action")
     }
   }
+
+  const toggleAction = async (action, enabled) => {
+    console.log(action, enabled)
+    if (enabled) {
+      await API.rowActions.enableView({
+        tableId,
+        rowActionId: action.id,
+        viewId: ds.id,
+      })
+    } else {
+      await API.rowActions.disableView({
+        tableId,
+        rowActionId: action.id,
+        viewId: ds.id,
+      })
+    }
+  }
 </script>
 
 <DetailPopover title="Row actions">
@@ -87,7 +107,10 @@
           <svelte:fragment slot="right">
             {#if isView}
               <span>
-                <Toggle />
+                <Toggle
+                  value={action.enabled}
+                  on:change={e => toggleAction(action, e.detail)}
+                />
               </span>
             {/if}
           </svelte:fragment>
diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
index 866dcd2972a..427a86b3f8d 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
@@ -15,10 +15,13 @@
   }))
   $: viewOptions = $viewsV2.list.map(view => ({
     label: view.name,
+    tableId: view.tableId,
     resourceId: view.id,
   }))
+  $: console.log($viewsV2.list)
   $: datasourceOptions = [...(tableOptions || []), ...(viewOptions || [])]
-  $: fetchRowActions(parameters.resourceId)
+  $: resourceId = parameters.resourceId
+  $: fetchRowActions(resourceId)
   $: rowActionOptions = rowActions.map(action => ({
     label: action.name,
     value: action.id,
@@ -30,8 +33,15 @@
       return
     }
     try {
-      const res = await API.rowActions.fetch(resourceId)
-      rowActions = Object.values(res || {})
+      const isView = resourceId.startsWith("view_")
+      let tableId = resourceId
+      if (isView) {
+        tableId = viewOptions.find(x => x.resourceId === resourceId).tableId
+      }
+      const res = await API.rowActions.fetch(tableId)
+      rowActions = Object.values(res || {}).filter(action => {
+        return !isView || action.allowedViews?.includes(resourceId)
+      })
     } catch (err) {
       console.error(err)
       rowActions = []
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index e9b2b4853f6..15b43034735 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -63,7 +63,7 @@
       onClick: async row => {
         await API.rowActions.trigger({
           rowActionId: action.id,
-          tableId: id,
+          sourceId: id,
           rowId: row._id,
         })
       },
diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js
index 4dae0ad2739..847a32116d5 100644
--- a/packages/client/src/utils/buttonActions.js
+++ b/packages/client/src/utils/buttonActions.js
@@ -497,7 +497,7 @@ const rowActionHandler = async action => {
   const { resourceId, rowId, rowActionId } = action.parameters
   await API.rowActions.trigger({
     rowActionId,
-    tableId: resourceId,
+    sourceId: resourceId,
     rowId,
   })
 }
diff --git a/packages/frontend-core/src/api/rowActions.js b/packages/frontend-core/src/api/rowActions.js
index 85ae4f6b96e..4c0cd5f58bc 100644
--- a/packages/frontend-core/src/api/rowActions.js
+++ b/packages/frontend-core/src/api/rowActions.js
@@ -50,14 +50,38 @@ export const buildRowActionEndpoints = API => ({
     })
   },
 
+  /**
+   * Enables a row action for a certain view
+   * @param tableId the ID of the parent table
+   * @param rowActionId the ID of the row action
+   * @param viewId the ID of the view
+   */
+  enableView: async ({ tableId, rowActionId, viewId }) => {
+    return await API.post({
+      url: `/api/tables/${tableId}/actions/${rowActionId}/permissions/${viewId}`,
+    })
+  },
+
+  /**
+   * Disables a row action for a certain view
+   * @param tableId the ID of the parent table
+   * @param rowActionId the ID of the row action
+   * @param viewId the ID of the view
+   */
+  disableView: async ({ tableId, rowActionId, viewId }) => {
+    return await API.delete({
+      url: `/api/tables/${tableId}/actions/${rowActionId}/permissions/${viewId}`,
+    })
+  },
+
   /**
    * Triggers a row action.
    * @param tableId the ID of the table
    * @param rowActionId the ID of the row action to trigger
    */
-  trigger: async ({ tableId, rowActionId, rowId }) => {
+  trigger: async ({ sourceId, rowActionId, rowId }) => {
     return await API.post({
-      url: `/api/tables/${tableId}/actions/${rowActionId}/trigger`,
+      url: `/api/tables/${sourceId}/actions/${rowActionId}/trigger`,
       body: {
         rowId,
       },

From 2a02298ac2bd5b1cf2f37a04703f12400b2d813e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 10:56:38 +0100
Subject: [PATCH 055/455] Auto select first available row action

---
 .../controls/ButtonActionEditor/actions/RowAction.svelte      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
index 427a86b3f8d..a65531a2236 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
@@ -18,7 +18,6 @@
     tableId: view.tableId,
     resourceId: view.id,
   }))
-  $: console.log($viewsV2.list)
   $: datasourceOptions = [...(tableOptions || []), ...(viewOptions || [])]
   $: resourceId = parameters.resourceId
   $: fetchRowActions(resourceId)
@@ -46,6 +45,9 @@
       console.error(err)
       rowActions = []
     }
+
+    // Auto select first action
+    parameters.rowActionId = rowActions[0]?.id
   }
 </script>
 

From 7d5e884a8f4b3db4ca7e997abae91bf055f16aaa Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 11:50:20 +0100
Subject: [PATCH 056/455] Fix row action count for views

---
 .../backend/DataTable/buttons/grid/GridRowActionsButton.svelte | 3 +--
 .../app/[application]/data/table/[tableId]/index.svelte        | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index c44c2d0f1ca..193a977a466 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -23,8 +23,7 @@
   $: tableId = ds?.tableId
   $: isView = ds?.type === "viewV2"
   $: fetchRowActions(tableId)
-  $: viewActiveCount = 0
-  $: actionCount = isView ? viewActiveCount : rowActions.length
+  $: actionCount = rowActions.filter(action => !isView || action.enabled).length
 
   const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
     return ({ automationId }) => {
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 15b43034735..49e78a9428c 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -146,10 +146,10 @@
       {#if !isUsersTable}
         <GridRowActionsButton />
       {/if}
+      <GridScreensButton on:request-generate={() => generateButton?.show()} />
       <GridAutomationsButton
         on:request-generate={() => generateButton?.show()}
       />
-      <GridScreensButton on:request-generate={() => generateButton?.show()} />
       {#if !isUsersTable}
         <GridImportButton />
       {/if}

From 789df301cb2ce4bc3e94a73a35611997f758fa40 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 15:05:13 +0100
Subject: [PATCH 057/455] Enable renaming row actions and improve row actions
 in grids

---
 .../ButtonGroup/CollapsedButtonGroup.svelte   | 10 ++-
 .../FlowChart/FlowItem.svelte                 |  2 +-
 .../AutomationPanel/AutomationNavItem.svelte  | 23 ++++-
 .../UpdateRowActionModal.svelte               | 85 +++++++++++++++++++
 .../grid/GridColumnsSettingButton.svelte      |  8 +-
 .../buttons/grid/GridRowActionsButton.svelte  |  8 +-
 .../table/[tableId]/[viewId]/index.svelte     |  3 +-
 .../builder/src/stores/portal/licensing.js    |  3 +
 packages/frontend-core/src/api/rowActions.js  |  2 +-
 .../grid/layout/ButtonColumn.svelte           |  1 +
 .../server/src/sdk/app/automations/utils.ts   |  2 +-
 11 files changed, 131 insertions(+), 16 deletions(-)
 create mode 100644 packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte

diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
index e8362efc4b9..42e2451ad45 100644
--- a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -9,6 +9,7 @@
   export let size = "M"
   export let align = "left"
   export let offset
+  export let animate
 
   let anchor
   let popover
@@ -28,7 +29,14 @@
 >
   {text || "Action"}
 </Button>
-<Popover bind:this={popover} {align} {anchor} {offset} resizable={false}>
+<Popover
+  bind:this={popover}
+  {align}
+  {anchor}
+  {offset}
+  {animate}
+  resizable={false}
+>
   <Menu>
     {#each buttons as button}
       <MenuItem on:click={() => handleClick(button)} disabled={button.disabled}>
diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
index c88317c79f6..aca3e950c37 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
@@ -190,7 +190,7 @@
         {#if isTrigger && triggerInfo}
           <InlineAlert
             header={triggerInfo.type}
-            message={`This trigger is tied to the row action ${triggerInfo.rowAction.name} on your ${triggerInfo.table.name} table`}
+            message={`This trigger is tied to the "${triggerInfo.rowAction.name}" row action in your ${triggerInfo.table.name} table`}
           />
         {/if}
         {#if lastStep}
diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index 8ab9851ff07..ec9b9561904 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -9,6 +9,7 @@
   import { sdk } from "@budibase/shared-core"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import UpdateAutomationModal from "components/automation/AutomationPanel/UpdateAutomationModal.svelte"
+  import UpdateRowActionModal from "components/automation/AutomationPanel/UpdateRowActionModal.svelte"
   import NavItem from "components/common/NavItem.svelte"
 
   export let automation
@@ -16,6 +17,9 @@
 
   let confirmDeleteDialog
   let updateAutomationDialog
+  let updateRowActionDialog
+
+  $: isRowAction = sdk.automations.isRowAction(automation)
 
   async function deleteAutomation() {
     try {
@@ -37,7 +41,6 @@
   }
 
   const getContextMenuItems = () => {
-    const isRowAction = sdk.automations.isRowAction(automation)
     const pause = {
       icon: automation.disabled ? "CheckmarkCircle" : "Cancel",
       name: automation.disabled ? "Activate" : "Pause",
@@ -83,7 +86,16 @@
         del,
       ]
     } else {
-      return [del]
+      return [
+        {
+          icon: "Edit",
+          name: "Edit",
+          keyBind: null,
+          visible: true,
+          callback: updateRowActionDialog.show,
+        },
+        del,
+      ]
     }
   }
 
@@ -122,4 +134,9 @@
   <i>{automation.name}?</i>
   This action cannot be undone.
 </ConfirmDialog>
-<UpdateAutomationModal {automation} bind:this={updateAutomationDialog} />
+
+{#if isRowAction}
+  <UpdateRowActionModal {automation} bind:this={updateRowActionDialog} />
+{:else}
+  <UpdateAutomationModal {automation} bind:this={updateAutomationDialog} />
+{/if}
diff --git a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
new file mode 100644
index 00000000000..af754d7ca93
--- /dev/null
+++ b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
@@ -0,0 +1,85 @@
+<script>
+  import { automationStore } from "stores/builder"
+  import {
+    notifications,
+    Icon,
+    Input,
+    ModalContent,
+    Modal,
+  } from "@budibase/bbui"
+  import { API } from "api"
+
+  export let automation
+  export let onCancel = undefined
+
+  let name
+  let error = ""
+  let modal
+
+  export const show = () => {
+    name = automation?.displayName
+    modal.show()
+  }
+  export const hide = () => {
+    modal.hide()
+  }
+
+  async function saveAutomation() {
+    try {
+      await API.rowActions.update({
+        rowActionId: automation.definition.trigger.inputs.rowActionId,
+        tableId: automation.definition.trigger.inputs.tableId,
+        name,
+      })
+      await automationStore.actions.fetch()
+      notifications.success(`Row action updated successfully`)
+      hide()
+    } catch (error) {
+      notifications.error("Error saving row action")
+    }
+  }
+
+  function checkValid(evt) {
+    name = evt.target.value
+    if (!name) {
+      error = "Name is required"
+      return
+    }
+    error = ""
+  }
+</script>
+
+<Modal bind:this={modal} on:hide={onCancel}>
+  <ModalContent
+    title="Edit Row Action"
+    confirmText="Save"
+    size="L"
+    onConfirm={saveAutomation}
+    disabled={error}
+  >
+    <Input bind:value={name} label="Name" on:input={checkValid} {error} />
+    <a
+      slot="footer"
+      target="_blank"
+      href="https://docs.budibase.com/docs/automation-steps"
+    >
+      <Icon name="InfoOutline" />
+      <span>Learn about automations</span>
+    </a>
+  </ModalContent>
+</Modal>
+
+<style>
+  a {
+    color: var(--ink);
+    font-size: 14px;
+    vertical-align: middle;
+    display: flex;
+    align-items: center;
+    text-decoration: none;
+  }
+  a span {
+    text-decoration: underline;
+    margin-left: var(--spectrum-alias-item-padding-s);
+  }
+</style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
index 11109c0f78a..c75c1b85680 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
@@ -2,8 +2,7 @@
   import { getContext } from "svelte"
   import { ActionButton, Popover } from "@budibase/bbui"
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
-
-  export let allowViewReadonlyColumns = false
+  import { licensing } from "stores/portal"
 
   const { columns } = getContext("grid")
 
@@ -29,5 +28,8 @@
 </div>
 
 <Popover bind:open {anchor} align="left">
-  <ColumnsSettingContent columns={$columns} {allowViewReadonlyColumns} />
+  <ColumnsSettingContent
+    columns={$columns}
+    allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
+  />
 </Popover>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 193a977a466..9adbca76038 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -22,7 +22,7 @@
   $: ds = $datasource
   $: tableId = ds?.tableId
   $: isView = ds?.type === "viewV2"
-  $: fetchRowActions(tableId)
+  $: fetchRowActions(ds)
   $: actionCount = rowActions.filter(action => !isView || action.enabled).length
 
   const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
@@ -31,12 +31,12 @@
     }
   })
 
-  const fetchRowActions = async tableId => {
-    if (!tableId) {
+  const fetchRowActions = async datasource => {
+    if (!datasource?.tableId) {
       rowActions = []
       return
     }
-    const res = await API.rowActions.fetch(tableId)
+    const res = await API.rowActions.fetch(datasource.tableId)
     rowActions = Object.values(res || {}).map(action => ({
       ...action,
       enabled: !isView || action.allowedViews?.includes(ds.id),
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 798d1e0b51e..e3020394df2 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2 } from "stores/builder"
-  import { admin, licensing } from "stores/portal"
+  import { admin } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@@ -35,7 +35,6 @@
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}
-  allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
 >
   <svelte:fragment slot="controls">
     <GridFilterButton />
diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js
index 0e446504796..dd9747aa091 100644
--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@@ -104,6 +104,7 @@ export const createLicensingStore = () => {
       const isBusinessPlan = planType === Constants.PlanType.BUSINESS
       const isEnterpriseTrial =
         planType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL
+      console.log(license)
       const groupsEnabled = license.features.includes(
         Constants.Features.USER_GROUPS
       )
@@ -142,6 +143,8 @@ export const createLicensingStore = () => {
         Constants.Features.VIEW_READONLY_COLUMNS
       )
 
+      console.log(isViewReadonlyColumnsEnabled)
+
       store.update(state => {
         return {
           ...state,
diff --git a/packages/frontend-core/src/api/rowActions.js b/packages/frontend-core/src/api/rowActions.js
index 4c0cd5f58bc..071af953efa 100644
--- a/packages/frontend-core/src/api/rowActions.js
+++ b/packages/frontend-core/src/api/rowActions.js
@@ -31,7 +31,7 @@ export const buildRowActionEndpoints = API => ({
    * @param rowActionId the ID of the row action to update
    */
   update: async ({ tableId, rowActionId, name }) => {
-    return await API.post({
+    return await API.put({
       url: `/api/tables/${tableId}/actions/${rowActionId}`,
       body: {
         name,
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 25a594dbb52..4ec1fd68799 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -91,6 +91,7 @@
                   align="right"
                   offset={5}
                   size="S"
+                  animate={false}
                 />
               {:else}
                 {#each buttons as button}
diff --git a/packages/server/src/sdk/app/automations/utils.ts b/packages/server/src/sdk/app/automations/utils.ts
index 2ce0012b6ae..2c34de255b9 100644
--- a/packages/server/src/sdk/app/automations/utils.ts
+++ b/packages/server/src/sdk/app/automations/utils.ts
@@ -52,7 +52,7 @@ export async function getBuilderData(
     const rowActionName = await getRowActionName(tableId, rowActionId)
 
     result[automation._id!] = {
-      displayName: `${tableName}: ${automation.name}`,
+      displayName: rowActionName,
       triggerInfo: {
         type: "Automation trigger",
         table: { id: tableId, name: tableName },

From 0e6d903c744887c1f8f2eb25378a32bf7ecfa975 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 15:14:40 +0100
Subject: [PATCH 058/455] Select row when opening row actions popover

---
 packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte      | 3 +++
 .../src/components/grid/layout/ButtonColumn.svelte             | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
index 42e2451ad45..5f6eca0b130 100644
--- a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -26,6 +26,7 @@
   icon="ChevronDown"
   cta
   on:click={() => popover?.show()}
+  on:click
 >
   {text || "Action"}
 </Button>
@@ -36,6 +37,8 @@
   {offset}
   {animate}
   resizable={false}
+  on:close
+  on:open
 >
   <Menu>
     {#each buttons as button}
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 4ec1fd68799..a086972941f 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -92,6 +92,8 @@
                   offset={5}
                   size="S"
                   animate={false}
+                  on:open={() => selectedRows.set({ [row._id]: true })}
+                  on:close={() => selectedRows.set({})}
                 />
               {:else}
                 {#each buttons as button}

From 7703da159d793c6a6d13047423c9e75289f6603a Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 3 Sep 2024 17:16:49 +0100
Subject: [PATCH 059/455] Merge commit

---
 packages/frontend-core/src/constants.js  |  5 ++++
 packages/shared-core/src/filters.ts      | 35 +++++++++++++-----------
 packages/types/src/documents/app/view.ts |  4 +--
 3 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index bb6f4b4873d..c004f72dc20 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -182,3 +182,8 @@ export const OnEmptyFilter = {
   RETURN_ALL: "all",
   RETURN_NONE: "none",
 }
+
+export const FilterValueType = {
+  BINDING: "Binding",
+  VALUE: "Value",
+}
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 1cad32e790e..82ef6759d2a 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -27,7 +27,6 @@ import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
 import { decodeNonAscii } from "./helpers/schema"
-// import { Constants } from "@budibase/frontend-core"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
 
@@ -293,20 +292,21 @@ export class ColumnSplitter {
 const builderFilter = (expression: SearchFilter) => {
   // Filter body
   let query: SearchFilters = {
-    // string: {},
-    // fuzzy: {},
-    // range: {},
-    // equal: {},
-    // notEqual: {},
-    // empty: {},
-    // notEmpty: {},
-    // contains: {},
-    // notContains: {},
-    // oneOf: {},
-    // containsAny: {},
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
   }
 
   // DEAN -
+
   // This is the chattiest service we have, pruning the requests
   // of bloat should have meaningful impact
   // Further validation in this area is a must
@@ -427,6 +427,7 @@ export const buildQuery = (filter: SearchFilter[]) => {
     return query
   }
 
+  // DEAN Replace with builderFilter
   filter.forEach(expression => {
     let { operator, field, type, value, externalType, onEmptyFilter } =
       expression
@@ -527,10 +528,12 @@ export const buildQuery = (filter: SearchFilter[]) => {
   return query
 }
 
-/**
- * FOR TESTING - this still isnt type safe
- */
+// Grouped query
 export const buildQueryX = (filter: SearchFilterGroup) => {
+  if (!filter) {
+    return {}
+  }
+
   const operatorMap = {
     [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
     [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
@@ -545,7 +548,7 @@ export const buildQueryX = (filter: SearchFilterGroup) => {
       conditions: filter.groups?.map((group: SearchFilterGroup) => {
         return {
           [operatorMap[group.logicalOperator]]: {
-            conditions: group.filters?.map(x => builderFilter(x)), //buildFilters
+            conditions: group.filters?.map(x => builderFilter(x)),
           },
         }
       }),
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 24dad0bcca4..d02e5af1b11 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -1,4 +1,4 @@
-import { SearchFilter, SortOrder, SortType } from "../../api"
+import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
 import { DBView } from "../../sdk"
@@ -56,7 +56,7 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilter[]
+  query?: SearchFilter[] | SearchFilterGroup
   sort?: {
     field: string
     order?: SortOrder

From 53ff7e116785147f4c313cf827e360f3e2925afc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 18:58:22 +0100
Subject: [PATCH 060/455] Add new builder store for row action CRUD, ensuring
 consistent state everywhere

---
 .../buttons/grid/GridRowActionsButton.svelte  |  53 ++------
 .../table/[tableId]/[viewId]/index.svelte     |  19 ++-
 .../data/table/[tableId]/index.svelte         |  24 ++--
 packages/builder/src/stores/BudiStore.js      |  10 +-
 packages/builder/src/stores/builder/index.js  |   3 +
 .../builder/src/stores/builder/rowActions.js  | 114 ++++++++++++++++++
 .../builder/src/stores/portal/licensing.js    |   3 -
 .../grid/layout/ButtonColumn.svelte           |  10 +-
 8 files changed, 173 insertions(+), 63 deletions(-)
 create mode 100644 packages/builder/src/stores/builder/rowActions.js

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 9adbca76038..5bd2b757e46 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -9,21 +9,20 @@
   } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
   import { getContext } from "svelte"
-  import { appStore, automationStore } from "stores/builder"
+  import { appStore, rowActions } from "stores/builder"
   import { API } from "api"
   import { goto, url } from "@roxi/routify"
   import { derived } from "svelte/store"
-  import { getSequentialName } from "helpers/duplicate"
 
   const { datasource } = getContext("grid")
 
-  let rowActions = []
-
   $: ds = $datasource
   $: tableId = ds?.tableId
+  $: viewId = ds?.id
   $: isView = ds?.type === "viewV2"
-  $: fetchRowActions(ds)
-  $: actionCount = rowActions.filter(action => !isView || action.enabled).length
+  $: tableRowActions = $rowActions[tableId] || []
+  $: viewRowActions = $rowActions[viewId] || []
+  $: actionCount = isView ? viewRowActions.length : tableRowActions.length
 
   const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
     return ({ automationId }) => {
@@ -31,30 +30,11 @@
     }
   })
 
-  const fetchRowActions = async datasource => {
-    if (!datasource?.tableId) {
-      rowActions = []
-      return
-    }
-    const res = await API.rowActions.fetch(datasource.tableId)
-    rowActions = Object.values(res || {}).map(action => ({
-      ...action,
-      enabled: !isView || action.allowedViews?.includes(ds.id),
-    }))
-  }
-
   const createRowAction = async () => {
     try {
-      const name = getSequentialName(rowActions, "New row action ", {
-        getName: x => x.name,
-      })
-      const res = await API.rowActions.create({
-        name,
-        tableId,
-      })
-      await automationStore.actions.fetch()
+      const newRowAction = await rowActions.createRowAction(tableId, viewId)
       notifications.success("Row action created successfully")
-      $goto($rowActionUrl(res))
+      // $goto($rowActionUrl(newRowAction))
     } catch (error) {
       console.error(error)
       notifications.error("Error creating row action")
@@ -62,19 +42,10 @@
   }
 
   const toggleAction = async (action, enabled) => {
-    console.log(action, enabled)
     if (enabled) {
-      await API.rowActions.enableView({
-        tableId,
-        rowActionId: action.id,
-        viewId: ds.id,
-      })
+      await rowActions.enableView(tableId, viewId, action.id)
     } else {
-      await API.rowActions.disableView({
-        tableId,
-        rowActionId: action.id,
-        viewId: ds.id,
-      })
+      await rowActions.disableView(tableId, viewId, action.id)
     }
   }
 </script>
@@ -96,18 +67,18 @@
     Use the toggle to enable/disable row actions for this view.
     <br />
   {/if}
-  {#if !rowActions.length}
+  {#if !tableRowActions.length}
     <br />
     You haven't created any row actions.
   {:else}
     <List>
-      {#each rowActions as action}
+      {#each tableRowActions as action}
         <ListItem title={action.name} url={$rowActionUrl(action)} showArrow>
           <svelte:fragment slot="right">
             {#if isView}
               <span>
                 <Toggle
-                  value={action.enabled}
+                  value={action.allowedViews?.includes(viewId)}
                   on:change={e => toggleAction(action, e.detail)}
                 />
               </span>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index e3020394df2..3596f1b8323 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { viewsV2 } from "stores/builder"
+  import { viewsV2, rowActions } from "stores/builder"
   import { admin } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
@@ -21,6 +21,21 @@
     id,
     tableId: $viewsV2.selected?.tableId,
   }
+  $: buttons = makeRowActionButtons($rowActions[id])
+  $: rowActions.refreshRowActions(id)
+
+  const makeRowActionButtons = rowActions => {
+    return (rowActions || []).map(action => ({
+      text: action.name,
+      onClick: async row => {
+        await API.rowActions.trigger({
+          rowActionId: action.id,
+          sourceId: id,
+          rowId: row._id,
+        })
+      },
+    }))
+  }
 
   const handleGridViewUpdate = async e => {
     viewsV2.replaceView(id, e.detail)
@@ -35,6 +50,8 @@
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}
+  {buttons}
+  buttonsCollapsed
 >
   <svelte:fragment slot="controls">
     <GridFilterButton />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 49e78a9428c..788b178fc7d 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -1,6 +1,12 @@
 <script>
   import { Banner } from "@budibase/bbui"
-  import { datasources, tables, integrations, appStore } from "stores/builder"
+  import {
+    datasources,
+    tables,
+    integrations,
+    appStore,
+    rowActions,
+  } from "stores/builder"
   import { themeStore, admin } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
@@ -28,7 +34,6 @@
     status: { displayName: "Status", disabled: true },
   }
 
-  let rowActions = []
   let generateButton
 
   $: autoColumnStatus = verifyAutocolumns($tables?.selected)
@@ -54,11 +59,11 @@
   $: relationshipsEnabled = relationshipSupport(tableDatasource)
   $: currentTheme = $themeStore?.theme
   $: darkMode = !currentTheme.includes("light")
-  $: buttons = makeRowActionButtons(rowActions)
-  $: fetchRowActions(id)
+  $: buttons = makeRowActionButtons($rowActions[id])
+  $: rowActions.refreshRowActions(id)
 
   const makeRowActionButtons = rowActions => {
-    return rowActions.map(action => ({
+    return (rowActions || []).map(action => ({
       text: action.name,
       onClick: async row => {
         await API.rowActions.trigger({
@@ -98,15 +103,6 @@
       return acc
     }, {})
   }
-
-  const fetchRowActions = async tableId => {
-    if (!tableId) {
-      rowActions = []
-      return
-    }
-    const res = await API.rowActions.fetch(tableId)
-    rowActions = Object.values(res || {})
-  }
 </script>
 
 {#if $tables?.selected?.name}
diff --git a/packages/builder/src/stores/BudiStore.js b/packages/builder/src/stores/BudiStore.js
index 1acf299921c..e7be02f2831 100644
--- a/packages/builder/src/stores/BudiStore.js
+++ b/packages/builder/src/stores/BudiStore.js
@@ -2,9 +2,8 @@ import { writable } from "svelte/store"
 
 export default class BudiStore {
   constructor(init, opts) {
-    const store = writable({
-      ...init,
-    })
+    this.initialState = init
+    const store = writable({ ...init })
 
     /**
      * Internal Svelte store
@@ -23,6 +22,7 @@ export default class BudiStore {
      * *Store modification should be kept to a minimum
      */
     this.update = this.store.update
+    this.set = this.store.set
 
     /**
      * Optional debug mode to output the store updates to console
@@ -33,4 +33,8 @@ export default class BudiStore {
       })
     }
   }
+
+  reset = () => {
+    this.store.set({ ...this.initialState })
+  }
 }
diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js
index aa0062dd7d5..dbde7399518 100644
--- a/packages/builder/src/stores/builder/index.js
+++ b/packages/builder/src/stores/builder/index.js
@@ -29,6 +29,7 @@ import { integrations } from "./integrations"
 import { sortedIntegrations } from "./sortedIntegrations"
 import { queries } from "./queries"
 import { flags } from "./flags"
+import { rowActions } from "./rowActions"
 import componentTreeNodesStore from "./componentTreeNodes"
 
 export {
@@ -65,6 +66,7 @@ export {
   flags,
   hoverStore,
   snippets,
+  rowActions,
 }
 
 export const reset = () => {
@@ -74,6 +76,7 @@ export const reset = () => {
   componentStore.reset()
   layoutStore.reset()
   navigationStore.reset()
+  rowActions.reset()
 }
 
 const refreshBuilderData = async () => {
diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
new file mode 100644
index 00000000000..0255e4726c1
--- /dev/null
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -0,0 +1,114 @@
+import { get, derived } from "svelte/store"
+import BudiStore from "stores/BudiStore"
+import { tables } from "./tables"
+import { viewsV2 } from "./viewsV2"
+import { automationStore } from "./automations"
+import { API } from "api"
+import { getSequentialName } from "helpers/duplicate"
+
+const initialState = {}
+
+export class RowActionStore extends BudiStore {
+  constructor() {
+    super(initialState)
+  }
+
+  refreshRowActions = async sourceId => {
+    if (!sourceId) {
+      return
+    }
+
+    // Get the underlying table ID for this source ID
+    let tableId = get(tables).list.find(table => table._id === sourceId)?._id
+    if (!tableId) {
+      const view = get(viewsV2).list.find(view => view.id === sourceId)
+      tableId = view?.tableId
+    }
+    if (!tableId) {
+      return
+    }
+
+    // Fetch row actions for this table
+    const res = await API.rowActions.fetch(tableId)
+    const actions = Object.values(res || {})
+    this.update(state => ({
+      ...state,
+      [tableId]: actions,
+    }))
+  }
+
+  createRowAction = async (tableId, viewId) => {
+    if (!tableId) {
+      return
+    }
+
+    // Get a unique name for this action
+    const existingRowActions = get(this.store)[tableId] || []
+    const name = getSequentialName(existingRowActions, "New row action ", {
+      getName: x => x.name,
+    })
+
+    // Create the action and update state
+    const res = await API.rowActions.create({
+      name,
+      tableId,
+    })
+    this.update(state => ({
+      ...state,
+      [tableId]: [...(state[tableId] || []), res],
+    }))
+
+    // If adding to a view, enable on this view
+    if (viewId) {
+      await this.enableView(tableId, viewId, res.id)
+    }
+
+    // Refresh automations so we have this new row action automation
+    await automationStore.actions.fetch()
+
+    return res
+  }
+
+  enableView = async (tableId, viewId, rowActionId) => {
+    await API.rowActions.enableView({
+      tableId,
+      viewId,
+      rowActionId,
+    })
+    await this.refreshRowActions(tableId)
+  }
+
+  disableView = async (tableId, viewId, rowActionId) => {
+    await API.rowActions.disableView({
+      tableId,
+      viewId,
+      rowActionId,
+    })
+    await this.refreshRowActions(tableId)
+  }
+}
+
+const store = new RowActionStore()
+const derivedStore = derived(store, $store => {
+  let map = {}
+
+  // Generate an entry for every view as well
+  Object.keys($store || {}).forEach(tableId => {
+    map[tableId] = $store[tableId]
+    for (let action of $store[tableId]) {
+      for (let viewId of action.allowedViews || []) {
+        if (!map[viewId]) {
+          map[viewId] = []
+        }
+        map[viewId].push(action)
+      }
+    }
+  })
+
+  return map
+})
+
+export const rowActions = {
+  ...store,
+  subscribe: derivedStore.subscribe,
+}
diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js
index dd9747aa091..0e446504796 100644
--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@@ -104,7 +104,6 @@ export const createLicensingStore = () => {
       const isBusinessPlan = planType === Constants.PlanType.BUSINESS
       const isEnterpriseTrial =
         planType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL
-      console.log(license)
       const groupsEnabled = license.features.includes(
         Constants.Features.USER_GROUPS
       )
@@ -143,8 +142,6 @@ export const createLicensingStore = () => {
         Constants.Features.VIEW_READONLY_COLUMNS
       )
 
-      console.log(isViewReadonlyColumnsEnabled)
-
       store.update(state => {
         return {
           ...state,
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index a086972941f..03f5f4644e7 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -25,7 +25,7 @@
 
   let container
 
-  $: buttons = $props.buttons?.slice(0, 3) || []
+  $: buttons = getButtons($props)
   $: columnsWidth = $scrollableColumns.reduce(
     (total, col) => (total += col.width),
     0
@@ -34,6 +34,14 @@
   $: gridEnd = $width - $buttonColumnWidth - 1
   $: left = Math.min(columnEnd, gridEnd)
 
+  const getButtons = ({ buttons, buttonsCollapsed }) => {
+    let gridButtons = buttons || []
+    if (!buttonsCollapsed) {
+      return gridButtons.slice(0, 3)
+    }
+    return gridButtons
+  }
+
   const handleClick = async (button, row) => {
     await button.onClick?.(rows.actions.cleanRow(row))
     await rows.actions.refreshRow(row._id)

From 872446e5244d8b904332fa5d21b6529249279f09 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 19:47:12 +0100
Subject: [PATCH 061/455] Replace other usages of manual row actions API calls
 with store usage

---
 .../UpdateRowActionModal.svelte               | 13 ++++---
 .../actions/RowAction.svelte                  | 36 +++----------------
 .../table/[tableId]/[viewId]/index.svelte     |  6 +---
 .../data/table/[tableId]/index.svelte         |  6 +---
 .../builder/src/stores/builder/automations.js |  9 ++---
 .../builder/src/stores/builder/rowActions.js  | 30 ++++++++++++++++
 6 files changed, 48 insertions(+), 52 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
index af754d7ca93..db22dab678e 100644
--- a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "stores/builder"
+  import { automationStore, rowActions } from "stores/builder"
   import {
     notifications,
     Icon,
@@ -26,12 +26,11 @@
 
   async function saveAutomation() {
     try {
-      await API.rowActions.update({
-        rowActionId: automation.definition.trigger.inputs.rowActionId,
-        tableId: automation.definition.trigger.inputs.tableId,
-        name,
-      })
-      await automationStore.actions.fetch()
+      await rowActions.rename(
+        automation.definition.trigger.inputs.tableId,
+        automation.definition.trigger.inputs.rowActionId,
+        name
+      )
       notifications.success(`Row action updated successfully`)
       hide()
     } catch (error) {
diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
index a65531a2236..8158d2dea57 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte
@@ -1,14 +1,11 @@
 <script>
-  import { Select, Label, Checkbox, Body } from "@budibase/bbui"
-  import { tables, viewsV2 } from "stores/builder"
+  import { Select, Label, Checkbox } from "@budibase/bbui"
+  import { tables, viewsV2, rowActions } from "stores/builder"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
-  import { API } from "api"
 
   export let parameters
   export let bindings = []
 
-  let rowActions = []
-
   $: tableOptions = $tables.list.map(table => ({
     label: table.name,
     resourceId: table._id,
@@ -20,35 +17,12 @@
   }))
   $: datasourceOptions = [...(tableOptions || []), ...(viewOptions || [])]
   $: resourceId = parameters.resourceId
-  $: fetchRowActions(resourceId)
-  $: rowActionOptions = rowActions.map(action => ({
+  $: rowActions.refreshRowActions(resourceId)
+  $: enabledRowActions = $rowActions[resourceId] || []
+  $: rowActionOptions = enabledRowActions.map(action => ({
     label: action.name,
     value: action.id,
   }))
-
-  const fetchRowActions = async resourceId => {
-    if (!resourceId) {
-      rowActions = []
-      return
-    }
-    try {
-      const isView = resourceId.startsWith("view_")
-      let tableId = resourceId
-      if (isView) {
-        tableId = viewOptions.find(x => x.resourceId === resourceId).tableId
-      }
-      const res = await API.rowActions.fetch(tableId)
-      rowActions = Object.values(res || {}).filter(action => {
-        return !isView || action.allowedViews?.includes(resourceId)
-      })
-    } catch (err) {
-      console.error(err)
-      rowActions = []
-    }
-
-    // Auto select first action
-    parameters.rowActionId = rowActions[0]?.id
-  }
 </script>
 
 <div class="root">
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 3596f1b8323..698e8307def 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -28,11 +28,7 @@
     return (rowActions || []).map(action => ({
       text: action.name,
       onClick: async row => {
-        await API.rowActions.trigger({
-          rowActionId: action.id,
-          sourceId: id,
-          rowId: row._id,
-        })
+        await rowActions.trigger(id, action.id, row._id)
       },
     }))
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 788b178fc7d..09c20cf059c 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -66,11 +66,7 @@
     return (rowActions || []).map(action => ({
       text: action.name,
       onClick: async row => {
-        await API.rowActions.trigger({
-          rowActionId: action.id,
-          sourceId: id,
-          rowId: row._id,
-        })
+        await rowActions.trigger(id, action.id, row._id)
       },
     }))
   }
diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js
index 69a340d3401..d6d89567a78 100644
--- a/packages/builder/src/stores/builder/automations.js
+++ b/packages/builder/src/stores/builder/automations.js
@@ -7,6 +7,7 @@ import { notifications } from "@budibase/bbui"
 import { updateReferencesInObject } from "dataBinding"
 import { AutomationTriggerStepId } from "@budibase/types"
 import { sdk } from "@budibase/shared-core"
+import { rowActions } from "./rowActions"
 
 const initialAutomationState = {
   automations: [],
@@ -126,10 +127,10 @@ const automationActions = store => ({
   delete: async automation => {
     const isRowAction = sdk.automations.isRowAction(automation)
     if (isRowAction) {
-      await API.rowActions.delete({
-        tableId: automation.definition.trigger.inputs.tableId,
-        rowActionId: automation.definition.trigger.inputs.rowActionId,
-      })
+      await rowActions.delete(
+        automation.definition.trigger.inputs.tableId,
+        automation.definition.trigger.inputs.rowActionId
+      )
     } else {
       await API.deleteAutomation({
         automationId: automation?._id,
diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index 0255e4726c1..58a207bb2e1 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -86,6 +86,36 @@ export class RowActionStore extends BudiStore {
     })
     await this.refreshRowActions(tableId)
   }
+
+  rename = async (tableId, rowActionId, name) => {
+    await API.rowActions.update({
+      tableId,
+      rowActionId,
+      name,
+    })
+    await Promise.all([
+      this.refreshRowActions(tableId),
+      automationStore.actions.fetch(),
+    ])
+  }
+
+  delete = async (tableId, rowActionId) => {
+    await API.rowActions.delete({
+      tableId,
+      rowActionId,
+    })
+    await this.refreshRowActions(tableId)
+    // We don't need to refresh automations as we can only delete row actions
+    // from the automations store, so we already handle the state update there
+  }
+
+  trigger = async (sourceId, rowActionId, rowId) => {
+    await API.rowActions.trigger({
+      sourceId,
+      rowActionId,
+      rowId,
+    })
+  }
 }
 
 const store = new RowActionStore()

From e0c4273acf6d28f4a64203d3df6327b40041a82a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 19:56:32 +0100
Subject: [PATCH 062/455] Don't wait for things we don't care about

---
 .../builder/src/stores/builder/rowActions.js  | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index 58a207bb2e1..e91ca9bbefa 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -48,24 +48,20 @@ export class RowActionStore extends BudiStore {
       getName: x => x.name,
     })
 
-    // Create the action and update state
+    // Create the action
     const res = await API.rowActions.create({
       name,
       tableId,
     })
-    this.update(state => ({
-      ...state,
-      [tableId]: [...(state[tableId] || []), res],
-    }))
 
-    // If adding to a view, enable on this view
+    // Enable action on this view if adding via a view
     if (viewId) {
       await this.enableView(tableId, viewId, res.id)
+    } else {
+      await this.refreshRowActions(tableId)
     }
 
-    // Refresh automations so we have this new row action automation
-    await automationStore.actions.fetch()
-
+    automationStore.actions.fetch()
     return res
   }
 
@@ -93,10 +89,8 @@ export class RowActionStore extends BudiStore {
       rowActionId,
       name,
     })
-    await Promise.all([
-      this.refreshRowActions(tableId),
-      automationStore.actions.fetch(),
-    ])
+    await this.refreshRowActions(tableId)
+    automationStore.actions.fetch()
   }
 
   delete = async (tableId, rowActionId) => {

From 7db263dd3b669a3ead8f41c1c85e2a0c6e3ad1ef Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 20:05:14 +0100
Subject: [PATCH 063/455] Go to newly created row actions

---
 .../backend/DataTable/buttons/grid/GridRowActionsButton.svelte | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 5bd2b757e46..26383c4b4a5 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -10,7 +10,6 @@
   import DetailPopover from "components/common/DetailPopover.svelte"
   import { getContext } from "svelte"
   import { appStore, rowActions } from "stores/builder"
-  import { API } from "api"
   import { goto, url } from "@roxi/routify"
   import { derived } from "svelte/store"
 
@@ -34,7 +33,7 @@
     try {
       const newRowAction = await rowActions.createRowAction(tableId, viewId)
       notifications.success("Row action created successfully")
-      // $goto($rowActionUrl(newRowAction))
+      $goto($rowActionUrl(newRowAction))
     } catch (error) {
       console.error(error)
       notifications.error("Error creating row action")

From 26b57843451da97af39603c12dcbaa1b86dc3141 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Sep 2024 20:10:46 +0100
Subject: [PATCH 064/455] Fix navigation to new row actions not working

---
 packages/builder/src/stores/builder/rowActions.js | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index e91ca9bbefa..a6f214f3606 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -56,12 +56,17 @@ export class RowActionStore extends BudiStore {
 
     // Enable action on this view if adding via a view
     if (viewId) {
-      await this.enableView(tableId, viewId, res.id)
+      await Promise.all([
+        this.enableView(tableId, viewId, res.id),
+        automationStore.actions.fetch(),
+      ])
     } else {
-      await this.refreshRowActions(tableId)
+      await Promise.all([
+        this.refreshRowActions(tableId),
+        automationStore.actions.fetch(),
+      ])
     }
 
-    automationStore.actions.fetch()
     return res
   }
 

From 6ce481c08f14b10f6085f61a7dd71a6e99df0e9a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 08:40:37 +0100
Subject: [PATCH 065/455] Fix issue triggering row actions from data UI

---
 .../[application]/data/table/[tableId]/[viewId]/index.svelte  | 4 ++--
 .../app/[application]/data/table/[tableId]/index.svelte       | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 698e8307def..8b1ad6cda17 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -24,8 +24,8 @@
   $: buttons = makeRowActionButtons($rowActions[id])
   $: rowActions.refreshRowActions(id)
 
-  const makeRowActionButtons = rowActions => {
-    return (rowActions || []).map(action => ({
+  const makeRowActionButtons = actions => {
+    return (actions || []).map(action => ({
       text: action.name,
       onClick: async row => {
         await rowActions.trigger(id, action.id, row._id)
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 09c20cf059c..82d27bd4b84 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -62,8 +62,8 @@
   $: buttons = makeRowActionButtons($rowActions[id])
   $: rowActions.refreshRowActions(id)
 
-  const makeRowActionButtons = rowActions => {
-    return (rowActions || []).map(action => ({
+  const makeRowActionButtons = actions => {
+    return (actions || []).map(action => ({
       text: action.name,
       onClick: async row => {
         await rowActions.trigger(id, action.id, row._id)

From b067b8ad541ebe0550ff198be6adb23402fc2baf Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 09:19:51 +0100
Subject: [PATCH 066/455] Update relationship popover styles and position

---
 .../grid/cells/RelationshipCell.svelte        | 20 +++++++++++++------
 .../grid/overlays/GridPopover.svelte          |  3 ++-
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index 73c8a99cc27..c95faa0b9ff 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -29,6 +29,7 @@
   let searching = false
   let container
   let anchor
+  let relationshipAnchor
   let relationshipFields
 
   $: fieldValue = parseValue(value)
@@ -57,6 +58,7 @@
   }, {})
 
   $: showRelationshipFields =
+    relationshipAnchor &&
     relationshipFields &&
     Object.keys(relationshipFields).length &&
     focused &&
@@ -242,11 +244,13 @@
     return value
   }
 
-  const displayRelationshipFields = relationship => {
+  const displayRelationshipFields = (e, relationship) => {
+    relationshipAnchor = e.target
     relationshipFields = relationFields[relationship._id]
   }
 
   const hideRelationshipFields = () => {
+    relationshipAnchor = null
     relationshipFields = undefined
   }
 
@@ -281,7 +285,7 @@
           <div
             class="badge"
             class:extra-info={!!relationFields[relationship._id]}
-            on:mouseover={() => displayRelationshipFields(relationship)}
+            on:mouseenter={e => displayRelationshipFields(e, relationship)}
             on:focus={() => {}}
             on:mouseleave={() => hideRelationshipFields()}
           >
@@ -359,7 +363,7 @@
 {/if}
 
 {#if showRelationshipFields}
-  <GridPopover {anchor} minWidth={300} maxWidth={400}>
+  <GridPopover anchor={relationshipAnchor} maxWidth={400} offset={4}>
     <div class="relationship-fields">
       {#each Object.entries(relationshipFields) as [fieldName, fieldValue]}
         <div class="relationship-field-name">
@@ -543,15 +547,19 @@
   .relationship-fields {
     margin: var(--spacing-m) var(--spacing-l);
     display: grid;
-    grid-template-columns: minmax(auto, 50%) auto;
+    grid-template-columns: auto 1fr;
     grid-row-gap: var(--spacing-m);
-    grid-column-gap: var(--spacing-m);
+    grid-column-gap: var(--spacing-xl);
   }
 
   .relationship-field-name {
     text-transform: uppercase;
     color: var(--spectrum-global-color-gray-600);
-    font-size: var(--font-size-xs);
+    font-size: 12px;
+    overflow: hidden;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+    max-width: 120px;
   }
   .relationship-field-value {
     overflow: hidden;
diff --git a/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte b/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte
index ac2a9984287..80647a494e1 100644
--- a/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte
@@ -15,6 +15,7 @@
   export let open = true
   export let resizable = false
   export let wrap = true
+  export let offset = 0
 
   const { gridID } = getContext("grid")
   const dispatch = createEventDispatcher()
@@ -42,8 +43,8 @@
   {align}
   {resizable}
   {wrap}
+  {offset}
   portalTarget="#{gridID} .grid-popover-container"
-  offset={0}
 >
   <div
     class="grid-popover-contents"

From c8660da7d0f7ea637d20207dce57e7c20d23fd9d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 09:22:06 +0100
Subject: [PATCH 067/455] Update relationship column popover chevron to show
 active state when open

---
 .../backend/DataTable/buttons/grid/ColumnsSettingContent.svelte  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
index 835c650c702..0bdba160cba 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
@@ -205,6 +205,7 @@
               size="S"
               icon="ChevronRight"
               quiet
+              selected={relationshipFieldName === column.name}
             />
           </div>
         {/if}

From 857122f99e22df8caa5d3fd62fdfd7b3728cfb6d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 09:32:14 +0100
Subject: [PATCH 068/455] Remove unnecessary API calls when showing or hiding
 columns

---
 .../DataTable/buttons/grid/ColumnsSettingContent.svelte        | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
index 0bdba160cba..4787c8bcce7 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
@@ -160,9 +160,6 @@
       await datasource.actions.saveSchemaMutations()
     } catch (e) {
       notifications.error(e.message)
-    } finally {
-      await datasource.actions.resetSchemaMutations()
-      await datasource.actions.refreshDefinition()
     }
     dispatch(visible ? "show-column" : "hide-column")
   }

From 54e09267f1f1b259f71ac682ab97c2e03ce1d4f5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 09:35:56 +0100
Subject: [PATCH 069/455] Only show hover state in relationship badges when
 fields are available to show

---
 .../src/components/grid/cells/RelationshipCell.svelte         | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index c95faa0b9ff..3f3122d60da 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -465,10 +465,8 @@
     height: 20px;
     max-width: 100%;
   }
-  .values.wrap .badge:hover {
+  .values.wrap .badge.extra-info:hover {
     filter: brightness(1.25);
-  }
-  .values.wrap .badge.extra-info {
     cursor: pointer;
   }
 

From 409580c78726a47ee9d17893eacb2fb5434a375d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 09:49:34 +0100
Subject: [PATCH 070/455] Fix issue preventing initial click of deleting
 relationship from working when relationship popover is open

---
 .../src/components/grid/cells/RelationshipCell.svelte     | 8 +++++++-
 .../src/components/grid/overlays/GridPopover.svelte       | 2 ++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index 3f3122d60da..afdc3d2f303 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -208,6 +208,7 @@
 
   // Toggles whether a row is included in the relationship or not
   const toggleRow = async row => {
+    hideRelationshipFields()
     if (fieldValue?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
       // row to be the same position if possible
@@ -363,7 +364,12 @@
 {/if}
 
 {#if showRelationshipFields}
-  <GridPopover anchor={relationshipAnchor} maxWidth={400} offset={4}>
+  <GridPopover
+    anchor={relationshipAnchor}
+    maxWidth={400}
+    offset={4}
+    clickOutsideOverride
+  >
     <div class="relationship-fields">
       {#each Object.entries(relationshipFields) as [fieldName, fieldValue]}
         <div class="relationship-field-name">
diff --git a/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte b/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte
index 80647a494e1..dcbcb0fc239 100644
--- a/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/GridPopover.svelte
@@ -16,6 +16,7 @@
   export let resizable = false
   export let wrap = true
   export let offset = 0
+  export let clickOutsideOverride
 
   const { gridID } = getContext("grid")
   const dispatch = createEventDispatcher()
@@ -44,6 +45,7 @@
   {resizable}
   {wrap}
   {offset}
+  {clickOutsideOverride}
   portalTarget="#{gridID} .grid-popover-container"
 >
   <div

From db9822fa644da7e09589e83462f9cf513dcd609c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 09:51:26 +0100
Subject: [PATCH 071/455] Lint

---
 .../automation/AutomationPanel/UpdateRowActionModal.svelte     | 3 +--
 packages/builder/src/helpers/featureFlags.js                   | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
index db22dab678e..20f06608dc4 100644
--- a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { automationStore, rowActions } from "stores/builder"
+  import { rowActions } from "stores/builder"
   import {
     notifications,
     Icon,
@@ -7,7 +7,6 @@
     ModalContent,
     Modal,
   } from "@budibase/bbui"
-  import { API } from "api"
 
   export let automation
   export let onCancel = undefined
diff --git a/packages/builder/src/helpers/featureFlags.js b/packages/builder/src/helpers/featureFlags.js
index 708c605cd56..e9054e8a9ca 100644
--- a/packages/builder/src/helpers/featureFlags.js
+++ b/packages/builder/src/helpers/featureFlags.js
@@ -3,5 +3,5 @@ import { get } from "svelte/store"
 
 export const isEnabled = featureFlag => {
   const user = get(auth).user
-  return true
+  return !!user?.flags?.[featureFlag]
 }

From 07b9ac1a37528e75a935335ec1c39bc7a09aed56 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 10:34:13 +0100
Subject: [PATCH 072/455] Revert enriched relationships flag to false even in
 dev

---
 packages/backend-core/src/features/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index d7bbcb276cd..49f8044e8ff 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -268,5 +268,5 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   DEFAULT_VALUES: Flag.boolean(env.isDev()),
   SQS: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(false),
 })

From b60923f9292d173d6d10531852b8399f481a0f38 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 11:12:26 +0100
Subject: [PATCH 073/455] Update collapsed button groups to support alignment

---
 packages/client/manifest.json                 | 12 +---
 .../src/components/app/ButtonGroup.svelte     | 55 +++++++++----------
 2 files changed, 28 insertions(+), 39 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index c84495ac0cb..eefe3a74b1f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -517,11 +517,7 @@
                 "barTitle": "Align stretched horizontally"
               }
             ],
-            "defaultValue": "left",
-            "dependsOn": {
-              "setting": "collapsed",
-              "invert": true
-            }
+            "defaultValue": "left"
           },
           {
             "type": "select",
@@ -555,11 +551,7 @@
                 "barTitle": "Align stretched vertically"
               }
             ],
-            "defaultValue": "top",
-            "dependsOn": {
-              "setting": "collapsed",
-              "invert": true
-            }
+            "defaultValue": "top"
           },
           {
             "type": "select",
diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte
index 158bfb26b9b..6907f1b1294 100644
--- a/packages/client/src/components/app/ButtonGroup.svelte
+++ b/packages/client/src/components/app/ButtonGroup.svelte
@@ -12,8 +12,7 @@
   export let collapsed = false
   export let collapsedText = "Action"
 
-  const { styleable, enrichButtonActions } = getContext("sdk")
-  const component = getContext("component")
+  const { enrichButtonActions } = getContext("sdk")
   const context = getContext("context")
 
   $: collapsedButtons = collapsed ? makeCollapsed(buttons) : null
@@ -29,30 +28,28 @@
   }
 </script>
 
-{#if collapsed}
-  <div use:styleable={$component.styles}>
-    <CollapsedButtonGroup
-      text={collapsedText || "Action"}
-      buttons={collapsedButtons}
-    />
-  </div>
-{:else}
-  <Block>
-    <BlockComponent
-      type="container"
-      props={{
-        direction,
-        hAlign,
-        vAlign,
-        gap,
-        wrap: true,
-      }}
-      styles={{
-        normal: {
-          height: "100%",
-        },
-      }}
-    >
+<Block>
+  <BlockComponent
+    type="container"
+    props={{
+      direction,
+      hAlign,
+      vAlign,
+      gap,
+      wrap: true,
+    }}
+    styles={{
+      normal: {
+        height: "100%",
+      },
+    }}
+  >
+    {#if collapsed}
+      <CollapsedButtonGroup
+        text={collapsedText || "Action"}
+        buttons={collapsedButtons}
+      />
+    {:else}
       {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }}
         <BlockComponent
           type="button"
@@ -68,6 +65,6 @@
           }}
         />
       {/each}
-    </BlockComponent>
-  </Block>
-{/if}
+    {/if}
+  </BlockComponent>
+</Block>

From 07df8157b3d9f832c21404d1aecc8ac12c92f218 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 11:20:31 +0100
Subject: [PATCH 074/455] Update data section background colours for a better
 loading experience

---
 .../data/table/[tableId]/_components/ViewNavBar.svelte         | 1 +
 .../app/[application]/data/table/[tableId]/_layout.svelte      | 2 +-
 .../data/table/[tableId]/v1/[viewName]/index.svelte            | 3 +--
 packages/frontend-core/src/components/grid/layout/Grid.svelte  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index e403e0bcd7e..d74fb90cb29 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -328,6 +328,7 @@
     align-items: center;
     padding: 0 var(--spacing-xl);
     gap: 8px;
+    background: var(--background);
   }
   .nav__views {
     flex: 0 1 auto;
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte
index 4a33f8a9673..da05196c043 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte
@@ -32,6 +32,6 @@
     margin: -28px -40px -40px -40px;
     display: flex;
     flex-direction: column;
-    background: var(--background);
+    background: var(--spectrum-global-color-gray-50);
   }
 </style>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
index 16bdfabb41e..2c822569b73 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte
@@ -59,7 +59,6 @@
 <div class="view-v1">
   {#if view}
     <Table
-      title={decodeURI(name)}
       {schema}
       tableId={view.tableId}
       {data}
@@ -82,7 +81,7 @@
 
 <style>
   .view-v1 {
-    padding: var(--spacing-xl);
+    padding: 0 var(--spacing-xl);
   }
   i {
     font-size: var(--font-size-m);
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 7e28f249da4..5f6de670df1 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -321,7 +321,7 @@
     width: 100%;
     height: 100%;
     background: var(--grid-background-alt);
-    opacity: 0.6;
+    opacity: 0.3;
   }
 
   /* Error */

From 238b1cecf1aadddd212f0e2b69ebd0615ae673b8 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 11:28:49 +0100
Subject: [PATCH 075/455] Highlight row action row rather than select

---
 packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte      | 2 ++
 .../backend/DataTable/buttons/grid/GridGenerateButton.svelte   | 2 +-
 .../src/components/grid/layout/ButtonColumn.svelte             | 3 +--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
index 5f6eca0b130..4f94ac955ec 100644
--- a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -39,6 +39,8 @@
   resizable={false}
   on:close
   on:open
+  on:mouseenter
+  on:mouseleave
 >
   <Menu>
     {#each buttons as button}
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index cb2afbc0008..5cc3aca19e4 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -110,7 +110,7 @@
     <div class="generate-section__options">
       <div>
         <ListItem
-          title="CRUD app"
+          title="Table"
           icon="TableEdit"
           hoverable
           on:click={() => startScreenWizard(AutoScreenTypes.TABLE)}
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 03f5f4644e7..6602565c3ec 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -100,8 +100,7 @@
                   offset={5}
                   size="S"
                   animate={false}
-                  on:open={() => selectedRows.set({ [row._id]: true })}
-                  on:close={() => selectedRows.set({})}
+                  on:mouseenter={() => ($hoveredRowId = row._id)}
                 />
               {:else}
                 {#each buttons as button}

From 5b6809b913b5504d51a44eddbc26fc5dacf13c11 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Sep 2024 15:41:10 +0100
Subject: [PATCH 076/455] Fix tests

---
 packages/builder/src/stores/BudiStore.js          | 5 -----
 packages/builder/src/stores/builder/rowActions.js | 4 ++++
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/stores/BudiStore.js b/packages/builder/src/stores/BudiStore.js
index e7be02f2831..e854aa65883 100644
--- a/packages/builder/src/stores/BudiStore.js
+++ b/packages/builder/src/stores/BudiStore.js
@@ -2,7 +2,6 @@ import { writable } from "svelte/store"
 
 export default class BudiStore {
   constructor(init, opts) {
-    this.initialState = init
     const store = writable({ ...init })
 
     /**
@@ -33,8 +32,4 @@ export default class BudiStore {
       })
     }
   }
-
-  reset = () => {
-    this.store.set({ ...this.initialState })
-  }
 }
diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index a6f214f3606..1b6fb57e87a 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -13,6 +13,10 @@ export class RowActionStore extends BudiStore {
     super(initialState)
   }
 
+  reset = () => {
+    this.store.set(initialState)
+  }
+
   refreshRowActions = async sourceId => {
     if (!sourceId) {
       return

From da6ce2256af3c769a4a7f6396cc50bc790a94034 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Sep 2024 16:46:55 +0100
Subject: [PATCH 077/455] Fix issue with button column width and allow
 collapsed button groups to use a secondary variant

---
 packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte  | 4 +++-
 .../src/components/grid/layout/ButtonColumn.svelte         | 7 ++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
index 4f94ac955ec..12dee055e2c 100644
--- a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -10,6 +10,7 @@
   export let align = "left"
   export let offset
   export let animate
+  export let secondary = false
 
   let anchor
   let popover
@@ -24,7 +25,8 @@
   bind:ref={anchor}
   {size}
   icon="ChevronDown"
-  cta
+  cta={!secondary}
+  {secondary}
   on:click={() => popover?.show()}
   on:click
 >
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 6602565c3ec..1e40f326ff9 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -57,7 +57,7 @@
   onMount(() => {
     const observer = new ResizeObserver(entries => {
       const width = entries?.[0]?.contentRect?.width ?? 0
-      buttonColumnWidth.set(width)
+      buttonColumnWidth.set(width - 1)
     })
     observer.observe(container)
   })
@@ -68,7 +68,6 @@
   class="button-column"
   style="left:{left}px"
   class:hidden={$buttonColumnWidth === 0}
-  class:right-border={left !== gridEnd}
 >
   <div class="content" on:mouseleave={() => ($hoveredRowId = null)}>
     <GridScrollWrapper scrollVertically attachHandlers bind:ref={container}>
@@ -99,6 +98,7 @@
                   align="right"
                   offset={5}
                   size="S"
+                  secondary
                   animate={false}
                   on:mouseenter={() => ($hoveredRowId = row._id)}
                 />
@@ -188,7 +188,4 @@
   .button-column :global(.cell) {
     border-left: var(--cell-border);
   }
-  .button-column:not(.right-border) :global(.cell) {
-    border-right-color: transparent;
-  }
 </style>

From 61b82b8817f8d3525e977299e3f39cd0862bcaeb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 6 Sep 2024 09:20:00 +0100
Subject: [PATCH 078/455] Add quiet reverse button variant for collapsed button
 groups in grids

---
 packages/bbui/src/Button/Button.svelte              | 13 ++++++++++++-
 .../src/ButtonGroup/CollapsedButtonGroup.svelte     |  9 ++++++---
 .../src/components/grid/layout/ButtonColumn.svelte  |  4 ++--
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte
index 7971c0ea8d8..0a8917c3c1a 100644
--- a/packages/bbui/src/Button/Button.svelte
+++ b/packages/bbui/src/Button/Button.svelte
@@ -18,6 +18,7 @@
   export let newStyles = true
   export let id
   export let ref
+  export let reverse = false
 
   const dispatch = createEventDispatcher()
 </script>
@@ -43,6 +44,9 @@
       }
     }}
   >
+    {#if $$slots && reverse}
+      <span class="spectrum-Button-label"><slot /></span>
+    {/if}
     {#if icon}
       <svg
         class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
@@ -53,7 +57,7 @@
         <use xlink:href="#spectrum-icon-18-{icon}" />
       </svg>
     {/if}
-    {#if $$slots}
+    {#if $$slots && !reverse}
       <span class="spectrum-Button-label"><slot /></span>
     {/if}
   </button>
@@ -93,4 +97,11 @@
   .spectrum-Button--secondary.new-styles.is-disabled {
     color: var(--spectrum-global-color-gray-500);
   }
+  .spectrum-Button .spectrum-Button-label + .spectrum-Icon {
+    margin-left: var(--spectrum-button-primary-icon-gap);
+    margin-right: calc(
+      -1 * (var(--spectrum-button-primary-textonly-padding-left-adjusted) -
+            var(--spectrum-button-primary-padding-left-adjusted))
+    );
+  }
 </style>
diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
index 12dee055e2c..d7aad5ccffb 100644
--- a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
+++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte
@@ -10,7 +10,7 @@
   export let align = "left"
   export let offset
   export let animate
-  export let secondary = false
+  export let quiet = false
 
   let anchor
   let popover
@@ -25,10 +25,13 @@
   bind:ref={anchor}
   {size}
   icon="ChevronDown"
-  cta={!secondary}
-  {secondary}
+  {quiet}
+  primary={quiet}
+  cta={!quiet}
+  newStyles={!quiet}
   on:click={() => popover?.show()}
   on:click
+  reverse
 >
   {text || "Action"}
 </Button>
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 1e40f326ff9..371e8d46af3 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -1,6 +1,6 @@
 <script>
   import { getContext, onMount } from "svelte"
-  import { Button, CollapsedButtonGroup } from "@budibase/bbui"
+  import { Button, CollapsedButtonGroup, Select } from "@budibase/bbui"
   import GridCell from "../cells/GridCell.svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import { BlankRowID } from "../lib/constants"
@@ -98,7 +98,7 @@
                   align="right"
                   offset={5}
                   size="S"
-                  secondary
+                  quiet
                   animate={false}
                   on:mouseenter={() => ($hoveredRowId = row._id)}
                 />

From 5de81c624fe72ffda1d5f2a8493f77715367742a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 6 Sep 2024 11:16:27 +0100
Subject: [PATCH 079/455] Expose refresh datasource action from form blocks and
 add row action button templates

---
 .../ButtonConfiguration.svelte                | 21 ++++-
 .../Component/ComponentSettingsSection.svelte | 11 +++
 packages/builder/src/templates/rowActions.js  | 91 +++++++++++++++++++
 packages/client/manifest.json                 |  8 ++
 4 files changed, 127 insertions(+), 4 deletions(-)
 create mode 100644 packages/builder/src/templates/rowActions.js

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index db2289345fd..212f96bdca2 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -6,6 +6,7 @@
   import { componentStore } from "stores/builder"
   import { getEventContextBindings } from "dataBinding"
   import { cloneDeep, isEqual } from "lodash/fp"
+  import { getRowActionButtonTemplates } from "templates/rowActions"
 
   export let componentInstance
   export let componentBindings
@@ -19,11 +20,14 @@
 
   let focusItem
   let cachedValue
+  let rowActionTemplates = []
 
+  $: console.log(rowActionTemplates)
+
+  $: getRowActionTemplates(componentInstance)
   $: if (!isEqual(value, cachedValue)) {
     cachedValue = cloneDeep(value)
   }
-
   $: buttonList = sanitizeValue(cachedValue) || []
   $: buttonCount = buttonList.length
   $: eventContextBindings = getEventContextBindings({
@@ -39,6 +43,11 @@
   }
   $: canAddButtons = max == null || buttonList.length < max
 
+  const getRowActionTemplates = async instance => {
+    const templates = await getRowActionButtonTemplates({ component: instance })
+    rowActionTemplates = templates
+  }
+
   const sanitizeValue = val => {
     if (!Array.isArray(val)) {
       return null
@@ -79,9 +88,13 @@
   }
 
   const addButton = () => {
-    const newButton = buildPseudoInstance({
-      text: `Button ${buttonCount + 1}`,
-    })
+    // const newButton = buildPseudoInstance({
+    //   text: `Button ${buttonCount + 1}`,
+    // })
+    const newButton = rowActionTemplates[0]
+    if (!newButton) {
+      return
+    }
     dispatch("change", [...buttonList, newButton])
     focusItem = newButton._id
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index d5a696c6bfc..a433e5ae791 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -8,6 +8,7 @@
   import InfoDisplay from "./InfoDisplay.svelte"
   import analytics, { Events } from "analytics"
   import { shouldDisplaySetting } from "@budibase/frontend-core"
+  import { getContext, setContext } from "svelte"
 
   export let componentDefinition
   export let componentInstance
@@ -19,6 +20,16 @@
   export let includeHidden = false
   export let tag
 
+  // Sometimes we render component settings using a complicated nested
+  // component instance technique. This results in instances with IDs that
+  // don't exist anywhere in the tree. Therefore we need to keep track of
+  // what the real component tree ID is so we can always find it.
+  const rootId = getContext("rootId")
+  if (!rootId) {
+    setContext("rootId", componentInstance._id)
+  }
+  componentInstance._rootId = rootId || componentInstance._id
+
   $: sections = getSections(
     componentInstance,
     componentDefinition,
diff --git a/packages/builder/src/templates/rowActions.js b/packages/builder/src/templates/rowActions.js
new file mode 100644
index 00000000000..df02424f21e
--- /dev/null
+++ b/packages/builder/src/templates/rowActions.js
@@ -0,0 +1,91 @@
+import { get } from "svelte/store"
+import { getDatasourceForProvider } from "dataBinding"
+import { rowActions, selectedScreen, componentStore } from "stores/builder"
+import { Helpers } from "@budibase/bbui"
+import { findComponent } from "helpers/components"
+
+export const getRowActionButtonTemplates = async ({ screen, component }) => {
+  if (!component) {
+    return []
+  }
+  if (!screen) {
+    screen = get(selectedScreen)
+  }
+  const id = component._rootId
+  const instance = findComponent(screen?.props, id)
+  if (!instance) {
+    return []
+  }
+
+  // The row ID binding depends on what component this is.
+  // Therefore we need to whitelist this to only function for certain components.
+  const type = instance?._component
+  const isGridBlock = type?.endsWith("/gridblock")
+  const isFormBlock =
+    type?.endsWith("/formblock") || type?.endsWith("/multistepformblock")
+  if (!isGridBlock && !isFormBlock) {
+    return []
+  }
+
+  // Check we have a valid datasource that can contain row actions
+  const ds = getDatasourceForProvider(get(selectedScreen), instance)
+  if (ds?.type !== "table" && ds?.type !== "viewV2") {
+    return []
+  }
+  const resourceId = ds.id || ds.tableId
+  if (!resourceId) {
+    return []
+  }
+  await rowActions.refreshRowActions(resourceId)
+  const enabledActions = get(rowActions)[resourceId] || []
+
+  // Generate the row ID binding depending on the component
+  let rowIdBinding
+  if (isGridBlock) {
+    rowIdBinding = `{{ [${instance._id}].[_id] }}`
+  } else if (isFormBlock) {
+    rowIdBinding = `{{ [${instance._id}-repeater].[_id] }}`
+  }
+
+  // Create templates
+  return enabledActions.map(action => {
+    // Create a button instance
+    const button = componentStore.createInstance(
+      `@budibase/standard-components/button`,
+      {
+        _instanceName: Helpers.uuid(),
+        text: action.name,
+        type: "primary",
+      }
+    )
+
+    // Row action button action
+    const onClick = [
+      {
+        parameters: {
+          rowActionId: action.id,
+          resourceId,
+          rowId: rowIdBinding,
+        },
+        "##eventHandlerType": "Row Action",
+        id: Helpers.uuid(),
+      },
+    ]
+
+    // For form blocks we need to manually refresh the form after running the action
+    if (isFormBlock) {
+      onClick.push({
+        parameters: {
+          componentId: `${instance._id}-provider`,
+        },
+        "##eventHandlerType": "Refresh Data Provider",
+        id: Helpers.uuid(),
+      })
+    }
+
+    return {
+      ...button,
+      onClick,
+    }
+  })
+}
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index eefe3a74b1f..40d91bf628f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6998,6 +6998,10 @@
       {
         "type": "ChangeFormStep",
         "suffix": "form"
+      },
+      {
+        "type": "RefreshDatasource",
+        "suffix": "provider"
       }
     ],
     "context": [
@@ -7241,6 +7245,10 @@
       {
         "type": "ScrollTo",
         "suffix": "form"
+      },
+      {
+        "type": "RefreshDatasource",
+        "suffix": "provider"
       }
     ],
     "context": [

From 6172fddc4035997984b3af6728a3b4b6f35645f9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 6 Sep 2024 16:19:08 +0100
Subject: [PATCH 080/455] Improve button action templates and remove broken
 focus functionality

---
 .../ButtonConfiguration.svelte                | 62 ++++++++++++-------
 .../Component/ComponentSettingsSection.svelte |  2 +-
 packages/builder/src/templates/rowActions.js  |  2 +-
 3 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index 212f96bdca2..f75c378f5b1 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -1,8 +1,8 @@
 <script>
   import DraggableList from "../DraggableList/DraggableList.svelte"
   import ButtonSetting from "./ButtonSetting.svelte"
-  import { createEventDispatcher } from "svelte"
-  import { Helpers } from "@budibase/bbui"
+  import { createEventDispatcher, onMount } from "svelte"
+  import { Helpers, Menu, MenuItem, Popover } from "@budibase/bbui"
   import { componentStore } from "stores/builder"
   import { getEventContextBindings } from "dataBinding"
   import { cloneDeep, isEqual } from "lodash/fp"
@@ -18,13 +18,11 @@
 
   const dispatch = createEventDispatcher()
 
-  let focusItem
   let cachedValue
   let rowActionTemplates = []
+  let anchor
+  let popover
 
-  $: console.log(rowActionTemplates)
-
-  $: getRowActionTemplates(componentInstance)
   $: if (!isEqual(value, cachedValue)) {
     cachedValue = cloneDeep(value)
   }
@@ -43,11 +41,6 @@
   }
   $: canAddButtons = max == null || buttonList.length < max
 
-  const getRowActionTemplates = async instance => {
-    const templates = await getRowActionButtonTemplates({ component: instance })
-    rowActionTemplates = templates
-  }
-
   const sanitizeValue = val => {
     if (!Array.isArray(val)) {
       return null
@@ -82,21 +75,32 @@
         _instanceName: Helpers.uuid(),
         text: cfg.text,
         type: cfg.type || "primary",
-      },
-      {}
+      }
     )
   }
 
-  const addButton = () => {
-    // const newButton = buildPseudoInstance({
-    //   text: `Button ${buttonCount + 1}`,
-    // })
-    const newButton = rowActionTemplates[0]
-    if (!newButton) {
-      return
-    }
+  const addCustomButton = () => {
+    const newButton = buildPseudoInstance({
+      text: `Button ${buttonCount + 1}`,
+    })
     dispatch("change", [...buttonList, newButton])
-    focusItem = newButton._id
+    popover.hide()
+  }
+
+  const addRowActionTemplate = template => {
+    dispatch("change", [...buttonList, template])
+    popover.hide()
+  }
+
+  const addButton = async () => {
+    rowActionTemplates = await getRowActionButtonTemplates({
+      component: componentInstance,
+    })
+    if (rowActionTemplates.length) {
+      popover.show()
+    } else {
+      addCustomButton()
+    }
   }
 
   const removeButton = id => {
@@ -118,12 +122,11 @@
       listItemKey={"_id"}
       listType={ButtonSetting}
       listTypeProps={itemProps}
-      focus={focusItem}
       draggable={buttonCount > 1}
     />
   {/if}
-
   <div
+    bind:this={anchor}
     class="list-footer"
     class:disabled={!canAddButtons}
     on:click={addButton}
@@ -133,6 +136,17 @@
   </div>
 </div>
 
+<Popover bind:this={popover} {anchor} useAnchorWidth resizable={false}>
+  <Menu>
+    <MenuItem on:click={addCustomButton}>Custom button</MenuItem>
+    {#each rowActionTemplates as template}
+      <MenuItem on:click={() => addRowActionTemplate(template)}>
+        {template.text}
+      </MenuItem>
+    {/each}
+  </Menu>
+</Popover>
+
 <style>
   .button-configuration :global(.spectrum-ActionButton) {
     width: 100%;
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index a433e5ae791..d40f28e65ab 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -28,7 +28,7 @@
   if (!rootId) {
     setContext("rootId", componentInstance._id)
   }
-  componentInstance._rootId = rootId || componentInstance._id
+  $: componentInstance._rootId = rootId || componentInstance._id
 
   $: sections = getSections(
     componentInstance,
diff --git a/packages/builder/src/templates/rowActions.js b/packages/builder/src/templates/rowActions.js
index df02424f21e..6b40a848f26 100644
--- a/packages/builder/src/templates/rowActions.js
+++ b/packages/builder/src/templates/rowActions.js
@@ -28,7 +28,7 @@ export const getRowActionButtonTemplates = async ({ screen, component }) => {
   }
 
   // Check we have a valid datasource that can contain row actions
-  const ds = getDatasourceForProvider(get(selectedScreen), instance)
+  const ds = getDatasourceForProvider(screen, instance)
   if (ds?.type !== "table" && ds?.type !== "viewV2") {
     return []
   }

From 902112d5a795cd1b29433cfee2533932b4244e0d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Sat, 7 Sep 2024 19:02:36 +0100
Subject: [PATCH 081/455] Update form screen templates to include row actions
 and use a form block instead of multi step form block

---
 .../NewScreen/CreateScreenModal.svelte        | 21 ++++++----
 .../builder/src/templates/BaseStructure.js    |  4 +-
 packages/builder/src/templates/Component.js   |  1 +
 packages/builder/src/templates/rowActions.js  | 23 +++++++----
 .../src/templates/screenTemplating/form.js    | 41 +++++++++++++++----
 5 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
index 51e82a6fe2e..39026af6d03 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
@@ -123,15 +123,18 @@
   }
 
   const createFormScreen = async type => {
-    const screenTemplates = selectedTablesAndViews.flatMap(tableOrView =>
-      screenTemplating.form({
-        screens,
-        tableOrView,
-        type,
-        permissions: permissions[tableOrView.id],
-      })
-    )
-
+    const screenTemplates = (
+      await Promise.all(
+        selectedTablesAndViews.map(tableOrView =>
+          screenTemplating.form({
+            screens,
+            tableOrView,
+            type,
+            permissions: permissions[tableOrView.id],
+          })
+        )
+      )
+    ).flat()
     const newScreens = await createScreens(screenTemplates)
 
     if (type === "update" || type === "create") {
diff --git a/packages/builder/src/templates/BaseStructure.js b/packages/builder/src/templates/BaseStructure.js
index 71daca9d1b9..0398b1f2682 100644
--- a/packages/builder/src/templates/BaseStructure.js
+++ b/packages/builder/src/templates/BaseStructure.js
@@ -24,9 +24,9 @@ export class BaseStructure {
     if (this._children.length !== 0) {
       for (let child of this._children) {
         if (this._isScreen) {
-          structure.props._children.push(child.json())
+          structure.props._children.push(child.json?.() || child)
         } else {
-          structure._children.push(child.json())
+          structure._children.push(child.json?.() || child)
         }
       }
     }
diff --git a/packages/builder/src/templates/Component.js b/packages/builder/src/templates/Component.js
index 8f70cbe92be..0c672dab026 100644
--- a/packages/builder/src/templates/Component.js
+++ b/packages/builder/src/templates/Component.js
@@ -1,5 +1,6 @@
 import { Helpers } from "@budibase/bbui"
 import { BaseStructure } from "./BaseStructure"
+import { componentStore } from "stores/builder"
 
 export class Component extends BaseStructure {
   constructor(name, _id = Helpers.uuid()) {
diff --git a/packages/builder/src/templates/rowActions.js b/packages/builder/src/templates/rowActions.js
index 6b40a848f26..62a9052bcbd 100644
--- a/packages/builder/src/templates/rowActions.js
+++ b/packages/builder/src/templates/rowActions.js
@@ -4,15 +4,22 @@ import { rowActions, selectedScreen, componentStore } from "stores/builder"
 import { Helpers } from "@budibase/bbui"
 import { findComponent } from "helpers/components"
 
-export const getRowActionButtonTemplates = async ({ screen, component }) => {
-  if (!component) {
-    return []
-  }
-  if (!screen) {
-    screen = get(selectedScreen)
+export const getRowActionButtonTemplates = async ({
+  screen,
+  component,
+  instance,
+}) => {
+  // Find root component instance if not specified
+  if (!instance) {
+    if (!component) {
+      return []
+    }
+    if (!screen) {
+      screen = get(selectedScreen)
+    }
+    const id = component._rootId
+    instance = findComponent(screen?.props, id)
   }
-  const id = component._rootId
-  const instance = findComponent(screen?.props, id)
   if (!instance) {
     return []
   }
diff --git a/packages/builder/src/templates/screenTemplating/form.js b/packages/builder/src/templates/screenTemplating/form.js
index d0b75802802..769851cf88a 100644
--- a/packages/builder/src/templates/screenTemplating/form.js
+++ b/packages/builder/src/templates/screenTemplating/form.js
@@ -1,6 +1,9 @@
 import { Screen } from "./Screen"
 import { Component } from "../Component"
 import getValidRoute from "./getValidRoute"
+import { componentStore } from "stores/builder"
+import { Helpers } from "@budibase/bbui"
+import { getRowActionButtonTemplates } from "templates/rowActions"
 
 export const getTypeSpecificRoute = (tableOrView, type) => {
   if (type === "create") {
@@ -32,27 +35,49 @@ const getActionType = type => {
   }
 }
 
-const form = ({ tableOrView, type, permissions, screens }) => {
+const getTitle = type => {
+  if (type === "create") {
+    return "Create row"
+  } else if (type === "update") {
+    return "Update row"
+  }
+  return "Row details"
+}
+
+const form = async ({ tableOrView, type, permissions, screens }) => {
+  const id = Helpers.uuid()
   const typeSpecificRoute = getTypeSpecificRoute(tableOrView, type)
   const role = getRole(permissions, type)
 
-  const multistepFormBlock = new Component(
-    "@budibase/standard-components/multistepformblock"
-  )
+  let formBlock = new Component("@budibase/standard-components/formblock", id)
     .customProps({
-      actionType: getActionType(type),
       dataSource: tableOrView.tableSelectFormat,
-      steps: [{}],
+      actionType: getActionType(type),
+      title: getTitle(type),
       rowId: type === "new" ? undefined : `{{ url.id }}`,
     })
-    .instanceName(`${tableOrView.name} - Multistep Form block`)
+    .instanceName(`${tableOrView.name} - Form block`)
+    .json()
+
+  // Add default button config
+  componentStore.migrateSettings(formBlock)
+
+  // Add row action buttons if required
+  if (type !== "create") {
+    const rowActionButtons = await getRowActionButtonTemplates({
+      instance: formBlock,
+    })
+    if (rowActionButtons.length) {
+      formBlock.buttons = [...(formBlock.buttons || []), ...rowActionButtons]
+    }
+  }
 
   const template = new Screen()
     .route(getValidRoute(screens, typeSpecificRoute, role))
     .instanceName(`${tableOrView.name} - Form`)
     .role(role)
     .autoTableId(tableOrView.id)
-    .addChild(multistepFormBlock)
+    .addChild(formBlock)
     .json()
 
   return [

From 028dc40a0d7792935478670c9e941d28e4ff4153 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Sep 2024 11:35:43 +0100
Subject: [PATCH 082/455] Update form block autoscreens to render buttons at
 bottom and include quiet row action buttons

---
 packages/builder/src/templates/rowActions.js                | 1 +
 packages/builder/src/templates/screenTemplating/form.js     | 6 ++++++
 .../src/components/app/blocks/form/InnerFormBlock.svelte    | 2 +-
 packages/frontend-core/src/utils/utils.js                   | 2 +-
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/templates/rowActions.js b/packages/builder/src/templates/rowActions.js
index 62a9052bcbd..639db09d00e 100644
--- a/packages/builder/src/templates/rowActions.js
+++ b/packages/builder/src/templates/rowActions.js
@@ -63,6 +63,7 @@ export const getRowActionButtonTemplates = async ({
         _instanceName: Helpers.uuid(),
         text: action.name,
         type: "primary",
+        quiet: true,
       }
     )
 
diff --git a/packages/builder/src/templates/screenTemplating/form.js b/packages/builder/src/templates/screenTemplating/form.js
index 769851cf88a..e0fe815896c 100644
--- a/packages/builder/src/templates/screenTemplating/form.js
+++ b/packages/builder/src/templates/screenTemplating/form.js
@@ -55,6 +55,7 @@ const form = async ({ tableOrView, type, permissions, screens }) => {
       actionType: getActionType(type),
       title: getTitle(type),
       rowId: type === "new" ? undefined : `{{ url.id }}`,
+      buttonPosition: "bottom",
     })
     .instanceName(`${tableOrView.name} - Form block`)
     .json()
@@ -69,6 +70,11 @@ const form = async ({ tableOrView, type, permissions, screens }) => {
     })
     if (rowActionButtons.length) {
       formBlock.buttons = [...(formBlock.buttons || []), ...rowActionButtons]
+
+      // Collapse buttons if more than 3 row actions
+      if (rowActionButtons.length > 3) {
+        formBlock.buttonsCollapsed = true
+      }
     }
   }
 
diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
index b5b65398b65..cf833642d27 100644
--- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
@@ -113,7 +113,7 @@
         }}
         styles={{
           normal: {
-            "margin-top": "16",
+            "margin-top": "24",
           },
         }}
         order={1}
diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js
index 3e8cdcc781d..fef9a7a31bb 100644
--- a/packages/frontend-core/src/utils/utils.js
+++ b/packages/frontend-core/src/utils/utils.js
@@ -240,7 +240,7 @@ export const buildFormBlockButtonConfig = props => {
       _component: "@budibase/standard-components/button",
       onClick: onDelete,
       quiet: true,
-      type: "secondary",
+      type: "warning",
     })
   }
 

From 82a727ae888a93f8c028d64bca45626a7cf3390a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Sep 2024 11:36:52 +0100
Subject: [PATCH 083/455] Remove max limit on grid buttons

---
 packages/client/manifest.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 40d91bf628f..0a02189c187 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -7596,7 +7596,6 @@
             "type": "buttonConfiguration",
             "key": "buttons",
             "nested": true,
-            "max": 3,
             "context": [
               {
                 "label": "Clicked row",

From 624e13a49c7298bf0832fe690d06eaef996b0894 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Sep 2024 12:15:25 +0100
Subject: [PATCH 084/455] Add row action buttons to table autoscreens

---
 packages/builder/src/helpers/components.js    |  2 +-
 .../NewScreen/CreateScreenModal.svelte        | 22 ++++---
 .../templates/screenTemplating/table/index.js | 11 ++--
 .../screenTemplating/table/inline.js          | 16 ++++-
 .../templates/screenTemplating/table/modal.js | 48 ++++++++------
 .../screenTemplating/table/newScreen.js       | 64 ++++++-------------
 .../screenTemplating/table/sidePanel.js       | 37 +++++++----
 7 files changed, 108 insertions(+), 92 deletions(-)

diff --git a/packages/builder/src/helpers/components.js b/packages/builder/src/helpers/components.js
index 872a9441a48..ca1be3c7362 100644
--- a/packages/builder/src/helpers/components.js
+++ b/packages/builder/src/helpers/components.js
@@ -213,7 +213,7 @@ export const getComponentText = component => {
     return component._instanceName
   }
   const type =
-    component._component.replace("@budibase/standard-components/", "") ||
+    component._component?.replace("@budibase/standard-components/", "") ||
     "component"
   return capitalise(type)
 }
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
index 39026af6d03..47d7c765d60 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte
@@ -103,21 +103,23 @@
 
   const createBlankScreen = async ({ route }) => {
     const screenTemplates = screenTemplating.blank({ route, screens })
-
     const newScreens = await createScreens(screenTemplates)
     loadNewScreen(newScreens[0])
   }
 
   const createTableScreen = async type => {
-    const screenTemplates = selectedTablesAndViews.flatMap(tableOrView =>
-      screenTemplating.table({
-        screens,
-        tableOrView,
-        type,
-        permissions: permissions[tableOrView.id],
-      })
-    )
-
+    const screenTemplates = (
+      await Promise.all(
+        selectedTablesAndViews.map(tableOrView =>
+          screenTemplating.table({
+            screens,
+            tableOrView,
+            type,
+            permissions: permissions[tableOrView.id],
+          })
+        )
+      )
+    ).flat()
     const newScreens = await createScreens(screenTemplates)
     loadNewScreen(newScreens[0])
   }
diff --git a/packages/builder/src/templates/screenTemplating/table/index.js b/packages/builder/src/templates/screenTemplating/table/index.js
index 40cfde6f151..a94c7745bc0 100644
--- a/packages/builder/src/templates/screenTemplating/table/index.js
+++ b/packages/builder/src/templates/screenTemplating/table/index.js
@@ -2,21 +2,22 @@ import inline from "./inline"
 import modal from "./modal"
 import sidePanel from "./sidePanel"
 import newScreen from "./newScreen"
+import { getRowActionButtonTemplates } from "templates/rowActions"
 
-const createScreen = ({ tableOrView, type, permissions, screens }) => {
+const createScreen = async ({ tableOrView, type, permissions, screens }) => {
   if (type === "inline") {
-    return inline({ tableOrView, permissions, screens })
+    return await inline({ tableOrView, permissions, screens })
   }
   if (type === "modal") {
-    return modal({ tableOrView, permissions, screens })
+    return await modal({ tableOrView, permissions, screens })
   }
 
   if (type === "sidePanel") {
-    return sidePanel({ tableOrView, permissions, screens })
+    return await sidePanel({ tableOrView, permissions, screens })
   }
 
   if (type === "newScreen") {
-    return newScreen({ tableOrView, permissions, screens })
+    return await newScreen({ tableOrView, permissions, screens })
   }
 
   throw new Error(`Unrecognized table type ${type}`)
diff --git a/packages/builder/src/templates/screenTemplating/table/inline.js b/packages/builder/src/templates/screenTemplating/table/inline.js
index d218aa998ae..b7e0dd58b82 100644
--- a/packages/builder/src/templates/screenTemplating/table/inline.js
+++ b/packages/builder/src/templates/screenTemplating/table/inline.js
@@ -2,8 +2,9 @@ import { Screen } from "../Screen"
 import { Component } from "../../Component"
 import { capitalise } from "helpers"
 import getValidRoute from "../getValidRoute"
+import { getRowActionButtonTemplates } from "templates/rowActions"
 
-const inline = ({ tableOrView, permissions, screens }) => {
+const inline = async ({ tableOrView, permissions, screens }) => {
   const heading = new Component("@budibase/standard-components/heading")
     .instanceName("Table heading")
     .customProps({
@@ -12,7 +13,7 @@ const inline = ({ tableOrView, permissions, screens }) => {
     .gridDesktopColSpan(1, 13)
     .gridDesktopRowSpan(1, 3)
 
-  const tableBlock = new Component("@budibase/standard-components/gridblock")
+  let tableBlock = new Component("@budibase/standard-components/gridblock")
     .instanceName(`${tableOrView.name} - Table`)
     .customProps({
       table: tableOrView.datasourceSelectFormat,
@@ -20,6 +21,17 @@ const inline = ({ tableOrView, permissions, screens }) => {
     .gridDesktopColSpan(1, 13)
     .gridDesktopRowSpan(3, 21)
 
+  // Add row actions to table
+  const rowActionButtons = await getRowActionButtonTemplates({
+    instance: tableBlock.json(),
+  })
+  if (rowActionButtons.length) {
+    tableBlock = tableBlock.customProps({
+      buttons: rowActionButtons,
+      buttonsCollapsed: rowActionButtons.length > 1,
+    })
+  }
+
   const screenTemplate = new Screen()
     .route(getValidRoute(screens, tableOrView.name, permissions.write))
     .instanceName(`${tableOrView.name} - List`)
diff --git a/packages/builder/src/templates/screenTemplating/table/modal.js b/packages/builder/src/templates/screenTemplating/table/modal.js
index caf1582fbb8..6a97979b316 100644
--- a/packages/builder/src/templates/screenTemplating/table/modal.js
+++ b/packages/builder/src/templates/screenTemplating/table/modal.js
@@ -5,8 +5,9 @@ import { makePropSafe as safe } from "@budibase/string-templates"
 import { Utils } from "@budibase/frontend-core"
 import { capitalise } from "helpers"
 import getValidRoute from "../getValidRoute"
+import { getRowActionButtonTemplates } from "templates/rowActions"
 
-const modal = ({ tableOrView, permissions, screens }) => {
+const modal = async ({ tableOrView, permissions, screens }) => {
   /*
     Create Row
    */
@@ -56,7 +57,7 @@ const modal = ({ tableOrView, permissions, screens }) => {
   createFormBlock.instanceName("Create row form block").customProps({
     dataSource: tableOrView.tableSelectFormat,
     labelPosition: "left",
-    buttonPosition: "top",
+    buttonPosition: "bottom",
     actionType: "Create",
     title: "Create row",
     buttons: Utils.buildFormBlockButtonConfig({
@@ -81,23 +82,34 @@ const modal = ({ tableOrView, permissions, screens }) => {
       size: "large",
     })
 
-  const editFormBlock = new Component("@budibase/standard-components/formblock")
-  editFormBlock.instanceName("Edit row form block").customProps({
-    dataSource: tableOrView.tableSelectFormat,
-    labelPosition: "left",
-    buttonPosition: "top",
-    actionType: "Update",
-    title: "Edit",
-    rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
-    buttons: Utils.buildFormBlockButtonConfig({
-      _id: editFormBlock._json._id,
-      showDeleteButton: true,
-      showSaveButton: true,
-      saveButtonLabel: "Save",
-      deleteButtonLabel: "Delete",
-      actionType: "Update",
+  let editFormBlock = new Component("@budibase/standard-components/formblock")
+    .instanceName("Edit row form block")
+    .customProps({
       dataSource: tableOrView.tableSelectFormat,
-    }),
+      labelPosition: "left",
+      buttonPosition: "bottom",
+      actionType: "Update",
+      title: "Edit",
+      rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
+    })
+
+  // Generate button config including row actions
+  let buttons = Utils.buildFormBlockButtonConfig({
+    _id: editFormBlock._json._id,
+    showDeleteButton: true,
+    showSaveButton: true,
+    saveButtonLabel: "Save",
+    deleteButtonLabel: "Delete",
+    actionType: "Update",
+    dataSource: tableOrView.tableSelectFormat,
+  })
+  const rowActionButtons = await getRowActionButtonTemplates({
+    instance: editFormBlock.json(),
+  })
+  buttons = [...(buttons || []), ...rowActionButtons]
+  editFormBlock = editFormBlock.customProps({
+    buttons,
+    buttonsCollapsed: buttons.length > 5,
   })
 
   detailsModal.addChild(editFormBlock)
diff --git a/packages/builder/src/templates/screenTemplating/table/newScreen.js b/packages/builder/src/templates/screenTemplating/table/newScreen.js
index 34026bca7f5..b8b06d3d44f 100644
--- a/packages/builder/src/templates/screenTemplating/table/newScreen.js
+++ b/packages/builder/src/templates/screenTemplating/table/newScreen.js
@@ -4,6 +4,7 @@ import { capitalise } from "helpers"
 import { makePropSafe as safe } from "@budibase/string-templates"
 import getValidRoute from "../getValidRoute"
 import { Helpers } from "@budibase/bbui"
+import { getRowActionButtonTemplates } from "templates/rowActions"
 
 const getTableScreenTemplate = ({
   route,
@@ -92,7 +93,7 @@ const getTableScreenTemplate = ({
   }
 }
 
-const getUpdateScreenTemplate = ({
+const getUpdateScreenTemplate = async ({
   route,
   tableScreenRoute,
   tableOrView,
@@ -102,27 +103,11 @@ const getUpdateScreenTemplate = ({
   const formId = `${formBlockId}-form`
   const repeaterId = `${formBlockId}-repeater`
 
-  const backButton = new Component("@budibase/standard-components/button")
-    .instanceName("Back button")
-    .customProps({
-      type: "primary",
-      icon: "ri-arrow-go-back-fill",
-      text: "Back",
-      onClick: [
-        {
-          "##eventHandlerType": "Navigate To",
-          parameters: {
-            type: "url",
-            url: tableScreenRoute,
-          },
-        },
-      ],
-    })
-
   const deleteButton = new Component("@budibase/standard-components/button")
     .instanceName("Delete button")
     .customProps({
-      type: "secondary",
+      type: "warning",
+      quiet: true,
       text: "Delete",
       onClick: [
         {
@@ -173,7 +158,7 @@ const getUpdateScreenTemplate = ({
       ],
     })
 
-  const updateFormBlock = new Component(
+  let updateFormBlock = new Component(
     "@budibase/standard-components/formblock",
     formBlockId
   )
@@ -181,12 +166,22 @@ const getUpdateScreenTemplate = ({
     .customProps({
       dataSource: tableOrView.tableSelectFormat,
       labelPosition: "left",
-      buttonPosition: "top",
+      buttonPosition: "bottom",
       actionType: "Update",
       title: `Update ${tableOrView.name} row`,
-      buttons: [backButton.json(), saveButton.json(), deleteButton.json()],
     })
 
+  // Generate button config including row actions
+  let buttons = [saveButton.json(), deleteButton.json()]
+  const rowActionButtons = await getRowActionButtonTemplates({
+    instance: updateFormBlock.json(),
+  })
+  buttons = [...(buttons || []), ...rowActionButtons]
+  updateFormBlock = updateFormBlock.customProps({
+    buttons,
+    buttonsCollapsed: buttons.length > 5,
+  })
+
   const template = new Screen()
     .route(route)
     .instanceName(`Update row`)
@@ -210,23 +205,6 @@ const getCreateScreenTemplate = ({
   const formBlockId = Helpers.uuid()
   const formId = `${formBlockId}-form`
 
-  const backButton = new Component("@budibase/standard-components/button")
-    .instanceName("Back button")
-    .customProps({
-      type: "primary",
-      icon: "ri-arrow-go-back-fill",
-      text: "Back",
-      onClick: [
-        {
-          "##eventHandlerType": "Navigate To",
-          parameters: {
-            type: "url",
-            url: tableScreenRoute,
-          },
-        },
-      ],
-    })
-
   const saveButton = new Component("@budibase/standard-components/button")
     .instanceName("Save button")
     .customProps({
@@ -264,10 +242,10 @@ const getCreateScreenTemplate = ({
     .customProps({
       dataSource: tableOrView.tableSelectFormat,
       labelPosition: "left",
-      buttonPosition: "top",
+      buttonPosition: "bottom",
       actionType: "Create",
       title: `Create ${tableOrView.name} row`,
-      buttons: [backButton.json(), saveButton.json()],
+      buttons: [saveButton.json()],
     })
 
   const template = new Screen()
@@ -284,7 +262,7 @@ const getCreateScreenTemplate = ({
   }
 }
 
-const newScreen = ({ tableOrView, permissions, screens }) => {
+const newScreen = async ({ tableOrView, permissions, screens }) => {
   const tableScreenRoute = getValidRoute(
     screens,
     tableOrView.name,
@@ -312,7 +290,7 @@ const newScreen = ({ tableOrView, permissions, screens }) => {
     gridLayout: true,
   })
 
-  const updateScreenTemplate = getUpdateScreenTemplate({
+  const updateScreenTemplate = await getUpdateScreenTemplate({
     route: updateScreenRoute,
     tableScreenRoute,
     tableOrView,
diff --git a/packages/builder/src/templates/screenTemplating/table/sidePanel.js b/packages/builder/src/templates/screenTemplating/table/sidePanel.js
index ef885977334..139b117d1ba 100644
--- a/packages/builder/src/templates/screenTemplating/table/sidePanel.js
+++ b/packages/builder/src/templates/screenTemplating/table/sidePanel.js
@@ -5,8 +5,9 @@ import { makePropSafe as safe } from "@budibase/string-templates"
 import { Utils } from "@budibase/frontend-core"
 import { capitalise } from "helpers"
 import getValidRoute from "../getValidRoute"
+import { getRowActionButtonTemplates } from "templates/rowActions"
 
-const sidePanel = ({ tableOrView, permissions, screens }) => {
+const sidePanel = async ({ tableOrView, permissions, screens }) => {
   /*
     Create Row
    */
@@ -54,7 +55,7 @@ const sidePanel = ({ tableOrView, permissions, screens }) => {
   createFormBlock.instanceName("Create row form block").customProps({
     dataSource: tableOrView.tableSelectFormat,
     labelPosition: "left",
-    buttonPosition: "top",
+    buttonPosition: "bottom",
     actionType: "Create",
     title: "Create row",
     buttons: Utils.buildFormBlockButtonConfig({
@@ -77,23 +78,33 @@ const sidePanel = ({ tableOrView, permissions, screens }) => {
     "@budibase/standard-components/sidepanel"
   ).instanceName("Edit row side panel")
 
-  const editFormBlock = new Component("@budibase/standard-components/formblock")
+  let editFormBlock = new Component("@budibase/standard-components/formblock")
   editFormBlock.instanceName("Edit row form block").customProps({
     dataSource: tableOrView.tableSelectFormat,
     labelPosition: "left",
-    buttonPosition: "top",
+    buttonPosition: "bottom",
     actionType: "Update",
     title: "Edit",
     rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
-    buttons: Utils.buildFormBlockButtonConfig({
-      _id: editFormBlock._json._id,
-      showDeleteButton: true,
-      showSaveButton: true,
-      saveButtonLabel: "Save",
-      deleteButtonLabel: "Delete",
-      actionType: "Update",
-      dataSource: tableOrView.tableSelectFormat,
-    }),
+  })
+
+  // Generate button config including row actions
+  let buttons = Utils.buildFormBlockButtonConfig({
+    _id: editFormBlock._json._id,
+    showDeleteButton: true,
+    showSaveButton: true,
+    saveButtonLabel: "Save",
+    deleteButtonLabel: "Delete",
+    actionType: "Update",
+    dataSource: tableOrView.tableSelectFormat,
+  })
+  const rowActionButtons = await getRowActionButtonTemplates({
+    instance: editFormBlock.json(),
+  })
+  buttons = [...(buttons || []), ...rowActionButtons]
+  editFormBlock = editFormBlock.customProps({
+    buttons,
+    buttonsCollapsed: buttons.length > 5,
   })
 
   detailsSidePanel.addChild(editFormBlock)

From df54d5dd2bdef528eafe3ddf907a5403666d3f52 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Sep 2024 12:21:52 +0100
Subject: [PATCH 085/455] Lint

---
 .../controls/ButtonConfiguration/ButtonConfiguration.svelte     | 2 +-
 packages/builder/src/templates/Component.js                     | 1 -
 packages/builder/src/templates/screenTemplating/table/index.js  | 1 -
 .../src/components/grid/layout/ButtonColumn.svelte              | 2 +-
 4 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
index f75c378f5b1..6f3a13a7453 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte
@@ -1,7 +1,7 @@
 <script>
   import DraggableList from "../DraggableList/DraggableList.svelte"
   import ButtonSetting from "./ButtonSetting.svelte"
-  import { createEventDispatcher, onMount } from "svelte"
+  import { createEventDispatcher } from "svelte"
   import { Helpers, Menu, MenuItem, Popover } from "@budibase/bbui"
   import { componentStore } from "stores/builder"
   import { getEventContextBindings } from "dataBinding"
diff --git a/packages/builder/src/templates/Component.js b/packages/builder/src/templates/Component.js
index 0c672dab026..8f70cbe92be 100644
--- a/packages/builder/src/templates/Component.js
+++ b/packages/builder/src/templates/Component.js
@@ -1,6 +1,5 @@
 import { Helpers } from "@budibase/bbui"
 import { BaseStructure } from "./BaseStructure"
-import { componentStore } from "stores/builder"
 
 export class Component extends BaseStructure {
   constructor(name, _id = Helpers.uuid()) {
diff --git a/packages/builder/src/templates/screenTemplating/table/index.js b/packages/builder/src/templates/screenTemplating/table/index.js
index a94c7745bc0..943d93f456d 100644
--- a/packages/builder/src/templates/screenTemplating/table/index.js
+++ b/packages/builder/src/templates/screenTemplating/table/index.js
@@ -2,7 +2,6 @@ import inline from "./inline"
 import modal from "./modal"
 import sidePanel from "./sidePanel"
 import newScreen from "./newScreen"
-import { getRowActionButtonTemplates } from "templates/rowActions"
 
 const createScreen = async ({ tableOrView, type, permissions, screens }) => {
   if (type === "inline") {
diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 371e8d46af3..0e2cf7951e4 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -1,6 +1,6 @@
 <script>
   import { getContext, onMount } from "svelte"
-  import { Button, CollapsedButtonGroup, Select } from "@budibase/bbui"
+  import { Button, CollapsedButtonGroup } from "@budibase/bbui"
   import GridCell from "../cells/GridCell.svelte"
   import GridScrollWrapper from "./GridScrollWrapper.svelte"
   import { BlankRowID } from "../lib/constants"

From 714d05a9d27e35db0455da8e0589e998522dcc9a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Sep 2024 12:36:03 +0100
Subject: [PATCH 086/455] Request new name for row actions rather than
 autonaming

---
 packages/bbui/src/Modal/ModalContent.svelte   |  4 +-
 .../buttons/grid/GridRowActionsButton.svelte  | 62 +++++++++++++++----
 .../builder/src/stores/builder/rowActions.js  | 12 ++--
 3 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte
index 61ceaeb00ac..22a14c358f2 100644
--- a/packages/bbui/src/Modal/ModalContent.svelte
+++ b/packages/bbui/src/Modal/ModalContent.svelte
@@ -30,7 +30,9 @@
   export let custom = false
 
   const { hide, cancel } = getContext(Context.Modal)
+
   let loading = false
+
   $: confirmDisabled = disabled || loading
 
   async function secondary(e) {
@@ -90,7 +92,7 @@
 
     <!-- TODO: Remove content-grid class once Layout components are in bbui -->
     <section class="spectrum-Dialog-content content-grid">
-      <slot />
+      <slot {loading} />
     </section>
     {#if showCancelButton || showConfirmButton || $$slots.footer}
       <div
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 26383c4b4a5..3f0d6f11c5a 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -6,6 +6,9 @@
     Button,
     Toggle,
     notifications,
+    Modal,
+    ModalContent,
+    Input,
   } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
   import { getContext } from "svelte"
@@ -15,6 +18,10 @@
 
   const { datasource } = getContext("grid")
 
+  let popover
+  let createModal
+  let newName
+
   $: ds = $datasource
   $: tableId = ds?.tableId
   $: viewId = ds?.id
@@ -22,6 +29,7 @@
   $: tableRowActions = $rowActions[tableId] || []
   $: viewRowActions = $rowActions[viewId] || []
   $: actionCount = isView ? viewRowActions.length : tableRowActions.length
+  $: newNameInvalid = newName && tableRowActions.some(x => x.name === newName)
 
   const rowActionUrl = derived([url, appStore], ([$url, $appStore]) => {
     return ({ automationId }) => {
@@ -29,9 +37,27 @@
     }
   })
 
+  const toggleAction = async (action, enabled) => {
+    if (enabled) {
+      await rowActions.enableView(tableId, viewId, action.id)
+    } else {
+      await rowActions.disableView(tableId, viewId, action.id)
+    }
+  }
+
+  const showCreateModal = () => {
+    newName = null
+    popover.hide()
+    createModal.show()
+  }
+
   const createRowAction = async () => {
     try {
-      const newRowAction = await rowActions.createRowAction(tableId, viewId)
+      const newRowAction = await rowActions.createRowAction(
+        tableId,
+        viewId,
+        newName
+      )
       notifications.success("Row action created successfully")
       $goto($rowActionUrl(newRowAction))
     } catch (error) {
@@ -39,17 +65,9 @@
       notifications.error("Error creating row action")
     }
   }
-
-  const toggleAction = async (action, enabled) => {
-    if (enabled) {
-      await rowActions.enableView(tableId, viewId, action.id)
-    } else {
-      await rowActions.disableView(tableId, viewId, action.id)
-    }
-  }
 </script>
 
-<DetailPopover title="Row actions">
+<DetailPopover title="Row actions" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="Engagement"
@@ -88,12 +106,34 @@
     </List>
   {/if}
   <div>
-    <Button secondary icon="Engagement" on:click={createRowAction}>
+    <Button secondary icon="Engagement" on:click={showCreateModal}>
       Create row action
     </Button>
   </div>
 </DetailPopover>
 
+<Modal bind:this={createModal}>
+  <ModalContent
+    size="S"
+    title="Create row action"
+    confirmText="Create"
+    showCancelButton={false}
+    showDivider={false}
+    showCloseIcon={false}
+    disabled={!newName || newNameInvalid}
+    onConfirm={createRowAction}
+    let:loading
+  >
+    <Input
+      label="Name"
+      bind:value={newName}
+      error={newNameInvalid && !loading
+        ? "A row action with this name already exists"
+        : null}
+    />
+  </ModalContent>
+</Modal>
+
 <style>
   span :global(.spectrum-Switch) {
     min-height: 0;
diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index 1b6fb57e87a..a8532b0c720 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -41,16 +41,18 @@ export class RowActionStore extends BudiStore {
     }))
   }
 
-  createRowAction = async (tableId, viewId) => {
+  createRowAction = async (tableId, viewId, name) => {
     if (!tableId) {
       return
     }
 
     // Get a unique name for this action
-    const existingRowActions = get(this.store)[tableId] || []
-    const name = getSequentialName(existingRowActions, "New row action ", {
-      getName: x => x.name,
-    })
+    if (!name) {
+      const existingRowActions = get(this.store)[tableId] || []
+      name = getSequentialName(existingRowActions, "New row action ", {
+        getName: x => x.name,
+      })
+    }
 
     // Create the action
     const res = await API.rowActions.create({

From 11b146fcbf0a39e344e859cb63ec6840f15de15d Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 9 Sep 2024 16:36:31 +0100
Subject: [PATCH 087/455] Updates to filter UI and API requests across budibase

---
 packages/backend-core/src/users/users.ts      |  29 -
 .../SetupPanel/AutomationBlockSetup.svelte    |  36 +-
 .../automation/SetupPanel/RowSelector.svelte  |  10 +-
 .../buttons/TableFilterButton.svelte          |  35 +-
 .../FilterEditor/FilterBuilder.svelte         |  83 +--
 .../controls/FilterEditor/FilterEditor.svelte |  40 +-
 .../GridColumnConfiguration.svelte            |  30 +-
 .../controls/TableConditionEditor.svelte      |   5 +-
 .../builder/src/stores/builder/components.js  |  13 +-
 .../builder/src/stores/portal/featureFlags.js |  14 +
 packages/builder/src/stores/portal/index.js   |   1 +
 packages/client/manifest.json                 |   3 +-
 .../src/components/app/DataProvider.svelte    |  31 +-
 .../app/dynamic-filter/DynamicFilter.svelte   |  40 +-
 .../app/dynamic-filter/FilterModal.svelte     |  14 -
 .../src/components/CoreFilterBuilder.svelte   | 505 ++++++++++++++++++
 .../src/components/FilterBuilder.svelte       | 379 -------------
 .../src/components/FilterField.svelte         | 318 +++++++++++
 .../src/components/FilterUsers.svelte         |   3 +-
 .../src/components/grid/stores/filter.js      |  18 +-
 .../frontend-core/src/components/index.js     |   2 +-
 .../frontend-core/src/fetch/CustomFetch.js    |   8 +
 packages/frontend-core/src/fetch/DataFetch.js |  13 +-
 packages/frontend-core/src/fetch/UserFetch.js |  12 +-
 .../frontend-core/src/fetch/ViewV2Fetch.js    |   6 +-
 packages/frontend-core/src/utils/utils.js     | 141 ++---
 .../server/src/api/controllers/row/views.ts   |  13 +-
 packages/shared-core/src/filters.ts           | 177 +-----
 packages/shared-core/src/utils.ts             | 135 +++++
 packages/types/src/api/web/searchFilter.ts    |  13 +-
 packages/types/src/sdk/search.ts              |   5 +
 .../src/api/controllers/global/users.ts       |   4 +-
 32 files changed, 1302 insertions(+), 834 deletions(-)
 create mode 100644 packages/builder/src/stores/portal/featureFlags.js
 delete mode 100644 packages/client/src/components/app/dynamic-filter/FilterModal.svelte
 create mode 100644 packages/frontend-core/src/components/CoreFilterBuilder.svelte
 delete mode 100644 packages/frontend-core/src/components/FilterBuilder.svelte
 create mode 100644 packages/frontend-core/src/components/FilterField.svelte

diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index 0c994d8287d..d8546afa8bf 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -17,11 +17,8 @@ import {
   ContextUser,
   CouchFindOptions,
   DatabaseQueryOpts,
-  SearchFilters,
   SearchUsersRequest,
   User,
-  BasicOperator,
-  ArrayOperator,
 } from "@budibase/types"
 import * as context from "../context"
 import { getGlobalDB } from "../context"
@@ -45,32 +42,6 @@ function removeUserPassword(users: User | User[]) {
   return users
 }
 
-export function isSupportedUserSearch(query: SearchFilters) {
-  const allowed = [
-    { op: BasicOperator.STRING, key: "email" },
-    { op: BasicOperator.EQUAL, key: "_id" },
-    { op: ArrayOperator.ONE_OF, key: "_id" },
-  ]
-  for (let [key, operation] of Object.entries(query)) {
-    if (typeof operation !== "object") {
-      return false
-    }
-    const fields = Object.keys(operation || {})
-    // this filter doesn't contain options - ignore
-    if (fields.length === 0) {
-      continue
-    }
-    const allowedOperation = allowed.find(
-      allow =>
-        allow.op === key && fields.length === 1 && fields[0] === allow.key
-    )
-    if (!allowedOperation) {
-      return false
-    }
-  }
-  return true
-}
-
 export async function bulkGetGlobalUsersById(
   userIds: string[],
   opts?: GetOpts
diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index 6c4865b539c..577f4a3f908 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -62,6 +62,7 @@
   } from "@budibase/types"
   import { FIELDS } from "constants/backend"
   import PropField from "./PropField.svelte"
+  import { utils } from "@budibase/shared-core"
 
   export let block
   export let testData
@@ -95,8 +96,14 @@
   $: memoEnvVariables.set($environment.variables)
   $: memoBlock.set(block)
 
-  $: filters = lookForFilters(schemaProperties) || []
-  $: tempFilters = filters
+  $: filters = lookForFilters(schemaProperties)
+  $: filterCount =
+    filters?.groups?.reduce((acc, group) => {
+      acc = acc += group?.filters?.length || 0
+      return acc
+    }, 0) || 0
+
+  $: tempFilters = cloneDeep(filters)
   $: stepId = $memoBlock.stepId
 
   $: automationBindings = getAvailableBindings(
@@ -768,14 +775,13 @@
         break
       }
     }
-    return filters || []
+    return utils.processSearchFilters(filters)
   }
 
   function saveFilters(key) {
-    const filters = QueryUtils.buildQuery(tempFilters)
-
+    const query = QueryUtils.buildQuery(tempFilters)
     onChange({
-      [key]: filters,
+      [key]: query,
       [`${key}-def`]: tempFilters, // need to store the builder definition in the automation
     })
 
@@ -1004,18 +1010,24 @@
                 </div>
               </div>
             {:else if value.customType === AutomationCustomIOType.FILTERS || value.customType === AutomationCustomIOType.TRIGGER_FILTER}
-              <ActionButton fullWidth on:click={drawer.show}
-                >{filters.length > 0
-                  ? "Update Filter"
-                  : "No Filter set"}</ActionButton
+              <ActionButton fullWidth on:click={drawer.show}>
+                {filterCount > 0 ? "Update Filter" : "No Filter set"}
+              </ActionButton>
+              <Drawer
+                bind:this={drawer}
+                title="Filtering"
+                forceModal
+                on:drawerShow={() => {
+                  tempFilters = filters
+                }}
               >
-              <Drawer bind:this={drawer} title="Filtering">
                 <Button cta slot="buttons" on:click={() => saveFilters(key)}>
                   Save
                 </Button>
+
                 <DrawerContent slot="body">
                   <FilterBuilder
-                    {filters}
+                    filters={tempFilters}
                     {bindings}
                     {schemaFields}
                     datasource={{ type: "table", tableId }}
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
index 0d4361954f1..88e2f06be90 100644
--- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
@@ -233,6 +233,14 @@
     )
     dispatch("change", result)
   }
+
+  /**
+   * Converts arrays into strings. The CodeEditor expects a string or encoded JS
+   * @param{object} fieldValue
+   */
+  const drawerValue = fieldValue => {
+    return Array.isArray(fieldValue) ? fieldValue.join(",") : fieldValue
+  }
 </script>
 
 {#each schemaFields || [] as [field, schema]}
@@ -257,7 +265,7 @@
             panel={AutomationBindingPanel}
             type={schema.type}
             {schema}
-            value={editableRow[field]}
+            value={drawerValue(editableRow[field])}
             on:change={e =>
               onChange({
                 row: {
diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index 91ec9a9b016..6c6ce8c56a2 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -5,6 +5,7 @@
   import { getUserBindings } from "dataBinding"
   import { makePropSafe } from "@budibase/string-templates"
   import { search } from "@budibase/frontend-core"
+  import { utils } from "@budibase/shared-core"
   import { tables } from "stores/builder"
 
   export let schema
@@ -16,15 +17,19 @@
 
   let drawer
 
-  $: tempValue = filters || []
+  $: localFilters = utils.processSearchFilters(filters)
+
   $: schemaFields = search.getFields(
     $tables.list,
     Object.values(schema || {}),
     { allowLinks: true }
   )
 
-  $: text = getText(filters)
-  $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
+  $: filterCount =
+    localFilters?.groups?.reduce((acc, group) => {
+      return (acc += group.filters.filter(filter => filter.field).length)
+    }, 0) || 0
+
   $: bindings = [
     {
       type: "context",
@@ -38,28 +43,32 @@
     },
     ...getUserBindings(),
   ]
-  const getText = filters => {
-    const count = filters?.filter(filter => filter.field)?.length
-    return count ? `Filter (${count})` : "Filter"
-  }
 </script>
 
-<ActionButton icon="Filter" quiet {disabled} on:click={drawer.show} {selected}>
-  {text}
+<ActionButton
+  icon="Filter"
+  quiet
+  {disabled}
+  on:click={drawer.show}
+  selected={filterCount > 0}
+>
+  {filterCount ? `Filter (${filterCount})` : "Filter"}
 </ActionButton>
 
 <Drawer
   bind:this={drawer}
   title="Filtering"
   on:drawerHide
-  on:drawerShow
+  on:drawerShow={() => {
+    localFilters = utils.processSearchFilters(filters)
+  }}
   forceModal
 >
   <Button
     cta
     slot="buttons"
     on:click={() => {
-      dispatch("change", tempValue)
+      dispatch("change", localFilters)
       drawer.hide()
     }}
   >
@@ -67,10 +76,10 @@
   </Button>
   <DrawerContent slot="body">
     <FilterBuilder
-      {filters}
+      filters={localFilters}
       {schemaFields}
       datasource={{ type: "table", tableId }}
-      on:change={e => (tempValue = e.detail)}
+      on:change={e => (localFilters = e.detail)}
       {bindings}
     />
   </DrawerContent>
diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte
index 64e93675d9d..1e79f61bae2 100644
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte
@@ -1,87 +1,32 @@
 <script>
-  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
-
-  import { dataFilters } from "@budibase/shared-core"
-  import { FilterBuilder } from "@budibase/frontend-core"
+  import { CoreFilterBuilder } from "@budibase/frontend-core"
   import { tables } from "stores/builder"
-
-  import { createEventDispatcher, onMount } from "svelte"
+  import {
+    runtimeToReadableBinding,
+    readableToRuntimeBinding,
+  } from "dataBinding"
 
   export let schemaFields
-  export let filters = []
+  export let filters
   export let bindings = []
   export let panel = ClientBindingPanel
   export let allowBindings = true
   export let datasource
   export let showFilterEmptyDropdown
-  const dispatch = createEventDispatcher()
-
-  let rawFilters
-
-  $: parseFilters(rawFilters)
-  $: dispatch("change", enrichFilters(rawFilters))
-
-  // Remove field key prefixes and determine which behaviours to use
-  const parseFilters = filters => {
-    rawFilters = (filters || []).map(filter => {
-      const { field } = filter
-      let newFilter = { ...filter }
-      delete newFilter.allOr
-      newFilter.field = dataFilters.removeKeyNumbering(field)
-      return newFilter
-    })
-  }
-
-  onMount(() => {
-    parseFilters(filters)
-    rawFilters.forEach(filter => {
-      filter.type =
-        schemaFields.find(field => field.name === filter.field)?.type ||
-        filter.type
-    })
-  })
-
-  // Add field key prefixes and a special metadata filter object to indicate
-  // how to handle filter behaviour
-  const enrichFilters = rawFilters => {
-    let count = 1
-    return rawFilters
-      .filter(filter => filter.field)
-      .map(filter => ({
-        ...filter,
-        field: `${count++}:${filter.field}`,
-      }))
-      .concat(...rawFilters.filter(filter => !filter.field))
-  }
 </script>
 
-<FilterBuilder
-  bind:filters={rawFilters}
+<CoreFilterBuilder
+  toReadable={runtimeToReadableBinding}
+  toRuntime={readableToRuntimeBinding}
   behaviourFilters={true}
   tables={$tables.list}
+  {filters}
+  {panel}
   {schemaFields}
   {datasource}
   {allowBindings}
   {showFilterEmptyDropdown}
->
-  <div slot="filtering-hero-content" />
-
-  <DrawerBindableInput
-    let:filter
-    slot="binding"
-    disabled={filter.noValue}
-    title={filter.field}
-    value={filter.value}
-    placeholder="Value"
-    {panel}
-    {bindings}
-    on:change={event => {
-      const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id)
-      rawFilters[indexToUpdate] = {
-        ...rawFilters[indexToUpdate],
-        value: event.detail,
-      }
-    }}
-  />
-</FilterBuilder>
+  {bindings}
+  on:change
+/>
diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
index ed5e36cd655..d7b0c07b26a 100644
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
@@ -5,9 +5,14 @@
     Button,
     Drawer,
     DrawerContent,
+    Helpers,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
-  import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
+  import {
+    getDatasourceForProvider,
+    getSchemaForDatasource,
+    readableToRuntimeBinding,
+  } from "dataBinding"
   import FilterBuilder from "./FilterBuilder.svelte"
   import { tables, selectedScreen } from "stores/builder"
   import { search } from "@budibase/frontend-core"
@@ -21,7 +26,7 @@
 
   let drawer
 
-  $: tempValue = value
+  $: localFilters = Helpers.cloneDeep(value)
   $: datasource = getDatasourceForProvider($selectedScreen, componentInstance)
   $: dsSchema = getSchemaForDatasource($selectedScreen, datasource)?.schema
   $: schemaFields = search.getFields(
@@ -29,19 +34,24 @@
     Object.values(schema || dsSchema || {}),
     { allowLinks: true }
   )
-  $: text = getText(value?.filter(filter => filter.field))
+
+  $: text = getText(value?.groups)
 
   async function saveFilter() {
-    dispatch("change", tempValue)
+    dispatch("change", localFilters)
     notifications.success("Filters saved")
     drawer.hide()
   }
 
-  const getText = filters => {
-    if (!filters?.length) {
+  const getText = (filterGroups = []) => {
+    const allFilters = filterGroups.reduce((acc, group) => {
+      return (acc += group.filters.filter(filter => filter.field).length)
+    }, 0)
+
+    if (allFilters === 0) {
       return "No filters set"
     } else {
-      return `${filters.length} filter${filters.length === 1 ? "" : "s"} set`
+      return `${allFilters} filter${allFilters === 1 ? "" : "s"} set`
     }
   }
 </script>
@@ -49,15 +59,25 @@
 <div class="filter-editor">
   <ActionButton on:click={drawer.show}>{text}</ActionButton>
 </div>
-<Drawer bind:this={drawer} title="Filtering" on:drawerHide on:drawerShow>
+<Drawer
+  bind:this={drawer}
+  title="Filtering"
+  on:drawerHide
+  on:drawerShow={() => {
+    // Reset to the currently available value.
+    localFilters = Helpers.cloneDeep(value)
+  }}
+>
   <Button cta slot="buttons" on:click={saveFilter}>Save</Button>
   <DrawerContent slot="body">
     <FilterBuilder
-      filters={value}
+      filters={localFilters}
       {bindings}
       {schemaFields}
       {datasource}
-      on:change={e => (tempValue = e.detail)}
+      on:change={e => {
+        localFilters = e.detail
+      }}
     />
   </DrawerContent>
 </Drawer>
diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
index 17cb171da56..eae26348fd2 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
@@ -6,6 +6,7 @@
   import FieldSetting from "./FieldSetting.svelte"
   import PrimaryColumnFieldSetting from "./PrimaryColumnFieldSetting.svelte"
   import getColumns from "./getColumns.js"
+  import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
 
   export let value
   export let componentInstance
@@ -58,16 +59,25 @@
     </div>
   </div>
 {/if}
-<DraggableList
-  on:change={e => columns.updateSortable(e.detail)}
-  on:itemChange={e => columns.update(e.detail)}
-  items={columns.sortable}
-  listItemKey={"_id"}
-  listType={FieldSetting}
-  listTypeProps={{
-    bindings,
-  }}
-/>
+
+{#if columns?.sortable?.length}
+  <DraggableList
+    on:change={e => columns.updateSortable(e.detail)}
+    on:itemChange={e => columns.update(e.detail)}
+    items={columns.sortable}
+    listItemKey={"_id"}
+    listType={FieldSetting}
+    listTypeProps={{
+      bindings,
+    }}
+  />
+{:else}
+  <InfoDisplay
+    body={datasource?.type !== "custom"
+      ? "No available columns"
+      : "No available columns for JSON/CSV data sources"}
+  />
+{/if}
 
 <style>
   .right-content {
diff --git a/packages/builder/src/components/design/settings/controls/TableConditionEditor.svelte b/packages/builder/src/components/design/settings/controls/TableConditionEditor.svelte
index d58cb1d4ba3..05770a3c5c7 100644
--- a/packages/builder/src/components/design/settings/controls/TableConditionEditor.svelte
+++ b/packages/builder/src/components/design/settings/controls/TableConditionEditor.svelte
@@ -243,11 +243,14 @@
                   {/if}
                 {:else if (type === FieldType.BB_REFERENCE || type === FieldType.BB_REFERENCE_SINGLE) && condition.valueType === type}
                   <FilterUsers
-                    bind:value={condition.referenceValue}
+                    value={condition.referenceValue}
                     multiselect={[
                       Constants.OperatorOptions.In.value,
                       Constants.OperatorOptions.ContainsAny.value,
                     ].includes(condition.operator)}
+                    on:change={e => {
+                      condition.referenceValue = e.detail
+                    }}
                     disabled={condition.noValue}
                     type={condition.valueType}
                   />
diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 7029831fdad..9cc5bf6c407 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -29,8 +29,9 @@ import {
   DB_TYPE_EXTERNAL,
 } from "constants/backend"
 import BudiStore from "../BudiStore"
-import { Utils, Constants } from "@budibase/frontend-core"
+import { Utils } from "@budibase/frontend-core"
 import { FieldType } from "@budibase/types"
+import { utils } from "@budibase/shared-core"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -204,18 +205,11 @@ export class ComponentStore extends BudiStore {
     const filterableTypes = def.settings.filter(setting =>
       setting?.type?.startsWith("filter")
     )
-
-    // Map this?
     for (let setting of filterableTypes) {
-      const updateFilter = Utils.migrateSearchFilters(
+      enrichedComponent[setting.key] = utils.processSearchFilters(
         enrichedComponent[setting.key]
       )
-      // DEAN - switch to real value when complete
-      if (updateFilter) {
-        enrichedComponent[setting.key + "_x"] = updateFilter
-      }
     }
-
     return migrated
   }
 
@@ -581,7 +575,6 @@ export class ComponentStore extends BudiStore {
       const patchResult = patchFn(component, screen)
 
       // Post processing
-      // DEAN - SKIP ON SAVE FOR THE MOMENT
       const migrated = null
 
       this.migrateSettings(component)
diff --git a/packages/builder/src/stores/portal/featureFlags.js b/packages/builder/src/stores/portal/featureFlags.js
new file mode 100644
index 00000000000..796fe76c979
--- /dev/null
+++ b/packages/builder/src/stores/portal/featureFlags.js
@@ -0,0 +1,14 @@
+import { derived } from "svelte/store"
+import { auth } from "stores/portal"
+
+export const INITIAL_FEATUREFLAG_STATE = {
+  SQS: false,
+  DEFAULT_VALUES: false,
+}
+
+export const featureFlags = derived([auth], ([$auth]) => {
+  return {
+    ...INITIAL_FEATUREFLAG_STATE,
+    ...($auth?.user?.flags || {}),
+  }
+})
diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js
index ab3be2ce79e..d4ee9f4e011 100644
--- a/packages/builder/src/stores/portal/index.js
+++ b/packages/builder/src/stores/portal/index.js
@@ -19,5 +19,6 @@ export { features } from "./features"
 export { themeStore } from "./theme"
 export { temporalStore } from "./temporal"
 export { navigation } from "./navigation"
+export { featureFlags } from "./featureFlags"
 
 export const sideBarCollapsed = writable(false)
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index b1552a974ae..e0ab01ccf3f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5101,7 +5101,8 @@
       {
         "type": "filter",
         "label": "Filtering",
-        "key": "filter"
+        "key": "filter",
+        "resetOn": "dataSource"
       },
       {
         "type": "field/sortable",
diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index 3e005b02cfe..7b90432bbf9 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -1,7 +1,8 @@
 <script>
   import { getContext } from "svelte"
-  import { Pagination, ProgressCircle } from "@budibase/bbui"
+  import { Pagination, ProgressCircle, Helpers } from "@budibase/bbui"
   import { fetchData, QueryUtils } from "@budibase/frontend-core"
+  import { LogicalOperator, EmptyFilterOption } from "@budibase/types"
 
   export let dataSource
   export let filter
@@ -16,10 +17,16 @@
 
   let interval
   let queryExtensions = {}
+  let localFilters
+
+  $: if (filter) {
+    localFilters = Helpers.cloneDeep(filter)
+  }
+
+  $: defaultQuery = QueryUtils.buildQuery(localFilters)
 
   // We need to manage our lucene query manually as we want to allow components
   // to extend it
-  $: defaultQuery = QueryUtils.buildQuery(filter)
   $: query = extendQuery(defaultQuery, queryExtensions)
   $: fetch = createFetch(dataSource)
   $: fetch.update({
@@ -124,17 +131,15 @@
   }
 
   const extendQuery = (defaultQuery, extensions) => {
-    const extensionValues = Object.values(extensions || {})
-    let extendedQuery = { ...defaultQuery }
-    extensionValues.forEach(extension => {
-      Object.entries(extension || {}).forEach(([operator, fields]) => {
-        extendedQuery[operator] = {
-          ...extendedQuery[operator],
-          ...fields,
-        }
-      })
-    })
-    return extendedQuery
+    return {
+      [LogicalOperator.AND]: {
+        conditions: [
+          ...(defaultQuery ? [defaultQuery] : []),
+          ...Object.values(extensions || {}),
+        ],
+      },
+      onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+    }
   }
 
   const setUpAutoRefresh = autoRefresh => {
diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
index 516b26ee694..a1b64b181a9 100644
--- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
+++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
@@ -1,9 +1,12 @@
 <script>
   import { get } from "svelte/store"
   import { getContext, onDestroy } from "svelte"
-  import { ModalContent, Modal } from "@budibase/bbui"
-  import FilterModal from "./FilterModal.svelte"
-  import { QueryUtils, Constants } from "@budibase/frontend-core"
+  import { ModalContent, Modal, Helpers } from "@budibase/bbui"
+  import {
+    QueryUtils,
+    Constants,
+    CoreFilterBuilder,
+  } from "@budibase/frontend-core"
   import Button from "../Button.svelte"
 
   export let dataProvider
@@ -15,14 +18,15 @@
     getContext("sdk")
 
   let modal
-  let tmpFilters = []
-  let filters = []
+  let editableFilters
+  let filters
   let schemaLoaded = false,
     schema
 
   $: dataProviderId = dataProvider?.id
   $: datasource = dataProvider?.datasource
   $: isDSPlus = ["table", "link", "viewV2"].includes(datasource?.type)
+
   $: addExtension = getAction(
     dataProviderId,
     ActionTypes.AddDataProviderQueryExtension
@@ -34,10 +38,15 @@
   $: fetchSchema(datasource)
   $: schemaFields = getSchemaFields(schema, allowedFields)
 
-  // Add query extension to data provider
+  $: filterCount = filters?.groups?.reduce((acc, group) => {
+    acc += group?.filters?.length || 0
+    return acc
+  }, 0)
+
   $: {
-    if (filters?.length) {
+    if (filterCount) {
       const queryExtension = QueryUtils.buildQuery(filters)
+      delete queryExtension.onEmptyFilter
       addExtension?.($component.id, queryExtension)
     } else {
       removeExtension?.($component.id)
@@ -80,12 +89,13 @@
     if (get(builderStore).inBuilder) {
       return
     }
-    tmpFilters = [...filters]
+    editableFilters = filters ? Helpers.cloneDeep(filters) : null
+
     modal.show()
   }
 
   const updateQuery = () => {
-    filters = [...tmpFilters]
+    filters = Helpers.cloneDeep(editableFilters)
   }
 
   onDestroy(() => {
@@ -101,12 +111,20 @@
     {size}
     type="secondary"
     quiet
-    active={filters?.length > 0}
+    active={filters?.groups?.length > 0}
   />
 
   <Modal bind:this={modal}>
     <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
-      <FilterModal bind:filters={tmpFilters} {schemaFields} {datasource} />
+      <CoreFilterBuilder
+        on:change={e => {
+          editableFilters = e.detail
+        }}
+        filters={editableFilters}
+        {schemaFields}
+        {datasource}
+        filtersLabel={null}
+      />
     </ModalContent>
   </Modal>
 {/if}
diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte
deleted file mode 100644
index 00d6b3c31a3..00000000000
--- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte
+++ /dev/null
@@ -1,14 +0,0 @@
-<script>
-  import { FilterBuilder } from "@budibase/frontend-core"
-
-  export let schemaFields
-  export let filters = []
-  export let datasource
-</script>
-
-<FilterBuilder bind:filters {schemaFields} {datasource} filtersLabel={null}>
-  <div slot="filtering-hero-content">
-    Results are filtered to only those which match all of the following
-    constraints.
-  </div>
-</FilterBuilder>
diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
new file mode 100644
index 00000000000..c0789018808
--- /dev/null
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -0,0 +1,505 @@
+<script>
+  import {
+    Body,
+    Button,
+    Icon,
+    Layout,
+    Select,
+    Helpers,
+    ActionButton,
+  } from "@budibase/bbui"
+  import {
+    FieldType,
+    FilterGroupLogicalOperator,
+    EmptyFilterOption,
+  } from "@budibase/types"
+  import { QueryUtils, Constants } from "@budibase/frontend-core"
+  import { getContext, createEventDispatcher } from "svelte"
+  import FilterField from "./FilterField.svelte"
+
+  const dispatch = createEventDispatcher()
+  const {
+    OperatorOptions,
+    DEFAULT_BB_DATASOURCE_ID,
+    FilterOperator,
+    OnEmptyFilter,
+    FilterValueType,
+  } = Constants
+
+  export let schemaFields
+  export let filters
+  export let tables = []
+  export let datasource
+  export let behaviourFilters = false
+  export let allowBindings = false
+
+  // Review
+  export let bindings
+  export let panel
+  export let toReadable
+  export let toRuntime
+
+  $: editableFilters = filters
+    ? Helpers.cloneDeep(filters)
+    : {
+        logicalOperator: FilterGroupLogicalOperator.ALL,
+        onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+        groups: [],
+      }
+
+  $: {
+    if (
+      tables.find(
+        table =>
+          table._id === datasource?.tableId &&
+          table.sourceId === DEFAULT_BB_DATASOURCE_ID
+      ) &&
+      !schemaFields.some(field => field.name === "_id")
+    ) {
+      schemaFields = [...schemaFields, { name: "_id", type: "string" }]
+    }
+  }
+
+  const filterOperatorOptions = Object.values(FilterOperator).map(entry => {
+    return { value: entry, label: Helpers.capitalise(entry) }
+  })
+
+  const onEmptyOptions = Object.values(OnEmptyFilter).map(entry => {
+    return { value: entry, label: Helpers.capitalise(entry) }
+  })
+
+  const context = getContext("context")
+
+  $: fieldOptions = (schemaFields || []).map(field => ({
+    label: field.displayName || field.name,
+    value: field.name,
+  }))
+
+  const onFieldChange = filter => {
+    const previousType = filter.type
+    sanitizeTypes(filter)
+    sanitizeOperator(filter)
+    sanitizeValue(filter, previousType)
+  }
+
+  const onOperatorChange = filter => {
+    sanitizeOperator(filter)
+    sanitizeValue(filter, filter.type)
+  }
+
+  const getSchema = filter => {
+    return schemaFields.find(field => field.name === filter.field)
+  }
+
+  const getValidOperatorsForType = filter => {
+    if (!filter?.field && !filter?.name) {
+      return []
+    }
+
+    return QueryUtils.getValidOperatorsForType(
+      filter,
+      filter.field || filter.name,
+      datasource
+    )
+  }
+
+  const sanitizeTypes = filter => {
+    // Update type based on field
+    const fieldSchema = schemaFields.find(x => x.name === filter.field)
+    filter.type = fieldSchema?.type
+    filter.subtype = fieldSchema?.subtype
+    filter.formulaType = fieldSchema?.formulaType
+    filter.constraints = fieldSchema?.constraints
+
+    // Update external type based on field
+    filter.externalType = getSchema(filter)?.externalType
+  }
+
+  const sanitizeOperator = filter => {
+    // Ensure a valid operator is selected
+    const operators = getValidOperatorsForType(filter).map(x => x.value)
+    if (!operators.includes(filter.operator)) {
+      filter.operator = operators[0] ?? OperatorOptions.Equals.value
+    }
+
+    // Update the noValue flag if the operator does not take a value
+    const noValueOptions = [
+      OperatorOptions.Empty.value,
+      OperatorOptions.NotEmpty.value,
+    ]
+    filter.noValue = noValueOptions.includes(filter.operator)
+  }
+
+  const sanitizeValue = (filter, previousType) => {
+    // Check if the operator allows a value at all
+    if (filter.noValue) {
+      filter.value = null
+      return
+    }
+    // Ensure array values are properly set and cleared
+    if (Array.isArray(filter.value)) {
+      if (filter.valueType !== "Value" || filter.type !== FieldType.ARRAY) {
+        filter.value = null
+      }
+    } else if (
+      filter.type === FieldType.ARRAY &&
+      filter.valueType === "Value"
+    ) {
+      filter.value = []
+    } else if (
+      previousType !== filter.type &&
+      (previousType === FieldType.BB_REFERENCE ||
+        filter.type === FieldType.BB_REFERENCE)
+    ) {
+      filter.value = filter.type === FieldType.ARRAY ? [] : null
+    }
+  }
+
+  const getGroupPrefix = groupIdx => {
+    if (groupIdx == 0) {
+      return "Where"
+    }
+    const operatorMapping = {
+      [FilterOperator.ANY]: "or",
+      [FilterOperator.ALL]: "and",
+    }
+    return operatorMapping[editableFilters.logicalOperator]
+  }
+
+  const onFilterFieldUpdate = (filter, groupIdx, filterIdx) => {
+    const updated = Helpers.cloneDeep(filter)
+
+    handleFilterChange({
+      groupIdx,
+      filterIdx,
+      filter: { ...updated },
+    })
+  }
+
+  const handleFilterChange = req => {
+    const {
+      groupIdx,
+      filterIdx,
+      filter,
+      group,
+      addFilter,
+      addGroup,
+      deleteGroup,
+      deleteFilter,
+      logicalOperator,
+      onEmptyFilter,
+    } = req
+
+    let editable = Helpers.cloneDeep(editableFilters)
+    let targetGroup = editable.groups?.[groupIdx]
+    let targetFilter = targetGroup?.filters?.[filterIdx]
+
+    if (targetFilter) {
+      if (deleteFilter) {
+        targetGroup.filters.splice(filterIdx, 1)
+      } else if (filter) {
+        targetGroup.filters[filterIdx] = filter
+      }
+    } else if (targetGroup) {
+      if (deleteGroup) {
+        editable.groups.splice(groupIdx, 1)
+      } else if (addFilter) {
+        targetGroup.filters.push({
+          valueType: FilterValueType.VALUE,
+        })
+      } else if (group) {
+        editable.groups[groupIdx] = {
+          ...targetGroup,
+          ...group,
+        }
+      }
+    } else if (addGroup) {
+      editable.groups.push({
+        logicalOperator: Constants.FilterOperator.ANY,
+        filters: [
+          {
+            valueType: FilterValueType.VALUE,
+          },
+        ],
+      })
+    } else if (logicalOperator) {
+      editable = {
+        ...editable,
+        logicalOperator,
+      }
+    } else if (onEmptyFilter) {
+      editable = {
+        ...editable,
+        onEmptyFilter,
+      }
+    }
+
+    dispatch("change", editable)
+  }
+</script>
+
+<div class="container" class:mobile={$context?.device?.mobile}>
+  <Layout noPadding>
+    {#if fieldOptions?.length}
+      <slot name="filtering-hero-content" />
+
+      {#if editableFilters?.groups?.length}
+        <div class="global-filter-header">
+          <span>Show data which matches</span>
+          <span class="operator-picker">
+            <Select
+              value={editableFilters?.logicalOperator}
+              options={filterOperatorOptions}
+              getOptionLabel={opt => opt.label}
+              getOptionValue={opt => opt.value}
+              on:change={e => {
+                handleFilterChange({
+                  logicalOperator: e.detail,
+                })
+              }}
+              placeholder={false}
+            />
+          </span>
+          <span>of the following filter groups:</span>
+        </div>
+      {/if}
+      {#if editableFilters?.groups?.length}
+        <div class="filter-groups">
+          {#each editableFilters?.groups as group, groupIdx}
+            <div class="group">
+              <div class="group-header">
+                <div class="group-options">
+                  <span>
+                    {getGroupPrefix(groupIdx, editableFilters.logicalOperator)}
+                  </span>
+                  <span class="operator-picker">
+                    <Select
+                      value={group?.logicalOperator}
+                      options={filterOperatorOptions}
+                      getOptionLabel={opt => opt.label}
+                      getOptionValue={opt => opt.value}
+                      on:change={e => {
+                        handleFilterChange({
+                          groupIdx,
+                          group: {
+                            logicalOperator: e.detail,
+                          },
+                        })
+                      }}
+                      placeholder={false}
+                    />
+                  </span>
+                  <span>of the following filters are met:</span>
+                </div>
+                <div class="group-actions">
+                  <Icon
+                    name="Add"
+                    hoverable
+                    hoverColor="var(--ink)"
+                    on:click={() => {
+                      handleFilterChange({
+                        groupIdx,
+                        addFilter: true,
+                      })
+                    }}
+                  />
+                  <Icon
+                    name="Delete"
+                    hoverable
+                    hoverColor="var(--ink)"
+                    on:click={() => {
+                      handleFilterChange({
+                        groupIdx,
+                        deleteGroup: true,
+                      })
+                    }}
+                  />
+                </div>
+              </div>
+
+              <div class="filters">
+                {#each group.filters as filter, filterIdx}
+                  <div class="filter">
+                    <Select
+                      value={filter.field}
+                      options={fieldOptions}
+                      on:change={e => {
+                        const updated = { ...filter, field: e.detail }
+                        onFieldChange(updated)
+                        onFilterFieldUpdate(updated, groupIdx, filterIdx)
+                      }}
+                      placeholder="Column"
+                    />
+
+                    <Select
+                      value={filter.operator}
+                      disabled={!filter.field}
+                      options={getValidOperatorsForType(filter)}
+                      on:change={e => {
+                        const updated = { ...filter, operator: e.detail }
+                        onOperatorChange(updated)
+                        onFilterFieldUpdate(updated, groupIdx, filterIdx)
+                      }}
+                      placeholder={false}
+                    />
+
+                    <FilterField
+                      placeholder="Value"
+                      {allowBindings}
+                      {filter}
+                      {schemaFields}
+                      {bindings}
+                      {panel}
+                      {toReadable}
+                      {toRuntime}
+                      on:change={e => {
+                        onFilterFieldUpdate(
+                          { ...filter, ...e.detail },
+                          groupIdx,
+                          filterIdx
+                        )
+                      }}
+                    />
+
+                    <ActionButton
+                      size="M"
+                      icon="Delete"
+                      on:click={() => {
+                        handleFilterChange({
+                          groupIdx,
+                          filterIdx,
+                          deleteFilter: true,
+                        })
+                      }}
+                    />
+                  </div>
+                {/each}
+              </div>
+            </div>
+          {/each}
+        </div>
+      {/if}
+
+      <div class="filters-footer">
+        {#if behaviourFilters && editableFilters?.groups.length}
+          <div class="empty-filter">
+            <span>Return</span>
+            <span class="empty-filter-picker">
+              <Select
+                value={editableFilters?.onEmptyFilter}
+                options={onEmptyOptions}
+                getOptionLabel={opt => opt.label}
+                getOptionValue={opt => opt.value}
+                on:change={e => {
+                  handleFilterChange({
+                    onEmptyFilter: e.detail,
+                  })
+                }}
+                placeholder={false}
+              />
+            </span>
+            <span>when all filters are empty</span>
+          </div>
+        {/if}
+        <div class="add-group">
+          <Button
+            icon="AddCircle"
+            size="M"
+            secondary
+            on:click={() => {
+              handleFilterChange({
+                addGroup: true,
+              })
+            }}
+          >
+            Add filter group
+          </Button>
+          <a
+            href="https://docs.budibase.com/docs/searchfilter-data"
+            target="_blank"
+          >
+            <Icon
+              name="HelpOutline"
+              color="var(--spectrum-global-color-gray-600)"
+            />
+          </a>
+        </div>
+      </div>
+    {:else}
+      <Body size="S">None of the table column can be used for filtering.</Body>
+    {/if}
+  </Layout>
+</div>
+
+<style>
+  .group-actions {
+    display: flex;
+    gap: var(--spacing-m);
+  }
+
+  .global-filter-header,
+  .empty-filter,
+  .group-options {
+    display: flex;
+    gap: var(--spacing-xl);
+    align-items: center;
+  }
+
+  .group-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+
+  .operator-picker {
+    width: 72px;
+  }
+
+  .empty-filter-picker {
+    width: 80px;
+  }
+
+  .filter-groups {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-xl);
+    /* overflow-x: scroll; */
+  }
+
+  .group {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-l);
+    border: 1px solid var(--spectrum-global-color-gray-400);
+    border-radius: 4px;
+    padding: var(--spacing-xl);
+  }
+
+  .filters {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-l);
+  }
+
+  .filter {
+    display: grid;
+    gap: var(--spacing-l);
+    grid-template-columns: minmax(150px, 1fr) 170px 200px 40px 40px;
+  }
+
+  .filters-footer {
+    display: flex;
+    gap: var(--spacing-xl);
+    flex-direction: column;
+  }
+
+  .add-group {
+    display: flex;
+    gap: var(--spacing-m);
+    align-items: center;
+  }
+
+  .container {
+    width: 100%;
+  }
+</style>
diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte
deleted file mode 100644
index 3a0c789b9e1..00000000000
--- a/packages/frontend-core/src/components/FilterBuilder.svelte
+++ /dev/null
@@ -1,379 +0,0 @@
-<script>
-  import {
-    Body,
-    Button,
-    Combobox,
-    DatePicker,
-    Icon,
-    Input,
-    Layout,
-    Select,
-    Label,
-    Multiselect,
-  } from "@budibase/bbui"
-  import { ArrayOperator, FieldType } from "@budibase/types"
-  import { generate } from "shortid"
-  import { QueryUtils, Constants } from "@budibase/frontend-core"
-  import { getContext } from "svelte"
-  import FilterUsers from "./FilterUsers.svelte"
-
-  const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants
-
-  export let schemaFields
-  export let filters = []
-  export let tables = []
-  export let datasource
-  export let behaviourFilters = false
-  export let allowBindings = false
-  export let filtersLabel = "Filters"
-  export let showFilterEmptyDropdown = true
-  $: {
-    if (
-      tables.find(
-        table =>
-          table._id === datasource?.tableId &&
-          table.sourceId === DEFAULT_BB_DATASOURCE_ID
-      ) &&
-      !schemaFields.some(field => field.name === "_id")
-    ) {
-      schemaFields = [...schemaFields, { name: "_id", type: "string" }]
-    }
-  }
-
-  $: matchAny = filters?.find(filter => filter.operator === "allOr") != null
-  $: onEmptyFilter =
-    filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
-
-  $: fieldFilters = filters.filter(
-    filter => filter.operator !== "allOr" && !filter.onEmptyFilter
-  )
-  const behaviourOptions = [
-    { value: "and", label: "Match all filters" },
-    { value: "or", label: "Match any filter" },
-  ]
-  const onEmptyOptions = [
-    { value: "all", label: "Return all table rows" },
-    { value: "none", label: "Return no rows" },
-  ]
-  const context = getContext("context")
-
-  $: fieldOptions = (schemaFields || []).map(field => ({
-    label: field.displayName || field.name,
-    value: field.name,
-  }))
-
-  const addFilter = () => {
-    filters = [
-      ...(filters || []),
-      {
-        id: generate(),
-        field: null,
-        operator: OperatorOptions.Equals.value,
-        value: null,
-        valueType: "Value",
-      },
-    ]
-  }
-
-  const removeFilter = id => {
-    filters = filters.filter(field => field.id !== id)
-
-    // Clear all filters when no fields are specified
-    if (filters.length === 1 && filters[0].onEmptyFilter) {
-      filters = []
-    }
-  }
-
-  const duplicateFilter = id => {
-    const existingFilter = filters.find(filter => filter.id === id)
-    const duplicate = { ...existingFilter, id: generate() }
-    filters = [...filters, duplicate]
-  }
-
-  const onFieldChange = filter => {
-    const previousType = filter.type
-    sanitizeTypes(filter)
-    sanitizeOperator(filter)
-    sanitizeValue(filter, previousType)
-  }
-
-  const onOperatorChange = filter => {
-    sanitizeOperator(filter)
-    sanitizeValue(filter, filter.type)
-  }
-
-  const onValueTypeChange = filter => {
-    sanitizeValue(filter)
-  }
-
-  const getFieldOptions = field => {
-    const schema = schemaFields.find(x => x.name === field)
-    return schema?.constraints?.inclusion || []
-  }
-
-  const getSchema = filter => {
-    return schemaFields.find(field => field.name === filter.field)
-  }
-
-  const getValidOperatorsForType = filter => {
-    if (!filter?.field && !filter?.name) {
-      return []
-    }
-
-    return QueryUtils.getValidOperatorsForType(
-      filter,
-      filter.field || filter.name,
-      datasource
-    )
-  }
-
-  $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
-
-  const sanitizeTypes = filter => {
-    // Update type based on field
-    const fieldSchema = schemaFields.find(x => x.name === filter.field)
-    filter.type = fieldSchema?.type
-    filter.subtype = fieldSchema?.subtype
-    filter.formulaType = fieldSchema?.formulaType
-    filter.constraints = fieldSchema?.constraints
-
-    // Update external type based on field
-    filter.externalType = getSchema(filter)?.externalType
-  }
-
-  const sanitizeOperator = filter => {
-    // Ensure a valid operator is selected
-    const operators = getValidOperatorsForType(filter).map(x => x.value)
-    if (!operators.includes(filter.operator)) {
-      filter.operator = operators[0] ?? OperatorOptions.Equals.value
-    }
-
-    // Update the noValue flag if the operator does not take a value
-    const noValueOptions = [
-      OperatorOptions.Empty.value,
-      OperatorOptions.NotEmpty.value,
-    ]
-    filter.noValue = noValueOptions.includes(filter.operator)
-  }
-
-  const sanitizeValue = (filter, previousType) => {
-    // Check if the operator allows a value at all
-    if (filter.noValue) {
-      filter.value = null
-      return
-    }
-
-    // Ensure array values are properly set and cleared
-    if (Array.isArray(filter.value)) {
-      if (filter.valueType !== "Value" || filter.type !== FieldType.ARRAY) {
-        filter.value = null
-      }
-    } else if (
-      filter.type === FieldType.ARRAY &&
-      filter.valueType === "Value"
-    ) {
-      filter.value = []
-    } else if (
-      previousType !== filter.type &&
-      (previousType === FieldType.BB_REFERENCE ||
-        filter.type === FieldType.BB_REFERENCE)
-    ) {
-      filter.value = filter.type === FieldType.ARRAY ? [] : null
-    }
-  }
-
-  function handleAllOr(option) {
-    filters = filters.filter(f => f.operator !== "allOr")
-    if (option === "or") {
-      filters.push({ operator: "allOr" })
-    }
-  }
-
-  function handleOnEmptyFilter(value) {
-    filters = filters?.filter(filter => !filter.onEmptyFilter)
-    filters.push({ onEmptyFilter: value })
-  }
-</script>
-
-<div class="container" class:mobile={$context?.device?.mobile}>
-  <Layout noPadding>
-    {#if fieldOptions?.length}
-      <Body size="S">
-        {#if !fieldFilters?.length}
-          Add your first filter expression.
-        {:else}
-          <slot name="filtering-hero-content" />
-          {#if behaviourFilters}
-            <div class="behaviour-filters">
-              <Select
-                label="Behaviour"
-                value={matchAny ? "or" : "and"}
-                options={behaviourOptions}
-                getOptionLabel={opt => opt.label}
-                getOptionValue={opt => opt.value}
-                on:change={e => handleAllOr(e.detail)}
-                placeholder={null}
-              />
-              {#if datasource?.type === "table" && showFilterEmptyDropdown}
-                <Select
-                  label="When filter empty"
-                  value={onEmptyFilter}
-                  options={onEmptyOptions}
-                  getOptionLabel={opt => opt.label}
-                  getOptionValue={opt => opt.value}
-                  on:change={e => handleOnEmptyFilter(e.detail)}
-                  placeholder={null}
-                />
-              {/if}
-            </div>
-          {/if}
-        {/if}
-      </Body>
-      {#if fieldFilters?.length}
-        <div>
-          {#if filtersLabel}
-            <div class="filter-label">
-              <Label>{filtersLabel}</Label>
-            </div>
-          {/if}
-          <div class="fields" class:with-bindings={allowBindings}>
-            {#each fieldFilters as filter}
-              <Select
-                bind:value={filter.field}
-                options={fieldOptions}
-                on:change={() => onFieldChange(filter)}
-                placeholder="Column"
-              />
-              <Select
-                disabled={!filter.field}
-                options={getValidOperatorsForType(filter)}
-                bind:value={filter.operator}
-                on:change={() => onOperatorChange(filter)}
-                placeholder={null}
-              />
-              {#if allowBindings}
-                <Select
-                  disabled={filter.noValue || !filter.field}
-                  options={valueTypeOptions}
-                  bind:value={filter.valueType}
-                  on:change={() => onValueTypeChange(filter)}
-                  placeholder={null}
-                />
-              {/if}
-              {#if allowBindings && filter.field && filter.valueType === "Binding"}
-                <slot name="binding" {filter} />
-              {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
-                <Input disabled={filter.noValue} bind:value={filter.value} />
-              {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)}
-                <Multiselect
-                  disabled={filter.noValue}
-                  options={getFieldOptions(filter.field)}
-                  bind:value={filter.value}
-                />
-              {:else if filter.type === FieldType.OPTIONS}
-                <Combobox
-                  disabled={filter.noValue}
-                  options={getFieldOptions(filter.field)}
-                  bind:value={filter.value}
-                />
-              {:else if filter.type === FieldType.BOOLEAN}
-                <Combobox
-                  disabled={filter.noValue}
-                  options={[
-                    { label: "True", value: "true" },
-                    { label: "False", value: "false" },
-                  ]}
-                  bind:value={filter.value}
-                />
-              {:else if filter.type === FieldType.DATETIME}
-                <DatePicker
-                  disabled={filter.noValue}
-                  enableTime={!getSchema(filter)?.dateOnly}
-                  timeOnly={getSchema(filter)?.timeOnly}
-                  bind:value={filter.value}
-                />
-              {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)}
-                <FilterUsers
-                  bind:value={filter.value}
-                  multiselect={[
-                    OperatorOptions.In.value,
-                    OperatorOptions.ContainsAny.value,
-                  ].includes(filter.operator)}
-                  disabled={filter.noValue}
-                  type={filter.valueType}
-                />
-              {:else}
-                <Input disabled />
-              {/if}
-              <div class="controls">
-                <Icon
-                  name="Duplicate"
-                  hoverable
-                  size="S"
-                  on:click={() => duplicateFilter(filter.id)}
-                />
-                <Icon
-                  name="Close"
-                  hoverable
-                  size="S"
-                  on:click={() => removeFilter(filter.id)}
-                />
-              </div>
-            {/each}
-          </div>
-        </div>
-      {/if}
-      <div>
-        <Button icon="AddCircle" size="M" secondary on:click={addFilter}>
-          Add filter
-        </Button>
-      </div>
-    {:else}
-      <Body size="S">None of the table column can be used for filtering.</Body>
-    {/if}
-  </Layout>
-</div>
-
-<style>
-  .container {
-    width: 100%;
-  }
-  .fields {
-    display: grid;
-    column-gap: var(--spacing-l);
-    row-gap: var(--spacing-s);
-    align-items: center;
-    grid-template-columns: 1fr 120px 1fr auto auto;
-  }
-  .fields.with-bindings {
-    grid-template-columns: minmax(150px, 1fr) 170px 120px minmax(150px, 1fr) 16px 16px;
-  }
-
-  .controls {
-    display: contents;
-  }
-
-  .container.mobile .fields {
-    grid-template-columns: 1fr;
-  }
-  .container.mobile .controls {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    padding: var(--spacing-s) 0;
-    gap: var(--spacing-s);
-  }
-
-  .filter-label {
-    margin-bottom: var(--spacing-s);
-  }
-
-  .behaviour-filters {
-    display: grid;
-    column-gap: var(--spacing-l);
-    row-gap: var(--spacing-s);
-    align-items: center;
-    grid-template-columns: minmax(150px, 1fr) 170px 120px minmax(150px, 1fr) 16px 16px;
-  }
-</style>
diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte
new file mode 100644
index 00000000000..03a09c9db0c
--- /dev/null
+++ b/packages/frontend-core/src/components/FilterField.svelte
@@ -0,0 +1,318 @@
+<script>
+  import {
+    Combobox,
+    DatePicker,
+    Input,
+    Multiselect,
+    Icon,
+    Drawer,
+    Button,
+  } from "@budibase/bbui"
+
+  import FilterUsers from "./FilterUsers.svelte"
+  import { FieldType, ArrayOperator } from "@budibase/types"
+  import * as Constants from "../constants"
+  import { isJSBinding, findHBSBlocks } from "@budibase/string-templates"
+  import { createEventDispatcher } from "svelte"
+
+  export let filter
+  export let disabled = false
+  export let bindings = []
+  export let allowBindings = false
+  export let schemaFields
+
+  // Export these to another field type
+  export let panel
+  export let toReadable
+  export let toRuntime
+  // Only required if you're in the builder, which we aint.
+
+  const dispatch = createEventDispatcher()
+  const { OperatorOptions, FilterValueType } = Constants
+
+  let bindingDrawer
+  let fieldValue
+
+  $: fieldValue = filter?.value
+  $: readableValue = toReadable ? toReadable(bindings, fieldValue) : fieldValue
+  $: drawerValue = toDrawerValue(fieldValue)
+  $: isJS = isJSBinding(fieldValue)
+  $: fieldIsValid = isValid(fieldValue)
+
+  const getFieldOptions = field => {
+    const schema = schemaFields.find(x => x.name === field)
+    return schema?.constraints?.inclusion || []
+  }
+
+  const getSchema = filter => {
+    return schemaFields.find(field => field.name === filter.field)
+  }
+
+  const onChange = e => {
+    fieldValue = e.detail
+    dispatch("change", {
+      value: toRuntime(bindings, fieldValue),
+    })
+  }
+
+  const onConfirm = () => {
+    dispatch("change", {
+      value: fieldValue,
+      valueType: fieldValue ? FilterValueType.BINDING : FilterValueType.VALUE,
+    })
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link =>
+      getSchema(filter)?.constraints?.inclusion?.includes(link)
+    )
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    bb_reference: hasValidLinks,
+    bb_reference_single: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    options: value => !isJSBinding(value) && !findHBSBlocks(value)?.length,
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[filter.type]
+    return validate ? validate(value) : true
+  }
+
+  /**
+   * Converts arrays into strings. The CodeEditor expects a string or encoded JS
+   * FilterValueType.VALUE values are emptied when first loaded in the binding drawer.
+   *
+   * @param{string} fieldValue
+   */
+  const toDrawerValue = fieldValue => {
+    if (filter.valueType == FilterValueType.VALUE) {
+      return ""
+    }
+    return Array.isArray(fieldValue) ? fieldValue.join(",") : readableValue
+  }
+</script>
+
+<div>
+  <Drawer
+    on:drawerHide
+    on:drawerShow
+    bind:this={bindingDrawer}
+    title={filter.field}
+    forceModal
+  >
+    <Button
+      cta
+      slot="buttons"
+      on:click={() => {
+        onConfirm()
+        bindingDrawer.hide()
+      }}
+    >
+      Confirm
+    </Button>
+
+    <svelte:component
+      this={panel}
+      slot="body"
+      value={drawerValue}
+      allowJS
+      allowHelpers
+      allowHBS
+      on:change={onChange}
+      {bindings}
+    />
+  </Drawer>
+
+  <div
+    class="field-wrap"
+    class:bindings={allowBindings}
+    class:valid={fieldIsValid}
+  >
+    <div class="field">
+      {#if filter.valueType === FilterValueType.BINDING}
+        <Input
+          disabled={filter.noValue}
+          readonly={true}
+          value={isJS ? "(JavaScript function)" : readableValue}
+          on:change={onChange}
+        />
+      {:else}
+        <div>
+          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
+            <Input
+              disabled={filter.noValue}
+              value={readableValue}
+              on:change={onChange}
+            />
+          {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)}
+            <Multiselect
+              disabled={filter.noValue}
+              options={getFieldOptions(filter.field)}
+              value={readableValue}
+              on:change={onChange}
+            />
+          {:else if filter.type === FieldType.OPTIONS}
+            <Combobox
+              disabled={filter.noValue}
+              options={getFieldOptions(filter.field)}
+              value={readableValue}
+              on:change={onChange}
+            />
+          {:else if filter.type === FieldType.BOOLEAN}
+            <Combobox
+              disabled={filter.noValue}
+              options={[
+                { label: "True", value: "true" },
+                { label: "False", value: "false" },
+              ]}
+              value={readableValue}
+              on:change={onChange}
+            />
+          {:else if filter.type === FieldType.DATETIME}
+            <DatePicker
+              disabled={filter.noValue}
+              enableTime={!getSchema(filter)?.dateOnly}
+              timeOnly={getSchema(filter)?.timeOnly}
+              value={readableValue}
+              on:change={onChange}
+            />
+          {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)}
+            <FilterUsers
+              multiselect={[
+                OperatorOptions.In.value,
+                OperatorOptions.ContainsAny.value,
+              ].includes(filter.operator)}
+              disabled={filter.noValue}
+              value={readableValue}
+              on:change={onChange}
+            />
+          {:else}
+            <Input disabled />
+          {/if}
+        </div>
+      {/if}
+    </div>
+
+    <div class="binding-control">
+      <!-- needs field, operator -->
+      {#if !disabled && allowBindings && filter.field && !filter.noValue}
+        <!-- svelte-ignore a11y-click-events-have-key-events -->
+        <!-- svelte-ignore a11y-no-static-element-interactions -->
+        <div
+          class="icon"
+          class:binding={filter.valueType === "Binding"}
+          on:click={() => {
+            bindingDrawer.show()
+          }}
+        >
+          <Icon size="S" name="FlashOn" />
+        </div>
+      {/if}
+    </div>
+  </div>
+</div>
+
+<style>
+  .field-wrap {
+    display: flex;
+  }
+  .field {
+    flex: 1;
+  }
+
+  .field-wrap.bindings .field :global(.spectrum-Form-itemField),
+  .field-wrap.bindings .field :global(input),
+  .field-wrap.bindings .field :global(.spectrum-Picker) {
+    border-top-right-radius: 0px;
+    border-bottom-right-radius: 0px;
+  }
+
+  .field-wrap.bindings
+    .field
+    :global(.spectrum-InputGroup.spectrum-Datepicker) {
+    min-width: unset;
+    border-radius: 0px;
+  }
+
+  .field-wrap.bindings
+    .field
+    :global(
+      .spectrum-InputGroup.spectrum-Datepicker
+        .spectrum-Textfield-input.spectrum-InputGroup-input
+    ) {
+    width: 100%;
+  }
+
+  .binding-control .icon {
+    border: 1px solid
+      var(
+        --spectrum-textfield-m-border-color,
+        var(--spectrum-alias-border-color)
+      );
+    border-left: 0px;
+    border-top-right-radius: 4px;
+    border-bottom-right-radius: 4px;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m));
+  }
+  .binding-control .icon.binding {
+    color: var(--yellow);
+  }
+
+  .binding-control .icon:hover {
+    cursor: pointer;
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+    color: var(--spectrum-alias-text-color-hover);
+  }
+
+  .binding-control .icon.binding:hover {
+    color: var(--yellow);
+  }
+</style>
diff --git a/packages/frontend-core/src/components/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte
index 489426df1ef..4640561afdc 100644
--- a/packages/frontend-core/src/components/FilterUsers.svelte
+++ b/packages/frontend-core/src/components/FilterUsers.svelte
@@ -27,7 +27,8 @@
 <div class="user-control">
   <svelte:component
     this={component}
-    bind:value
+    {value}
+    on:change
     autocomplete
     {options}
     getOptionLabel={option => option.email}
diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js
index a16b101bbbd..f5feee9c358 100644
--- a/packages/frontend-core/src/components/grid/stores/filter.js
+++ b/packages/frontend-core/src/components/grid/stores/filter.js
@@ -1,5 +1,5 @@
 import { writable, get, derived } from "svelte/store"
-import { FieldType } from "@budibase/types"
+import { FieldType, FilterGroupLogicalOperator } from "@budibase/types"
 
 export const createStores = context => {
   const { props } = context
@@ -16,11 +16,22 @@ export const createStores = context => {
 
 export const deriveStores = context => {
   const { filter, inlineFilters } = context
-
   const allFilters = derived(
     [filter, inlineFilters],
     ([$filter, $inlineFilters]) => {
-      return [...($filter || []), ...$inlineFilters]
+      const inlineFilterGroup = $inlineFilters?.length
+        ? {
+            logicalOperator: FilterGroupLogicalOperator.ALL,
+            filters: [...($inlineFilters || [])],
+          }
+        : null
+
+      return inlineFilterGroup
+        ? {
+            logicalOperator: FilterGroupLogicalOperator.ALL,
+            groups: [...($filter?.groups || []), inlineFilterGroup],
+          }
+        : $filter
     }
   )
 
@@ -54,7 +65,6 @@ export const createActions = context => {
       inlineFilter.operator = "contains"
     }
 
-    // Add this filter
     inlineFilters.update($inlineFilters => {
       // Remove any existing inline filter for this column
       $inlineFilters = $inlineFilters?.filter(x => x.id !== filterId)
diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js
index d494abb82de..0557ec080ec 100644
--- a/packages/frontend-core/src/components/index.js
+++ b/packages/frontend-core/src/components/index.js
@@ -7,5 +7,5 @@ export { default as UserAvatars } from "./UserAvatars.svelte"
 export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"
 export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte"
-export { default as FilterBuilder } from "./FilterBuilder.svelte"
+export { default as CoreFilterBuilder } from "./CoreFilterBuilder.svelte"
 export { default as FilterUsers } from "./FilterUsers.svelte"
diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index fc62d790e2a..a8b14a0809e 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -1,6 +1,14 @@
 import DataFetch from "./DataFetch.js"
 
 export default class CustomFetch extends DataFetch {
+  determineFeatureFlags() {
+    return {
+      supportsSearch: false,
+      supportsSort: false,
+      supportsPagination: false,
+    }
+  }
+
   // Gets the correct Budibase type for a JS value
   getType(value) {
     if (value == null) {
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index dedd06264ca..1743eef71e6 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
+import { FilterGroupLogicalOperator, EmptyFilterOption } from "@budibase/types"
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 
@@ -176,10 +177,16 @@ export default class DataFetch {
       }
     }
 
+    let defaultQuery = {
+      logicalOperator: FilterGroupLogicalOperator.ALL,
+      groups: [],
+    }
+
     // Build the query
-    let query = this.options.query
-    if (!query) {
-      query = buildQuery(filter)
+    let query = this.features.supportsSearch ? this.options.query : null
+
+    if (!query && this.features.supportsSearch) {
+      query = buildQuery(filter || defaultQuery)
     }
 
     // Update store
diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js
index cb2c045cc66..39a8d79214e 100644
--- a/packages/frontend-core/src/fetch/UserFetch.js
+++ b/packages/frontend-core/src/fetch/UserFetch.js
@@ -1,7 +1,7 @@
 import { get } from "svelte/store"
 import DataFetch from "./DataFetch.js"
 import { TableNames } from "../constants"
-import { QueryUtils } from "../utils"
+import { utils } from "@budibase/shared-core"
 
 export default class UserFetch extends DataFetch {
   constructor(opts) {
@@ -33,11 +33,11 @@ export default class UserFetch extends DataFetch {
     let finalQuery
     // convert old format to new one - we now allow use of the lucene format
     const { appId, paginated, ...rest } = query
-    if (!QueryUtils.hasFilters(query) && rest.email != null) {
-      finalQuery = { string: { email: rest.email } }
-    } else {
-      finalQuery = rest
-    }
+
+    finalQuery = utils.isSupportedUserSearch(rest)
+      ? query
+      : { string: { email: null } }
+
     try {
       const opts = {
         bookmark: cursor,
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 9d2f8c103a9..303414b2dc6 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -1,5 +1,6 @@
 import DataFetch from "./DataFetch.js"
 import { get } from "svelte/store"
+import { utils } from "@budibase/shared-core"
 
 export default class ViewV2Fetch extends DataFetch {
   determineFeatureFlags() {
@@ -53,7 +54,10 @@ export default class ViewV2Fetch extends DataFetch {
       this.options.sortColumn = definition.sort.field
       this.options.sortOrder = definition.sort.order
     }
-    if (!filter?.length && definition.query?.length) {
+
+    const parsed = utils.processSearchFilters(filter)
+
+    if (!parsed?.groups?.length && definition.query?.groups?.length) {
       this.options.filter = definition.query
     }
 
diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js
index 6c9acb0f897..7194aab07bc 100644
--- a/packages/frontend-core/src/utils/utils.js
+++ b/packages/frontend-core/src/utils/utils.js
@@ -353,98 +353,57 @@ export const buildMultiStepFormBlockDefaultProps = props => {
   }
 }
 
-/**
- * Processes the filter config. Filters are migrated
- * SearchFilter[] to SearchFilterGroup
- *
- *  This is supposed to be in shared-core
- * @param {Object[]} filter
- */
-export const migrateSearchFilters = filters => {
-  const defaultCfg = {
-    logicalOperator: Constants.FilterOperator.ALL,
-    groups: [],
-  }
-
-  const filterWhitelistKeys = [
-    "field",
-    "operator",
-    "valueType", // bb
-    "value",
-    "type",
-    "noValue", // bb
-  ]
-
-  /**
-   * Review these
-   * externalType, formulaType, subtype
-   */
-
-  if (Array.isArray(filters)) {
-    const baseGroup = {
-      filters: [],
-      logicalOperator: Constants.FilterOperator.ALL,
+export const handleFilterChange = (req, filters) => {
+  const {
+    groupIdx,
+    filterIdx,
+    filter,
+    group,
+    addFilter,
+    addGroup,
+    deleteGroup,
+    deleteFilter,
+    logicalOperator,
+    onEmptyFilter,
+  } = req
+
+  let editable = Helpers.cloneDeep(filters)
+  let targetGroup = editable.groups?.[groupIdx]
+  let targetFilter = targetGroup?.filters?.[filterIdx]
+
+  if (targetFilter) {
+    if (deleteFilter) {
+      targetGroup.filters.splice(filterIdx, 1)
+    } else if (filter) {
+      targetGroup.filters[filterIdx] = filter
     }
-
-    const migratedSetting = filters.reduce((acc, filter) => {
-      // Sort the properties for easier debugging
-      // Remove unset values
-      const filterEntries = Object.entries(filter)
-        .sort((a, b) => {
-          return a[0].localeCompare(b[0])
-        })
-        .filter(x => x[1] ?? false)
-
-      // Scrub invalid filters
-      const { operator, onEmptyFilter, field, valueType } = filter
-      if (!field || !valueType) {
-        // THIS SCRUBS THE 2 GLOBALS
-        // return acc
-      }
-
-      if (filterEntries.length == 1) {
-        console.log("### one entry ")
-        const [key, value] = filterEntries[0]
-        // Global
-        if (key === "onEmptyFilter") {
-          // unset otherwise, seems to be the default
-          acc.onEmptyFilter = value
-        } else if (key === "operator" && value === "allOr") {
-          // Group 1 logical operator
-          baseGroup.logicalOperator = Constants.FilterOperator.ANY
-        }
-
-        return acc
+  } else if (targetGroup) {
+    if (deleteGroup) {
+      editable.groups.splice(groupIdx, 1)
+    } else if (addFilter) {
+      targetGroup.filters.push({})
+    } else if (group) {
+      editable.groups[groupIdx] = {
+        ...targetGroup,
+        ...group,
       }
-
-      // Process settings??
-      const whiteListedFilterSettings = filterEntries.reduce((acc, entry) => {
-        const [key, value] = entry
-
-        if (filterWhitelistKeys.includes(key)) {
-          if (key === "field") {
-            acc.push([key, value.replace(/^[0-9]+:/, "")])
-          } else {
-            acc.push([key, value])
-          }
-        }
-        return acc
-      }, [])
-
-      const migratedFilter = Object.fromEntries(whiteListedFilterSettings)
-
-      baseGroup.filters.push(migratedFilter)
-
-      if (!acc.groups.length) {
-        // init the base group
-        acc.groups.push(baseGroup)
-      }
-
-      return acc
-    }, defaultCfg)
-
-    console.log("MIGRATED ", migratedSetting)
-    return migratedSetting
+    }
+  } else if (addGroup) {
+    editable.groups.push({
+      logicalOperator: Constants.FilterOperator.ANY,
+      filters: [{}],
+    })
+  } else if (logicalOperator) {
+    editable = {
+      ...editable,
+      logicalOperator,
+    }
+  } else if (onEmptyFilter) {
+    editable = {
+      ...editable,
+      onEmptyFilter,
+    }
   }
-  return false
+
+  return editable
 }
diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts
index 06a01646a73..220e2098f75 100644
--- a/packages/server/src/api/controllers/row/views.ts
+++ b/packages/server/src/api/controllers/row/views.ts
@@ -7,6 +7,7 @@ import {
   RowSearchParams,
   SearchFilterKey,
   LogicalOperator,
+  SearchFilter,
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -19,7 +20,7 @@ export async function searchView(
 ) {
   const { viewId } = ctx.params
 
-  const view = await sdk.views.get(viewId)
+  const view: ViewV2 = await sdk.views.get(viewId)
   if (!view) {
     ctx.throw(404, `View ${viewId} not found`)
   }
@@ -35,7 +36,7 @@ export async function searchView(
   // Enrich saved query with ephemeral query params.
   // We prevent searching on any fields that are saved as part of the query, as
   // that could let users find rows they should not be allowed to access.
-  let query = dataFilters.buildQuery(view.query || [])
+  let query: any = dataFilters.buildQuery(view.query ?? [])
   if (body.query) {
     // Delete extraneous search params that cannot be overridden
     delete body.query.onEmptyFilter
@@ -44,9 +45,15 @@ export async function searchView(
       !isExternalTableID(view.tableId) &&
       !(await features.flags.isEnabled("SQS"))
     ) {
+      // In the unlikely event that a Grouped Filter is in a non-SQS environment
+      // It needs to be ignored. Entirely
+      let queryFilters: SearchFilter[] = Array.isArray(view.query)
+        ? view.query
+        : []
+
       // Extract existing fields
       const existingFields =
-        view.query
+        queryFilters
           ?.filter(filter => filter.field)
           .map(filter => db.removeKeyNumbering(filter.field)) || []
 
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index c71dfe9dba9..6017b41d13d 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -24,6 +24,7 @@ import {
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
+import { processSearchFilters } from "./utils"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
 import { decodeNonAscii } from "./helpers/schema"
@@ -299,10 +300,6 @@ export class ColumnSplitter {
   }
 }
 
-/*
-  separate into buildQuery and build filter?
-*/
-
 /**
  * Builds a JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
@@ -323,13 +320,6 @@ const builderFilter = (expression: SearchFilter) => {
     oneOf: {},
     containsAny: {},
   }
-
-  // DEAN -
-
-  // This is the chattiest service we have, pruning the requests
-  // of bloat should have meaningful impact
-  // Further validation in this area is a must
-
   let { operator, field, type, value, externalType, onEmptyFilter } = expression
   const queryOperator = operator as SearchFilterOperator
   const isHbs =
@@ -343,6 +333,13 @@ const builderFilter = (expression: SearchFilter) => {
     query.onEmptyFilter = onEmptyFilter
     return
   }
+
+  // Default the value for noValue fields to ensure they are correctly added
+  // to the final query
+  if (queryOperator === "empty" || queryOperator === "notEmpty") {
+    value = null
+  }
+
   if (
     type === "datetime" &&
     !isHbs &&
@@ -426,131 +423,11 @@ const builderFilter = (expression: SearchFilter) => {
   return query
 }
 
-export const buildQuery = (filter: SearchFilter[]) => {
-  //
-  let query: SearchFilters = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-    oneOf: {},
-    containsAny: {},
-  }
-
-  if (!Array.isArray(filter)) {
-    return query
-  }
+export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => {
+  const parsedFilter = processSearchFilters(filter)
 
-  // DEAN Replace with builderFilter
-  filter.forEach(expression => {
-    let { operator, field, type, value, externalType, onEmptyFilter } =
-      expression
-    const queryOperator = operator as SearchFilterOperator
-    const isHbs =
-      typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
-    // Parse all values into correct types
-    if (operator === "allOr") {
-      query.allOr = true
-      return
-    }
-    if (onEmptyFilter) {
-      query.onEmptyFilter = onEmptyFilter
-      return
-    }
-    if (
-      type === "datetime" &&
-      !isHbs &&
-      queryOperator !== "empty" &&
-      queryOperator !== "notEmpty"
-    ) {
-      // Ensure date value is a valid date and parse into correct format
-      if (!value) {
-        return
-      }
-      try {
-        value = new Date(value).toISOString()
-      } catch (error) {
-        return
-      }
-    }
-    if (type === "number" && typeof value === "string" && !isHbs) {
-      if (queryOperator === "oneOf") {
-        value = value.split(",").map(item => parseFloat(item))
-      } else {
-        value = parseFloat(value)
-      }
-    }
-    if (type === "boolean") {
-      value = `${value}`?.toLowerCase() === "true"
-    }
-    if (
-      ["contains", "notContains", "containsAny"].includes(operator) &&
-      type === "array" &&
-      typeof value === "string"
-    ) {
-      value = value.split(",")
-    }
-    if (operator.startsWith("range") && query.range) {
-      const minint =
-        SqlNumberTypeRangeMap[
-          externalType as keyof typeof SqlNumberTypeRangeMap
-        ]?.min || Number.MIN_SAFE_INTEGER
-      const maxint =
-        SqlNumberTypeRangeMap[
-          externalType as keyof typeof SqlNumberTypeRangeMap
-        ]?.max || Number.MAX_SAFE_INTEGER
-      if (!query.range[field]) {
-        query.range[field] = {
-          low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-          high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-        }
-      }
-      if (operator === "rangeLow" && value != null && value !== "") {
-        query.range[field] = {
-          ...query.range[field],
-          low: value,
-        }
-      } else if (operator === "rangeHigh" && value != null && value !== "") {
-        query.range[field] = {
-          ...query.range[field],
-          high: value,
-        }
-      }
-    } else if (isLogicalSearchOperator(queryOperator)) {
-      // TODO
-    } else if (query[queryOperator] && operator !== "onEmptyFilter") {
-      if (type === "boolean") {
-        // Transform boolean filters to cope with null.
-        // "equals false" needs to be "not equals true"
-        // "not equals false" needs to be "equals true"
-        if (queryOperator === "equal" && value === false) {
-          query.notEqual = query.notEqual || {}
-          query.notEqual[field] = true
-        } else if (queryOperator === "notEqual" && value === false) {
-          query.equal = query.equal || {}
-          query.equal[field] = true
-        } else {
-          query[queryOperator] ??= {}
-          query[queryOperator]![field] = value
-        }
-      } else {
-        query[queryOperator] ??= {}
-        query[queryOperator]![field] = value
-      }
-    }
-  })
-  return query
-}
-
-// Grouped query
-export const buildQueryX = (filter: SearchFilterGroup) => {
-  if (!filter) {
-    return {}
+  if (!parsedFilter) {
+    return
   }
 
   const operatorMap = {
@@ -558,13 +435,15 @@ export const buildQueryX = (filter: SearchFilterGroup) => {
     [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
   }
 
-  const globalOnEmpty = filter.onEmptyFilter ? filter.onEmptyFilter : null
-  const globalOperator = operatorMap[filter.logicalOperator]
+  const globalOnEmpty = parsedFilter.onEmptyFilter
+    ? parsedFilter.onEmptyFilter
+    : null
+  const globalOperator = operatorMap[parsedFilter.logicalOperator]
 
   const coreRequest: SearchFilters = {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
-      conditions: filter.groups?.map((group: SearchFilterGroup) => {
+      conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
         return {
           [operatorMap[group.logicalOperator]]: {
             conditions: group.filters?.map(x => builderFilter(x)),
@@ -580,6 +459,9 @@ export const buildQueryX = (filter: SearchFilterGroup) => {
 // this we convert them to arrays at the controller level so that nothing below
 // this has to worry about the non-array values.
 export function fixupFilterArrays(filters: SearchFilters) {
+  if (!filters) {
+    return filters
+  }
   for (const searchField of Object.values(ArrayOperator)) {
     const field = filters[searchField]
     if (field == null || !isPlainObject(field)) {
@@ -1005,14 +887,13 @@ export const hasFilters = (query?: SearchFilters) => {
   if (!query) {
     return false
   }
-  const skipped = ["allOr", "onEmptyFilter"]
-  for (let [key, value] of Object.entries(query)) {
-    if (skipped.includes(key) || typeof value !== "object") {
-      continue
-    }
-    if (Object.keys(value || {}).length !== 0) {
-      return true
-    }
-  }
-  return false
+  const queryRoot = query[LogicalOperator.AND] ?? query[LogicalOperator.OR]
+
+  return (
+    (queryRoot?.conditions || []).reduce((acc, group) => {
+      const groupRoot = group[LogicalOperator.AND] ?? group[LogicalOperator.OR]
+      acc += groupRoot?.conditions?.length || 0
+      return acc
+    }, 0) > 0
+  )
 }
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index b69a0597451..0824c7e8d51 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -1,4 +1,14 @@
+import {
+  SearchFilter,
+  SearchFilterGroup,
+  FilterGroupLogicalOperator,
+  EmptyFilterOption,
+  SearchFilters,
+  BasicOperator,
+  ArrayOperator,
+} from "@budibase/types"
 import * as Constants from "./constants"
+import { removeKeyNumbering } from "./filters"
 
 export function unreachable(
   value: never,
@@ -77,3 +87,128 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
     )
   return result
 }
+
+/**
+ * Processes the filter config. Filters are migrated from
+ * SearchFilter[] to SearchFilterGroup
+ *
+ * If filters is not an array, the migration is skipped
+ *
+ * @param {SearchFilter[] | SearchFilterGroup} filters
+ */
+export const processSearchFilters = (
+  filters: SearchFilter[] | SearchFilterGroup
+) => {
+  if (!filters) {
+    return
+  }
+  const defaultCfg: SearchFilterGroup = {
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+    onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+    groups: [],
+  }
+
+  const filterWhitelistKeys = [
+    "field",
+    "operator",
+    "value",
+    "type",
+    "externalType",
+    "valueType",
+    "noValue",
+    "formulaType",
+  ]
+
+  if (Array.isArray(filters)) {
+    let baseGroup: SearchFilterGroup = {
+      filters: [],
+      logicalOperator: FilterGroupLogicalOperator.ALL,
+    }
+
+    const migratedSetting: SearchFilterGroup = filters.reduce(
+      (acc: SearchFilterGroup, filter: SearchFilter) => {
+        // Sort the properties for easier debugging
+        const filterEntries = Object.entries(filter)
+          .sort((a, b) => {
+            return a[0].localeCompare(b[0])
+          })
+          .filter(x => x[1] ?? false)
+
+        if (filterEntries.length == 1) {
+          const [key, value] = filterEntries[0]
+          // Global
+          if (key === "onEmptyFilter") {
+            // unset otherwise
+            acc.onEmptyFilter = value
+          } else if (key === "operator" && value === "allOr") {
+            // Group 1 logical operator
+            baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
+          }
+
+          return acc
+        }
+
+        const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
+          (acc: [string, any][], entry: [string, any]) => {
+            const [key, value] = entry
+
+            if (filterWhitelistKeys.includes(key)) {
+              if (key === "field") {
+                acc.push([key, removeKeyNumbering(value)])
+              } else {
+                acc.push([key, value])
+              }
+            }
+            return acc
+          },
+          []
+        )
+
+        const migratedFilter: SearchFilter = Object.fromEntries(
+          whiteListedFilterSettings
+        ) as SearchFilter
+
+        baseGroup.filters!.push(migratedFilter)
+
+        if (!acc.groups || !acc.groups.length) {
+          // init the base group
+          acc.groups = [baseGroup]
+        }
+
+        return acc
+      },
+      defaultCfg
+    )
+
+    return migratedSetting
+  } else if (!filters?.groups) {
+    return defaultCfg
+  }
+  return filters
+}
+
+export function isSupportedUserSearch(query: SearchFilters) {
+  const allowed = [
+    { op: BasicOperator.STRING, key: "email" },
+    { op: BasicOperator.EQUAL, key: "_id" },
+    { op: ArrayOperator.ONE_OF, key: "_id" },
+  ]
+  for (let [key, operation] of Object.entries(query)) {
+    if (typeof operation !== "object") {
+      return false
+    }
+    const fields = Object.keys(operation || {})
+    // this filter doesn't contain options - ignore
+    if (fields.length === 0) {
+      continue
+    }
+    const allowedOperation = allowed.find(
+      allow =>
+        allow.op === key && fields.length === 1 && fields[0] === allow.key
+    )
+    if (!allowedOperation) {
+      return false
+    }
+  }
+  return true
+}
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index 5223204a7fa..b3d577f0c87 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -1,5 +1,9 @@
 import { FieldType } from "../../documents"
-import { EmptyFilterOption, SearchFilters } from "../../sdk"
+import {
+  EmptyFilterOption,
+  FilterGroupLogicalOperator,
+  SearchFilters,
+} from "../../sdk"
 
 export type SearchFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
@@ -9,3 +13,10 @@ export type SearchFilter = {
   value: any
   externalType?: string
 }
+
+export type SearchFilterGroup = {
+  logicalOperator: FilterGroupLogicalOperator
+  onEmptyFilter?: EmptyFilterOption
+  groups?: SearchFilterGroup[]
+  filters?: SearchFilter[]
+}
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 6feea407666..4a98fe5bae0 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -180,6 +180,11 @@ export enum EmptyFilterOption {
   RETURN_NONE = "none",
 }
 
+export enum FilterGroupLogicalOperator {
+  ALL = "all",
+  ANY = "any",
+}
+
 export enum SqlClient {
   MS_SQL = "mssql",
   POSTGRES = "pg",
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 6ce0eef5a0c..921e0324d17 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -37,7 +37,7 @@ import {
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import { isEmailConfigured } from "../../../utilities/email"
-import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core"
+import { BpmStatusKey, BpmStatusValue, utils } from "@budibase/shared-core"
 
 const MAX_USERS_UPLOAD_LIMIT = 1000
 
@@ -256,7 +256,7 @@ export const search = async (ctx: Ctx<SearchUsersRequest>) => {
       }
     }
     // Validate we aren't trying to search on any illegal fields
-    if (!userSdk.core.isSupportedUserSearch(body.query)) {
+    if (!utils.isSupportedUserSearch(body.query)) {
       ctx.throw(400, "Can only search by string.email, equal._id or oneOf._id")
     }
   }

From ca5891b88bd4ff535c5c527cfef72f4b0bb3ad45 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 9 Sep 2024 16:48:32 +0100
Subject: [PATCH 088/455] Lint

---
 .../settings/controls/FilterEditor/FilterEditor.svelte      | 6 +-----
 packages/frontend-core/src/fetch/DataFetch.js               | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
index d7b0c07b26a..d6f1732e647 100644
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
@@ -8,11 +8,7 @@
     Helpers,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
-  import {
-    getDatasourceForProvider,
-    getSchemaForDatasource,
-    readableToRuntimeBinding,
-  } from "dataBinding"
+  import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
   import FilterBuilder from "./FilterBuilder.svelte"
   import { tables, selectedScreen } from "stores/builder"
   import { search } from "@budibase/frontend-core"
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index 1743eef71e6..545fc03c677 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -2,7 +2,7 @@ import { writable, derived, get } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
-import { FilterGroupLogicalOperator, EmptyFilterOption } from "@budibase/types"
+import { FilterGroupLogicalOperator } from "@budibase/types"
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 

From 4fd74c3a19d4bc2c7e457e458aff1a7df528f3ab Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Sep 2024 17:04:02 +0100
Subject: [PATCH 089/455] Add new access selection UI for tables and views

---
 .../bbui/src/InlineAlert/InlineAlert.svelte   |   1 +
 packages/bbui/src/List/ListItem.svelte        |   9 +-
 .../buttons/ManageAccessButton.svelte         | 195 +++++++++++++++++-
 .../backend/DataTable/modals/EditRoles.svelte |   5 +-
 .../DataTable/modals/ManageAccessModal.svelte | 127 ------------
 .../_components/Component/InfoDisplay.svelte  |  35 ++--
 packages/builder/src/stores/builder/roles.js  |  19 +-
 7 files changed, 239 insertions(+), 152 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte

diff --git a/packages/bbui/src/InlineAlert/InlineAlert.svelte b/packages/bbui/src/InlineAlert/InlineAlert.svelte
index 3b98936f627..edfa760eb8a 100644
--- a/packages/bbui/src/InlineAlert/InlineAlert.svelte
+++ b/packages/bbui/src/InlineAlert/InlineAlert.svelte
@@ -8,6 +8,7 @@
   export let onConfirm = undefined
   export let buttonText = ""
   export let cta = false
+
   $: icon = selectIcon(type)
   // if newlines used, convert them to different elements
   $: split = message.split("\n")
diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 2b1f6b030fc..abcdf2cc6e7 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -8,6 +8,7 @@
   export let url = null
   export let hoverable = false
   export let showArrow = false
+  export let selected = false
 </script>
 
 <a
@@ -15,6 +16,7 @@
   class="list-item"
   class:hoverable={hoverable || url != null}
   on:click
+  class:selected
 >
   <div class="left">
     {#if icon}
@@ -43,7 +45,7 @@
 
 <style>
   .list-item {
-    padding: var(--spacing-m);
+    padding: var(--spacing-m) var(--spacing-l);
     background: var(--spectrum-global-color-gray-75);
     display: flex;
     flex-direction: row;
@@ -66,8 +68,13 @@
   }
   .hoverable:hover {
     cursor: pointer;
+  }
+  .hoverable:not(.selected):hover {
     background: var(--spectrum-global-color-gray-200);
   }
+  .selected {
+    background: var(--spectrum-global-color-blue-100);
+  }
 
   .left,
   .right {
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 22a55ecc03d..bc16f563c47 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -1,31 +1,208 @@
 <script>
-  import { ActionButton } from "@budibase/bbui"
-  import { permissions } from "stores/builder"
-  import ManageAccessModal from "../modals/ManageAccessModal.svelte"
+  import {
+    ActionButton,
+    Icon,
+    Body,
+    Input,
+    Select,
+    Label,
+    List,
+    ListItem,
+    notifications,
+  } from "@budibase/bbui"
+  import { permissions as permissionsStore, roles } from "stores/builder"
   import DetailPopover from "components/common/DetailPopover.svelte"
   import EditRolesButton from "./EditRolesButton.svelte"
+  import { PermissionSource } from "@budibase/types"
+  import { capitalise } from "helpers"
+  import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
+  import { Roles } from "constants/backend"
 
   export let resourceId
 
-  let resourcePermissions
+  const inheritedRoleId = "inherited"
+  const builtins = [Roles.ADMIN, Roles.POWER, Roles.BASIC, Roles.PUBLIC]
+
+  let permissions
   let showPopover = true
+  let dependantsInfoMessage
 
   $: fetchPermissions(resourceId)
+  $: loadDependantInfo(resourceId)
+  $: roleMismatch = checkRoleMismatch(permissions)
+  $: selectedRole = roleMismatch ? null : permissions?.[0]?.value
+  $: readableRole = selectedRole
+    ? $roles.find(x => x._id === selectedRole)?.name
+    : null
+  $: buttonLabel = readableRole ? `Access: ${readableRole}` : "Access"
+  $: customRoles = $roles.filter(x => !builtins.includes(x._id))
+  $: highlight = roleMismatch || selectedRole === Roles.PUBLIC
 
   const fetchPermissions = async id => {
-    resourcePermissions = await permissions.forResourceDetailed(id)
+    const res = await permissionsStore.forResourceDetailed(id)
+    permissions = Object.entries(res?.permissions || {}).map(([perm, info]) => {
+      let enriched = {
+        permission: perm,
+        value:
+          info.permissionType === PermissionSource.INHERITED
+            ? inheritedRoleId
+            : info.role,
+        options: [...$roles],
+      }
+      if (info.inheritablePermission) {
+        enriched.options.unshift({
+          _id: inheritedRoleId,
+          name: `Inherit (${
+            $roles.find(x => x._id === info.inheritablePermission).name
+          })`,
+        })
+      }
+      return enriched
+    })
+  }
+
+  const checkRoleMismatch = permissions => {
+    if (!permissions || permissions.length < 2) {
+      return false
+    }
+    return (
+      permissions[0].value !== permissions[1].value ||
+      permissions[0].value === inheritedRoleId
+    )
+  }
+
+  const loadDependantInfo = async resourceId => {
+    const dependantsInfo = await permissionsStore.getDependantsInfo(resourceId)
+    const resourceByType = dependantsInfo?.resourceByType
+    if (resourceByType) {
+      const total = Object.values(resourceByType).reduce((p, c) => p + c, 0)
+      let resourceDisplay =
+        Object.keys(resourceByType).length === 1 && resourceByType.view
+          ? "view"
+          : "resource"
+
+      if (total === 1) {
+        dependantsInfoMessage = `1 ${resourceDisplay} is inheriting this access`
+      } else if (total > 1) {
+        dependantsInfoMessage = `${total} ${resourceDisplay}s are inheriting this access`
+      } else {
+        dependantsInfoMessage = null
+      }
+    } else {
+      dependantsInfoMessage = null
+    }
+  }
+
+  const changePermission = async role => {
+    try {
+      await permissionsStore.save({
+        level: "read",
+        role,
+        resource: resourceId,
+      })
+      await permissionsStore.save({
+        level: "write",
+        role,
+        resource: resourceId,
+      })
+      await fetchPermissions(resourceId)
+      notifications.success("Updated permissions")
+    } catch (error) {
+      console.error(error)
+      notifications.error("Error updating permissions")
+    }
   }
 </script>
 
-<DetailPopover title="Manage access" {showPopover}>
+<DetailPopover title="Select access role" {showPopover}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton icon="LockClosed" selected={open} quiet>Access</ActionButton>
+    <ActionButton
+      icon="LockClosed"
+      selected={open || highlight}
+      quiet
+      accentColor={highlight ? "#ff0000" : null}
+    >
+      {buttonLabel}
+    </ActionButton>
   </svelte:fragment>
-  {#if resourcePermissions}
-    <ManageAccessModal {resourceId} permissions={resourcePermissions} />
+
+  {#if roleMismatch}
+    <div class="row">
+      <Label extraSmall grey>Level</Label>
+      <Label extraSmall grey>Role</Label>
+      {#each permissions as permission}
+        <Input value={capitalise(permission.permission)} disabled />
+        <Select
+          placeholder={false}
+          value={permission.value}
+          on:change={e => changePermission(e.detail)}
+          disabled
+          options={permission.options}
+          getOptionLabel={x => x.name}
+          getOptionValue={x => x._id}
+        />
+      {/each}
+    </div>
+    <InfoDisplay
+      error
+      icon="Alert"
+      body="Your previous configuration is shown above.<br/> Please choose a new role for both read and write access."
+    />
+  {/if}
+
+  <List>
+    <ListItem
+      title="App admin"
+      subtitle="Only app admins can access this data"
+      hoverable
+      selected={selectedRole === Roles.ADMIN}
+      on:click={() => changePermission(Roles.ADMIN)}
+    />
+    <ListItem
+      title="App power user"
+      subtitle="Only app power users can access this data"
+      hoverable
+      selected={selectedRole === Roles.POWER}
+      on:click={() => changePermission(Roles.POWER)}
+    />
+    <ListItem
+      title="App user"
+      subtitle="Only logged in users can access this data"
+      hoverable
+      selected={selectedRole === Roles.BASIC}
+      on:click={() => changePermission(Roles.BASIC)}
+    />
+    <ListItem
+      title="Public user"
+      subtitle="Users are not required to log in to access this data"
+      hoverable
+      selected={selectedRole === Roles.PUBLIC}
+      on:click={() => changePermission(Roles.PUBLIC)}
+    />
+    {#each customRoles as role}
+      <ListItem
+        title={role.name}
+        hoverable
+        selected={selectedRole === role._id}
+        on:click={() => changePermission(role._id)}
+      />
+    {/each}
+  </List>
+
+  {#if dependantsInfoMessage}
+    <InfoDisplay body={dependantsInfoMessage} />
   {/if}
+
   <EditRolesButton
     on:show={() => (showPopover = false)}
     on:hide={() => (showPopover = true)}
   />
 </DetailPopover>
+
+<style>
+  .row {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    grid-gap: var(--spacing-s);
+  }
+</style>
diff --git a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte
index 6cec28de1dc..e46a2aa3dc7 100644
--- a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte
@@ -17,7 +17,7 @@
   let basePermissions = []
   let selectedRole = BASE_ROLE
   let errors = []
-  let builtInRoles = ["Admin", "Power", "Basic", "Public"]
+  let builtInRoles = ["App admin", "App power user", "App user", "Public user"]
   let validRegex = /^[a-zA-Z0-9_]*$/
   // Don't allow editing of public role
   $: editableRoles = $roles.filter(role => role._id !== "PUBLIC")
@@ -108,6 +108,9 @@
   }
 
   const getRoleNameError = name => {
+    if (builtInRoles.includes(name)) {
+      return null
+    }
     const hasUniqueRoleName = !otherRoles
       ?.map(role => role.name)
       ?.includes(name)
diff --git a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
deleted file mode 100644
index bc5c437c572..00000000000
--- a/packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte
+++ /dev/null
@@ -1,127 +0,0 @@
-<script>
-  import { PermissionSource } from "@budibase/types"
-  import { roles, permissions as permissionsStore } from "stores/builder"
-  import {
-    Label,
-    Input,
-    Select,
-    notifications,
-    Body,
-    Icon,
-  } from "@budibase/bbui"
-  import { capitalise } from "helpers"
-  import { get } from "svelte/store"
-
-  export let resourceId
-  export let permissions
-
-  const inheritedRoleId = "inherited"
-
-  let dependantsInfoMessage
-
-  $: loadDependantInfo(resourceId)
-  $: computedPermissions = Object.entries(permissions.permissions).reduce(
-    (p, [level, roleInfo]) => {
-      p[level] = {
-        selectedValue:
-          roleInfo.permissionType === PermissionSource.INHERITED
-            ? inheritedRoleId
-            : roleInfo.role,
-        options: [...$roles],
-      }
-      if (roleInfo.inheritablePermission) {
-        p[level].inheritOption = roleInfo.inheritablePermission
-        p[level].options.unshift({
-          _id: inheritedRoleId,
-          name: `Inherit (${
-            get(roles).find(x => x._id === roleInfo.inheritablePermission).name
-          })`,
-        })
-      }
-      return p
-    },
-    {}
-  )
-
-  async function changePermission(level, role) {
-    try {
-      if (role === inheritedRoleId) {
-        await permissionsStore.remove({
-          level,
-          role,
-          resource: resourceId,
-        })
-      } else {
-        await permissionsStore.save({
-          level,
-          role,
-          resource: resourceId,
-        })
-      }
-
-      // Show updated permissions in UI: REMOVE
-      permissions = await permissionsStore.forResourceDetailed(resourceId)
-      notifications.success("Updated permissions")
-    } catch (error) {
-      notifications.error("Error updating permissions")
-    }
-  }
-
-  async function loadDependantInfo(resourceId) {
-    const dependantsInfo = await permissionsStore.getDependantsInfo(resourceId)
-    const resourceByType = dependantsInfo?.resourceByType
-    if (resourceByType) {
-      const total = Object.values(resourceByType).reduce((p, c) => p + c, 0)
-      let resourceDisplay =
-        Object.keys(resourceByType).length === 1 && resourceByType.view
-          ? "view"
-          : "resource"
-
-      if (total === 1) {
-        dependantsInfoMessage = `1 ${resourceDisplay} is inheriting this access.`
-      } else if (total > 1) {
-        dependantsInfoMessage = `${total} ${resourceDisplay}s are inheriting this access.`
-      }
-    }
-  }
-</script>
-
-<Body size="S">Specify the minimum access level role for this data.</Body>
-<div class="row">
-  <Label extraSmall grey>Level</Label>
-  <Label extraSmall grey>Role</Label>
-  {#each Object.keys(computedPermissions) as level}
-    <Input value={capitalise(level)} disabled />
-    <Select
-      placeholder={false}
-      value={computedPermissions[level].selectedValue}
-      on:change={e => changePermission(level, e.detail)}
-      options={computedPermissions[level].options}
-      getOptionLabel={x => x.name}
-      getOptionValue={x => x._id}
-    />
-  {/each}
-</div>
-
-{#if dependantsInfoMessage}
-  <div class="inheriting-resources">
-    <Icon name="Alert" />
-    <Body size="S">
-      <i>
-        {dependantsInfoMessage}
-      </i>
-    </Body>
-  </div>
-{/if}
-
-<style>
-  .row {
-    display: grid;
-    grid-template-columns: 1fr 1fr;
-    grid-gap: var(--spacing-s);
-  }
-  .inheriting-resources {
-    display: flex;
-    gap: var(--spacing-s);
-  }
-</style>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
index 03bf771beb8..180ea5bd834 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
@@ -4,9 +4,11 @@
   export let title
   export let body
   export let icon = "HelpOutline"
+  export let warning = false
+  export let error = false
 </script>
 
-<div class="info" class:noTitle={!title}>
+<div class="info" class:noTitle={!title} class:warning class:error>
   {#if title}
     <div class="title">
       <Icon name={icon} />
@@ -16,7 +18,7 @@
     {@html body}
   {:else}
     <span class="icon">
-      <Icon name={icon} />
+      <Icon size="S" name={icon} />
     </span>
     <!-- eslint-disable-next-line svelte/no-at-html-tags -->
     {@html body}
@@ -24,6 +26,23 @@
 </div>
 
 <style>
+  .info {
+    padding: var(--spacing-m) var(--spacing-l);
+    background-color: var(--spectrum-global-color-gray-200);
+    border-radius: var(--border-radius-s);
+    font-size: 13px;
+  }
+  .warning {
+    background: rgba(255, 200, 0, 0.2);
+  }
+  .error {
+    background: rgba(255, 0, 0, 0.2);
+  }
+  .noTitle {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
   .title {
     font-size: 12px;
     font-weight: 600;
@@ -39,17 +58,7 @@
   .icon {
     color: var(--spectrum-global-color-gray-600);
   }
-  .info {
-    padding: var(--spacing-m) var(--spacing-l) var(--spacing-l) var(--spacing-l);
-    background-color: var(--background-alt);
-    border-radius: var(--border-radius-s);
-    font-size: 13px;
-  }
-  .noTitle {
-    display: flex;
-    align-items: center;
-    gap: var(--spacing-m);
-  }
+
   .info :global(a) {
     color: inherit;
     transition: color 130ms ease-out;
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index ac395aa232d..8f1bfdc177a 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -1,6 +1,14 @@
 import { writable } from "svelte/store"
 import { API } from "api"
 import { RoleUtils } from "@budibase/frontend-core"
+import { Roles } from "constants/backend"
+
+const ROLE_NAMES = {
+  [Roles.ADMIN]: "App admin",
+  [Roles.POWER]: "App power user",
+  [Roles.BASIC]: "App user",
+  [Roles.PUBLIC]: "Public user",
+}
 
 export function createRolesStore() {
   const { subscribe, update, set } = writable([])
@@ -17,7 +25,16 @@ export function createRolesStore() {
 
   const actions = {
     fetch: async () => {
-      const roles = await API.getRoles()
+      let roles = await API.getRoles()
+
+      // Update labels
+      for (let [roleId, name] of Object.entries(ROLE_NAMES)) {
+        const idx = roles.findIndex(x => x._id === roleId)
+        if (idx !== -1) {
+          roles[idx].name = name
+        }
+      }
+
       setRoles(roles)
     },
     fetchByAppId: async appId => {

From 608bc97cdd1a33200fd0e6b4c142ae054a8d2dd5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Sep 2024 08:46:26 +0100
Subject: [PATCH 090/455] Update access button

---
 .../backend/DataTable/buttons/ManageAccessButton.svelte       | 4 ++--
 .../[application]/data/table/[tableId]/[viewId]/index.svelte  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index bc16f563c47..1b9c4826f8b 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -146,7 +146,7 @@
     <InfoDisplay
       error
       icon="Alert"
-      body="Your previous configuration is shown above.<br/> Please choose a new role for both read and write access."
+      body="Your previous configuration is shown above.<br/> Please choose a single role for read and write access."
     />
   {/if}
 
@@ -190,7 +190,7 @@
   </List>
 
   {#if dependantsInfoMessage}
-    <InfoDisplay body={dependantsInfoMessage} />
+    <InfoDisplay info body={dependantsInfoMessage} />
   {/if}
 
   <EditRolesButton
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 8b1ad6cda17..82ffbd9ba5f 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -50,11 +50,11 @@
   buttonsCollapsed
 >
   <svelte:fragment slot="controls">
+    <GridManageAccessButton />
     <GridFilterButton />
     <GridSortButton />
     <GridSizeButton />
     <GridColumnsSettingButton />
-    <GridManageAccessButton />
     <GridRowActionsButton />
     <GridScreensButton on:request-generate={() => generateButton?.show()} />
   </svelte:fragment>

From eb20686de8b8674f0ea1672acf9c765471d4283c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Sep 2024 08:48:24 +0100
Subject: [PATCH 091/455] Remove licensing restriction for configuring access
 on views

---
 .../server/src/api/controllers/permission.ts  |  15 --
 .../src/api/routes/tests/permissions.spec.ts  |  52 ------
 .../server/src/sdk/app/permissions/index.ts   | 151 ------------------
 .../app/permissions/tests/permissions.spec.ts |  53 ------
 packages/types/src/api/web/app/permission.ts  |   1 -
 5 files changed, 272 deletions(-)
 delete mode 100644 packages/server/src/sdk/app/permissions/index.ts
 delete mode 100644 packages/server/src/sdk/app/permissions/tests/permissions.spec.ts

diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index cdfa6d8b1c3..14a450a4de7 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -45,18 +45,6 @@ async function updatePermissionOnRole(
   }: { roleId: string; resourceId: string; level: PermissionLevel },
   updateType: PermissionUpdateType
 ) {
-  const allowedAction = await sdk.permissions.resourceActionAllowed({
-    resourceId,
-    level,
-  })
-
-  if (!allowedAction.allowed) {
-    throw new HTTPError(
-      `You are not allowed to '${allowedAction.level}' the resource type '${allowedAction.resourceType}'`,
-      403
-    )
-  }
-
   const db = context.getAppDB()
   const remove = updateType === PermissionUpdateType.REMOVE
   const isABuiltin = roles.isBuiltin(roleId)
@@ -182,9 +170,6 @@ export async function getResourcePerms(
       },
       {} as Record<string, ResourcePermissionInfo>
     ),
-    requiresPlanToModify: (
-      await sdk.permissions.allowsExplicitPermissions(resourceId)
-    ).minPlan,
   }
 }
 
diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 838e1aca0b9..0fba930144c 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -1,8 +1,4 @@
 const mockedSdk = sdk.permissions as jest.Mocked<typeof sdk.permissions>
-jest.mock("../../../sdk/app/permissions", () => ({
-  ...jest.requireActual("../../../sdk/app/permissions"),
-  resourceActionAllowed: jest.fn(),
-}))
 
 import sdk from "../../../sdk"
 
@@ -40,8 +36,6 @@ describe("/permission", () => {
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-    mockedSdk.resourceActionAllowed.mockResolvedValue({ allowed: true })
-
     table = (await config.createTable()) as typeof table
     row = await config.createRow()
     view = await config.api.viewV2.create({
@@ -112,29 +106,6 @@ describe("/permission", () => {
       expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID)
       expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID)
     })
-
-    it("throw forbidden if the action is not allowed for the resource", async () => {
-      mockedSdk.resourceActionAllowed.mockResolvedValue({
-        allowed: false,
-        resourceType: DocumentType.DATASOURCE,
-        level: PermissionLevel.READ,
-      })
-
-      await config.api.permission.add(
-        {
-          roleId: STD_ROLE_ID,
-          resourceId: table._id,
-          level: PermissionLevel.EXECUTE,
-        },
-        {
-          status: 403,
-          body: {
-            message:
-              "You are not allowed to 'read' the resource type 'datasource'",
-          },
-        }
-      )
-    })
   })
 
   describe("remove", () => {
@@ -148,29 +119,6 @@ describe("/permission", () => {
       const permsRes = await config.api.permission.get(table._id)
       expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
     })
-
-    it("throw forbidden if the action is not allowed for the resource", async () => {
-      mockedSdk.resourceActionAllowed.mockResolvedValue({
-        allowed: false,
-        resourceType: DocumentType.DATASOURCE,
-        level: PermissionLevel.READ,
-      })
-
-      await config.api.permission.revoke(
-        {
-          roleId: STD_ROLE_ID,
-          resourceId: table._id,
-          level: PermissionLevel.EXECUTE,
-        },
-        {
-          status: 403,
-          body: {
-            message:
-              "You are not allowed to 'read' the resource type 'datasource'",
-          },
-        }
-      )
-    })
   })
 
   describe("check public user allowed", () => {
diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts
deleted file mode 100644
index 18a376aaf0d..00000000000
--- a/packages/server/src/sdk/app/permissions/index.ts
+++ /dev/null
@@ -1,151 +0,0 @@
-import { db, roles } from "@budibase/backend-core"
-import { features } from "@budibase/pro"
-import {
-  DocumentType,
-  PermissionLevel,
-  PermissionSource,
-  PlanType,
-  VirtualDocumentType,
-} from "@budibase/types"
-import { extractViewInfoFromID, isViewID } from "../../../db/utils"
-import {
-  CURRENTLY_SUPPORTED_LEVELS,
-  getBasePermissions,
-} from "../../../utilities/security"
-import sdk from "../../../sdk"
-import { isV2 } from "../views"
-
-type ResourceActionAllowedResult =
-  | { allowed: true }
-  | {
-      allowed: false
-      level: PermissionLevel
-      resourceType: DocumentType | VirtualDocumentType
-    }
-
-export async function resourceActionAllowed({
-  resourceId,
-  level,
-}: {
-  resourceId: string
-  level: PermissionLevel
-}): Promise<ResourceActionAllowedResult> {
-  if (!isViewID(resourceId)) {
-    return { allowed: true }
-  }
-
-  if (await features.isViewPermissionEnabled()) {
-    return { allowed: true }
-  }
-
-  return {
-    allowed: false,
-    level,
-    resourceType: VirtualDocumentType.VIEW,
-  }
-}
-
-type ResourcePermissions = Record<
-  string,
-  { role: string; type: PermissionSource }
->
-
-export async function getInheritablePermissions(
-  resourceId: string
-): Promise<ResourcePermissions | undefined> {
-  if (isViewID(resourceId)) {
-    return await getResourcePerms(extractViewInfoFromID(resourceId).tableId)
-  }
-}
-
-export async function allowsExplicitPermissions(resourceId: string) {
-  if (isViewID(resourceId)) {
-    const allowed = await features.isViewPermissionEnabled()
-    const minPlan = !allowed ? PlanType.PREMIUM_PLUS : undefined
-
-    return {
-      allowed,
-      minPlan,
-    }
-  }
-
-  return { allowed: true }
-}
-
-export async function getResourcePerms(
-  resourceId: string
-): Promise<ResourcePermissions> {
-  const rolesList = await roles.getAllRoles()
-
-  let permissions: ResourcePermissions = {}
-
-  const permsToInherit = await getInheritablePermissions(resourceId)
-
-  const allowsExplicitPerm = (await allowsExplicitPermissions(resourceId))
-    .allowed
-
-  for (let level of CURRENTLY_SUPPORTED_LEVELS) {
-    // update the various roleIds in the resource permissions
-    for (let role of rolesList) {
-      const rolePerms = allowsExplicitPerm
-        ? roles.checkForRoleResourceArray(role.permissions || {}, resourceId)
-        : {}
-      if (rolePerms[resourceId]?.indexOf(level) > -1) {
-        permissions[level] = {
-          role: roles.getExternalRoleID(role._id!, role.version),
-          type: PermissionSource.EXPLICIT,
-        }
-      } else if (
-        !permissions[level] &&
-        permsToInherit &&
-        permsToInherit[level]
-      ) {
-        permissions[level] = {
-          role: permsToInherit[level].role,
-          type: PermissionSource.INHERITED,
-        }
-      }
-    }
-  }
-
-  const basePermissions = Object.entries(
-    getBasePermissions(resourceId)
-  ).reduce<ResourcePermissions>((p, [level, role]) => {
-    p[level] = { role, type: PermissionSource.BASE }
-    return p
-  }, {})
-  const result = Object.assign(basePermissions, permissions)
-  return result
-}
-
-export async function getDependantResources(
-  resourceId: string
-): Promise<Record<string, number> | undefined> {
-  if (db.isTableId(resourceId)) {
-    const dependants: Record<string, Set<string>> = {}
-
-    const table = await sdk.tables.getTable(resourceId)
-    const views = Object.values(table.views || {})
-
-    for (const view of views) {
-      if (!isV2(view)) {
-        continue
-      }
-
-      const permissions = await getResourcePerms(view.id)
-      for (const [, roleInfo] of Object.entries(permissions)) {
-        if (roleInfo.type === PermissionSource.INHERITED) {
-          dependants[VirtualDocumentType.VIEW] ??= new Set()
-          dependants[VirtualDocumentType.VIEW].add(view.id)
-        }
-      }
-    }
-
-    return Object.entries(dependants).reduce((p, [type, resources]) => {
-      p[type] = resources.size
-      return p
-    }, {} as Record<string, number>)
-  }
-
-  return
-}
diff --git a/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts b/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts
deleted file mode 100644
index 4c233e68fa6..00000000000
--- a/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { PermissionLevel } from "@budibase/types"
-import { mocks, structures } from "@budibase/backend-core/tests"
-import { resourceActionAllowed } from ".."
-import { generateViewID } from "../../../../db/utils"
-import { initProMocks } from "../../../../tests/utilities/mocks/pro"
-
-initProMocks()
-
-describe("permissions sdk", () => {
-  beforeEach(() => {
-    mocks.licenses.useCloudFree()
-  })
-
-  describe("resourceActionAllowed", () => {
-    it("non view resources actions are always allowed", async () => {
-      const resourceId = structures.users.user()._id!
-
-      const result = await resourceActionAllowed({
-        resourceId,
-        level: PermissionLevel.READ,
-      })
-
-      expect(result).toEqual({ allowed: true })
-    })
-
-    it("view resources actions allowed if the feature flag is enabled", async () => {
-      mocks.licenses.useViewPermissions()
-      const resourceId = generateViewID(structures.generator.guid())
-
-      const result = await resourceActionAllowed({
-        resourceId,
-        level: PermissionLevel.READ,
-      })
-
-      expect(result).toEqual({ allowed: true })
-    })
-
-    it("view resources actions allowed if the feature flag is disabled", async () => {
-      const resourceId = generateViewID(structures.generator.guid())
-
-      const result = await resourceActionAllowed({
-        resourceId,
-        level: PermissionLevel.READ,
-      })
-
-      expect(result).toEqual({
-        allowed: false,
-        level: "read",
-        resourceType: "view",
-      })
-    })
-  })
-})
diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts
index 88ff4e9d2fb..719be4f78eb 100644
--- a/packages/types/src/api/web/app/permission.ts
+++ b/packages/types/src/api/web/app/permission.ts
@@ -8,7 +8,6 @@ export interface ResourcePermissionInfo {
 
 export interface GetResourcePermsResponse {
   permissions: Record<string, ResourcePermissionInfo>
-  requiresPlanToModify?: PlanType
 }
 
 export interface GetDependantResourcesResponse {

From 162d0d0c7ee9ea4965659e27ba8aab94d74f63a7 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 10 Sep 2024 09:08:43 +0100
Subject: [PATCH 092/455] Clean up dead code and improve behaviour when
 emptying filter config

---
 .../src/components/CoreFilterBuilder.svelte   | 27 +++++----
 packages/frontend-core/src/fetch/DataFetch.js |  2 +-
 packages/frontend-core/src/utils/utils.js     | 56 -------------------
 packages/shared-core/src/filters.ts           |  9 ++-
 packages/shared-core/src/utils.ts             |  2 +-
 5 files changed, 27 insertions(+), 69 deletions(-)

diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index c0789018808..839b370eca7 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -39,13 +39,13 @@
   export let toReadable
   export let toRuntime
 
-  $: editableFilters = filters
-    ? Helpers.cloneDeep(filters)
-    : {
-        logicalOperator: FilterGroupLogicalOperator.ALL,
-        onEmptyFilter: EmptyFilterOption.RETURN_NONE,
-        groups: [],
-      }
+  const defaultConf = {
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+    onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+    groups: [],
+  }
+
+  $: editableFilters = filters ? Helpers.cloneDeep(filters) : null
 
   $: {
     if (
@@ -191,7 +191,7 @@
     } = req
 
     let editable = Helpers.cloneDeep(editableFilters)
-    let targetGroup = editable.groups?.[groupIdx]
+    let targetGroup = editable?.groups?.[groupIdx]
     let targetFilter = targetGroup?.filters?.[filterIdx]
 
     if (targetFilter) {
@@ -202,7 +202,11 @@
       }
     } else if (targetGroup) {
       if (deleteGroup) {
-        editable.groups.splice(groupIdx, 1)
+        if (editable.groups.length > 1) {
+          editable.groups.splice(groupIdx, 1)
+        } else {
+          editable = {}
+        }
       } else if (addFilter) {
         targetGroup.filters.push({
           valueType: FilterValueType.VALUE,
@@ -214,6 +218,9 @@
         }
       }
     } else if (addGroup) {
+      if (!editable) {
+        editable = defaultConf
+      }
       editable.groups.push({
         logicalOperator: Constants.FilterOperator.ANY,
         filters: [
@@ -381,7 +388,7 @@
       {/if}
 
       <div class="filters-footer">
-        {#if behaviourFilters && editableFilters?.groups.length}
+        {#if behaviourFilters && editableFilters?.groups?.length}
           <div class="empty-filter">
             <span>Return</span>
             <span class="empty-filter-picker">
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index 545fc03c677..18dc72bf24d 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -183,7 +183,7 @@ export default class DataFetch {
     }
 
     // Build the query
-    let query = this.features.supportsSearch ? this.options.query : null
+    let query = this.options.query || null
 
     if (!query && this.features.supportsSearch) {
       query = buildQuery(filter || defaultQuery)
diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js
index 7194aab07bc..3e8cdcc781d 100644
--- a/packages/frontend-core/src/utils/utils.js
+++ b/packages/frontend-core/src/utils/utils.js
@@ -1,6 +1,5 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { Helpers } from "@budibase/bbui"
-import * as Constants from "../constants"
 
 export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
@@ -352,58 +351,3 @@ export const buildMultiStepFormBlockDefaultProps = props => {
     title,
   }
 }
-
-export const handleFilterChange = (req, filters) => {
-  const {
-    groupIdx,
-    filterIdx,
-    filter,
-    group,
-    addFilter,
-    addGroup,
-    deleteGroup,
-    deleteFilter,
-    logicalOperator,
-    onEmptyFilter,
-  } = req
-
-  let editable = Helpers.cloneDeep(filters)
-  let targetGroup = editable.groups?.[groupIdx]
-  let targetFilter = targetGroup?.filters?.[filterIdx]
-
-  if (targetFilter) {
-    if (deleteFilter) {
-      targetGroup.filters.splice(filterIdx, 1)
-    } else if (filter) {
-      targetGroup.filters[filterIdx] = filter
-    }
-  } else if (targetGroup) {
-    if (deleteGroup) {
-      editable.groups.splice(groupIdx, 1)
-    } else if (addFilter) {
-      targetGroup.filters.push({})
-    } else if (group) {
-      editable.groups[groupIdx] = {
-        ...targetGroup,
-        ...group,
-      }
-    }
-  } else if (addGroup) {
-    editable.groups.push({
-      logicalOperator: Constants.FilterOperator.ANY,
-      filters: [{}],
-    })
-  } else if (logicalOperator) {
-    editable = {
-      ...editable,
-      logicalOperator,
-    }
-  } else if (onEmptyFilter) {
-    editable = {
-      ...editable,
-      onEmptyFilter,
-    }
-  }
-
-  return editable
-}
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index ded45746a9d..4bb3a6a7df6 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -321,6 +321,11 @@ const builderFilter = (expression: SearchFilter) => {
     containsAny: {},
   }
   let { operator, field, type, value, externalType, onEmptyFilter } = expression
+
+  if (!operator || !field) {
+    return
+  }
+
   const queryOperator = operator as SearchFilterOperator
   const isHbs =
     typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
@@ -446,7 +451,9 @@ export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => {
       conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
         return {
           [operatorMap[group.logicalOperator]]: {
-            conditions: group.filters?.map(x => builderFilter(x)),
+            conditions: group.filters
+              ?.map(x => builderFilter(x))
+              .filter(filter => filter),
           },
         }
       }),
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 0824c7e8d51..027db8e914e 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -182,7 +182,7 @@ export const processSearchFilters = (
 
     return migratedSetting
   } else if (!filters?.groups) {
-    return defaultCfg
+    return null
   }
   return filters
 }

From 9da84b19f985dc3cefb7877d212f8feb83b9f5a8 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Sep 2024 09:11:52 +0100
Subject: [PATCH 093/455] Add scaffolding for new role editor modal

---
 .../DataTable/buttons/EditRolesButton.svelte  |  8 ++---
 .../buttons/ManageAccessButton.svelte         | 17 ++++++----
 .../DataTable/buttons/RoleEditor.svelte       | 32 +++++++++++++++++++
 3 files changed, 46 insertions(+), 11 deletions(-)
 create mode 100644 packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
index 3b31de170c2..bf9606ff973 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
@@ -5,11 +5,9 @@
   let modal
 </script>
 
-<div>
-  <Button secondary icon="UsersLock" on:click on:click={modal.show}>
-    Edit roles
-  </Button>
-</div>
+<Button secondary icon="UsersLock" on:click on:click={modal.show}>
+  Edit roles
+</Button>
 <Modal bind:this={modal} on:show on:hide>
   <EditRolesModal />
 </Modal>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 1b9c4826f8b..7316fc79a98 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -1,8 +1,6 @@
 <script>
   import {
     ActionButton,
-    Icon,
-    Body,
     Input,
     Select,
     Label,
@@ -17,6 +15,7 @@
   import { capitalise } from "helpers"
   import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
   import { Roles } from "constants/backend"
+  import RoleEditor from "./RoleEditor.svelte"
 
   export let resourceId
 
@@ -193,10 +192,16 @@
     <InfoDisplay info body={dependantsInfoMessage} />
   {/if}
 
-  <EditRolesButton
-    on:show={() => (showPopover = false)}
-    on:hide={() => (showPopover = true)}
-  />
+  <div class="buttons">
+    <EditRolesButton
+      on:show={() => (showPopover = false)}
+      on:hide={() => (showPopover = true)}
+    />
+    <RoleEditor
+      on:drawerShow={() => (showPopover = false)}
+      on:drawerHide={() => (showPopover = true)}
+    />
+  </div>
 </DetailPopover>
 
 <style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte b/packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte
new file mode 100644
index 00000000000..6b7e79eabc0
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte
@@ -0,0 +1,32 @@
+<script>
+  import {
+    Button,
+    Modal,
+    ModalContent,
+    Drawer,
+    DrawerContent,
+  } from "@budibase/bbui"
+
+  let drawer
+
+  $: invalid = false
+
+  const save = async () => {
+    drawer.hide()
+  }
+</script>
+
+<Button secondary icon="UsersLock" on:click on:click={drawer.show}>
+  Role editor
+</Button>
+
+<Drawer
+  bind:this={drawer}
+  title="Role editor"
+  on:drawerHide
+  on:drawerShow
+  forceModal
+>
+  <Button disabled={invalid} cta slot="buttons" on:click={save}>Save</Button>
+  <DrawerContent slot="body">asdasdasd</DrawerContent>
+</Drawer>

From e389bd8121bdb147c38901e1b5f9b969cfab960e Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 10 Sep 2024 09:51:34 +0100
Subject: [PATCH 094/455] Fix for resetting the default config when the groups
 are cleared

---
 .../src/components/CoreFilterBuilder.svelte        | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index 839b370eca7..405373e0a7a 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -39,12 +39,6 @@
   export let toReadable
   export let toRuntime
 
-  const defaultConf = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
-    onEmptyFilter: EmptyFilterOption.RETURN_NONE,
-    groups: [],
-  }
-
   $: editableFilters = filters ? Helpers.cloneDeep(filters) : null
 
   $: {
@@ -218,8 +212,12 @@
         }
       }
     } else if (addGroup) {
-      if (!editable) {
-        editable = defaultConf
+      if (!editable?.groups?.length) {
+        editable = {
+          logicalOperator: FilterGroupLogicalOperator.ALL,
+          onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+          groups: [],
+        }
       }
       editable.groups.push({
         logicalOperator: Constants.FilterOperator.ANY,

From 5cc8433773a2512f57ad76a6854fae9564042477 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 10 Sep 2024 11:05:49 +0100
Subject: [PATCH 095/455] Builder test fixes

---
 packages/builder/src/stores/builder/components.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 9cc5bf6c407..0250ed6556e 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -202,10 +202,10 @@ export class ComponentStore extends BudiStore {
     }
 
     const def = this.getDefinition(enrichedComponent?._component)
-    const filterableTypes = def.settings.filter(setting =>
+    const filterableTypes = def?.settings?.filter(setting =>
       setting?.type?.startsWith("filter")
     )
-    for (let setting of filterableTypes) {
+    for (let setting of filterableTypes || []) {
       enrichedComponent[setting.key] = utils.processSearchFilters(
         enrichedComponent[setting.key]
       )

From f9e8689ce9cddb14fca5ad18ef142a9f5970ca0e Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 10 Sep 2024 12:19:41 +0100
Subject: [PATCH 096/455] Test fixes for view search

---
 packages/server/src/api/controllers/row/views.ts | 2 +-
 packages/shared-core/src/filters.ts              | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts
index 220e2098f75..8a1e5e08ced 100644
--- a/packages/server/src/api/controllers/row/views.ts
+++ b/packages/server/src/api/controllers/row/views.ts
@@ -36,7 +36,7 @@ export async function searchView(
   // Enrich saved query with ephemeral query params.
   // We prevent searching on any fields that are saved as part of the query, as
   // that could let users find rows they should not be allowed to access.
-  let query: any = dataFilters.buildQuery(view.query ?? [])
+  let query: any = dataFilters.buildQuery(view.query)
   if (body.query) {
     // Delete extraneous search params that cannot be overridden
     delete body.query.onEmptyFilter
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 4bb3a6a7df6..79c95380e2d 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -428,7 +428,11 @@ const builderFilter = (expression: SearchFilter) => {
   return query
 }
 
-export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => {
+export const buildQuery = (filter?: SearchFilterGroup | SearchFilter[]) => {
+  if (!filter) {
+    return
+  }
+
   const parsedFilter = processSearchFilters(filter)
 
   if (!parsedFilter) {

From 3c158c5357f37f3318c2a54152005d8d6ea34e42 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Sep 2024 13:30:49 +0100
Subject: [PATCH 097/455] Add initial version of new RBAC editor

---
 packages/builder/package.json                 |   2 +
 .../buttons/ManageAccessButton.svelte         |   5 -
 .../DataTable/buttons/RoleEditor.svelte       |  32 -----
 .../DatasourceNavigator.svelte                |   6 +
 .../backend/RoleEditor/Controls.svelte        |  95 ++++++++++++++
 .../backend/RoleEditor/RoleEditor.svelte      |  97 ++++++++++++++
 .../backend/RoleEditor/RoleNode.svelte        | 120 ++++++++++++++++++
 .../backend/RoleEditor/constants.js           |   2 +
 .../components/backend/RoleEditor/layout.js   | 120 ++++++++++++++++++
 .../app/[application]/data/roles.svelte       |   5 +
 10 files changed, 447 insertions(+), 37 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte
 create mode 100644 packages/builder/src/components/backend/RoleEditor/Controls.svelte
 create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
 create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
 create mode 100644 packages/builder/src/components/backend/RoleEditor/constants.js
 create mode 100644 packages/builder/src/components/backend/RoleEditor/layout.js
 create mode 100644 packages/builder/src/pages/builder/app/[application]/data/roles.svelte

diff --git a/packages/builder/package.json b/packages/builder/package.json
index f44c2ea5498..36ec099b565 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -59,12 +59,14 @@
     "@codemirror/state": "^6.2.0",
     "@codemirror/theme-one-dark": "^6.1.2",
     "@codemirror/view": "^6.11.2",
+    "@dagrejs/dagre": "1.1.4",
     "@fontsource/source-sans-pro": "^5.0.3",
     "@fortawesome/fontawesome-svg-core": "^6.4.2",
     "@fortawesome/free-brands-svg-icons": "^6.4.2",
     "@fortawesome/free-solid-svg-icons": "^6.4.2",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
+    "@xyflow/svelte": "^0.1.18",
     "@zerodevx/svelte-json-view": "^1.0.7",
     "codemirror": "^5.65.16",
     "dayjs": "^1.10.8",
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 7316fc79a98..5c8ccddf592 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -15,7 +15,6 @@
   import { capitalise } from "helpers"
   import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
   import { Roles } from "constants/backend"
-  import RoleEditor from "./RoleEditor.svelte"
 
   export let resourceId
 
@@ -197,10 +196,6 @@
       on:show={() => (showPopover = false)}
       on:hide={() => (showPopover = true)}
     />
-    <RoleEditor
-      on:drawerShow={() => (showPopover = false)}
-      on:drawerHide={() => (showPopover = true)}
-    />
   </div>
 </DetailPopover>
 
diff --git a/packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte b/packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte
deleted file mode 100644
index 6b7e79eabc0..00000000000
--- a/packages/builder/src/components/backend/DataTable/buttons/RoleEditor.svelte
+++ /dev/null
@@ -1,32 +0,0 @@
-<script>
-  import {
-    Button,
-    Modal,
-    ModalContent,
-    Drawer,
-    DrawerContent,
-  } from "@budibase/bbui"
-
-  let drawer
-
-  $: invalid = false
-
-  const save = async () => {
-    drawer.hide()
-  }
-</script>
-
-<Button secondary icon="UsersLock" on:click on:click={drawer.show}>
-  Role editor
-</Button>
-
-<Drawer
-  bind:this={drawer}
-  title="Role editor"
-  on:drawerHide
-  on:drawerShow
-  forceModal
->
-  <Button disabled={invalid} cta slot="buttons" on:click={save}>Save</Button>
-  <DrawerContent slot="body">asdasdasd</DrawerContent>
-</Drawer>
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
index a6078e38fbd..621b8073a6e 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
@@ -76,6 +76,12 @@
       selectedBy={$userSelectedResourceMap[TableNames.USERS]}
     />
   {/if}
+  <NavItem
+    icon="UserAdmin"
+    text="Manage roles"
+    selected={$isActive("./roles")}
+    on:click={() => $goto("./roles")}
+  />
   {#each enrichedDataSources.filter(ds => ds.show) as datasource}
     <DatasourceNavItem
       {datasource}
diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
new file mode 100644
index 00000000000..07f070f8493
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -0,0 +1,95 @@
+<script>
+  import { Button, Helpers, ActionButton } from "@budibase/bbui"
+  import { useSvelteFlow, Position } from "@xyflow/svelte"
+  import { getContext } from "svelte"
+  import { dagreLayout } from "./layout"
+
+  const { nodes, edges } = getContext("flow")
+  const flow = useSvelteFlow()
+
+  const addRole = () => {
+    nodes.update(state => [
+      ...state,
+      {
+        id: Helpers.uuid(),
+        sourcePosition: Position.Right,
+        targetPosition: Position.Left,
+        type: "role",
+        data: {
+          label: "New role",
+          description: "Custom role",
+          custom: true,
+        },
+        position: { x: 0, y: 0 },
+      },
+    ])
+    autoLayout()
+  }
+
+  const autoLayout = () => {
+    const layout = dagreLayout({ nodes: $nodes, edges: $edges })
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+    flow.fitView({ maxZoom: 1 })
+  }
+</script>
+
+<div class="control top-left">
+  <div class="group">
+    <ActionButton icon="Add" quiet on:click={flow.zoomIn} />
+    <ActionButton icon="Remove" quiet on:click={flow.zoomOut} />
+  </div>
+  <Button secondary on:click={() => flow.fitView({ maxZoom: 1 })}>
+    Zoom to fit
+  </Button>
+  <Button secondary on:click={autoLayout}>Auto layout</Button>
+</div>
+<div class="control bottom-right">
+  <Button icon="Add" cta on:click={addRole}>Add role</Button>
+</div>
+
+<style>
+  .control {
+    position: absolute;
+    z-index: 999;
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    gap: 8px;
+  }
+  .top-left {
+    top: 20px;
+    left: 20px;
+  }
+  .bottom-right {
+    bottom: 20px;
+    right: 20px;
+  }
+  .top-left :global(.spectrum-Button),
+  .top-left :global(.spectrum-ActionButton),
+  .top-left :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+  .top-left :global(.spectrum-Button),
+  .top-left :global(.spectrum-ActionButton) {
+    background: var(--spectrum-global-color-gray-200) !important;
+  }
+  .top-left :global(.spectrum-Button:hover),
+  .top-left :global(.spectrum-ActionButton:hover) {
+    background: var(--spectrum-global-color-gray-300) !important;
+  }
+  .group {
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+  }
+  .group :global(> *:not(:first-child)) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+    border-left: 2px solid var(--spectrum-global-color-gray-300);
+  }
+  .group :global(> *:not(:last-child)) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+  }
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
new file mode 100644
index 00000000000..0bc4bd6aef4
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -0,0 +1,97 @@
+<script>
+  import { Heading } from "@budibase/bbui"
+  import { writable } from "svelte/store"
+  import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
+  import "@xyflow/svelte/dist/style.css"
+  import RoleNode from "./RoleNode.svelte"
+  import { initialLayout, dagreLayout } from "./layout"
+  import { onMount, setContext } from "svelte"
+  import Controls from "./Controls.svelte"
+
+  const nodes = writable([])
+  const edges = writable([])
+
+  setContext("flow", { nodes, edges })
+
+  onMount(() => {
+    const layout = dagreLayout(initialLayout())
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+  })
+</script>
+
+<div class="title">
+  <div class="heading">
+    <Heading size="S">Manage roles</Heading>
+  </div>
+  <div class="description">Roles inherit permissions from each other.</div>
+</div>
+<div class="flow">
+  <SvelteFlow
+    fitView
+    {nodes}
+    {edges}
+    snapGrid={[25, 25]}
+    nodeTypes={{ role: RoleNode }}
+    proOptions={{ hideAttribution: true }}
+    fitViewOptions={{ maxZoom: 1 }}
+    defaultEdgeOptions={{ type: "bezier", animated: true }}
+    on:nodeclick={event => console.log("on node click", event.detail.node)}
+  >
+    <Background variant={BackgroundVariant.Dots} />
+    <Controls />
+  </SvelteFlow>
+</div>
+
+<style>
+  .heading {
+    border-bottom: 1px solid var(--spectrum-global-color-gray-300);
+    margin-bottom: 20px;
+    padding-bottom: 12px;
+  }
+  .description {
+    color: var(--spectrum-global-color-gray-600);
+    margin-bottom: calc(20px - var(--spacing-l));
+  }
+  .flow {
+    flex: 1 1 auto;
+    border-radius: 8px;
+    overflow: hidden;
+    position: relative;
+    --background-color: var(--spectrum-global-color-gray-50);
+    --node-background: var(--spectrum-global-color-gray-100);
+    --node-background-hover: var(--spectrum-global-color-gray-300);
+    --border-color: var(--spectrum-global-color-gray-300);
+    --edge-color: var(--spectrum-global-color-gray-500);
+    --handle-color: var(--spectrum-global-color-gray-600);
+    --selected-color: var(--spectrum-global-color-blue-400);
+  }
+  .flow :global(.svelte-flow) {
+    /* Panel */
+    --xy-background-color: var(--background-color);
+    --xy-background-pattern-color-props: transparent;
+
+    /* Controls */
+    --xy-controls-button-background-color: var(--node-background);
+    --xy-controls-button-background-color-hover: var(--node-background-hover);
+    --xy-controls-button-border-color: var(--border-color);
+
+    /* Handles */
+    --xy-handle-background-color: var(--handle-color);
+    --xy-handle-border-color: var(--handle-color);
+
+    /* Edges */
+    --xy-edge-stroke: var(--edge-color);
+    --xy-edge-stroke-selected: var(--selected-color);
+
+    /* Minimap */
+    /* --xy-minimap-background-color-props: var(--node-background);
+    --xy-minimap-mask-background-color-props: var(--translucent-grey); */
+  }
+
+  /* Arrow edge markers */
+  /* .flow :global(.svelte-flow__arrowhead > polyline) {
+    fill: var(--edge-color);
+    stroke: var(--edge-color);
+  } */
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
new file mode 100644
index 00000000000..6261ac6decb
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -0,0 +1,120 @@
+<script>
+  import { RoleUtils } from "@budibase/frontend-core"
+  import { Handle, Position, useSvelteFlow } from "@xyflow/svelte"
+  import { Icon } from "@budibase/bbui"
+  import { Roles } from "constants/backend"
+  import { NodeWidth, NodeHeight } from "./constants"
+
+  export let data
+  export let isConnectable
+  export let id
+
+  const flow = useSvelteFlow()
+  const { label, description, custom } = data
+
+  $: color = RoleUtils.getRoleColour(id)
+
+  const deleteNode = () => {
+    flow.deleteElements({
+      nodes: [{ id }],
+    })
+  }
+</script>
+
+<div
+  class="node"
+  class:selected={false}
+  style={`--color:${color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
+>
+  <div class="color" />
+  <div class="content">
+    <div class="title">
+      <div class="label">
+        {label}
+      </div>
+      {#if custom}
+        <div class="buttons">
+          <Icon size="S" name="Edit" hoverable />
+          <Icon size="S" name="Delete" hoverable on:click={deleteNode} />
+        </div>
+      {/if}
+    </div>
+    {#if description}
+      <div class="description">
+        {description}
+      </div>
+    {/if}
+  </div>
+  {#if id !== Roles.BASIC}
+    <Handle type="target" position={Position.Left} {isConnectable} />
+  {/if}
+  {#if id !== Roles.ADMIN}
+    <Handle type="source" position={Position.Right} {isConnectable} />
+  {/if}
+</div>
+
+<style>
+  .node {
+    position: relative;
+    background: var(--node-background);
+    border-radius: 4px;
+    border: 1px solid transparent;
+    width: var(--width);
+    height: var(--height);
+    display: flex;
+    flex-direction: column;
+  }
+  .node.selected {
+    background: var(--spectrum-global-color-blue-100);
+  }
+  .color {
+    border-top-left-radius: 4px;
+    border-top-right-radius: 4px;
+    height: 8px;
+    width: 100%;
+    background: var(--color);
+  }
+  .content {
+    flex: 1 1 auto;
+    padding: 0 14px 0 14px;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: stretch;
+    gap: 2px;
+    border: 1px solid var(--border-color);
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+  }
+  .node.selected .content {
+    border-color: transparent;
+  }
+  .title,
+  .buttons {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    gap: 6px;
+  }
+  .title {
+    justify-content: space-between;
+  }
+  .buttons {
+    display: none;
+  }
+  .title :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-600);
+  }
+  .label {
+    white-space: nowrap;
+    text-overflow: ellipsis;
+    overflow: hidden;
+  }
+  .description {
+    color: var(--spectrum-global-color-gray-600);
+    font-size: 12px;
+  }
+  .node:hover .buttons {
+    display: flex;
+  }
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
new file mode 100644
index 00000000000..5c08561a230
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -0,0 +1,2 @@
+export const NodeWidth = 220
+export const NodeHeight = 66
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
new file mode 100644
index 00000000000..da24c0c4d0b
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -0,0 +1,120 @@
+import dagre from "@dagrejs/dagre"
+import { NodeWidth, NodeHeight } from "./constants"
+import { Position } from "@xyflow/svelte"
+import { roles } from "stores/builder"
+import { Roles } from "constants/backend"
+import { get } from "svelte/store"
+
+export const initialLayout = () => {
+  const builtins = [Roles.BASIC, Roles.POWER, Roles.ADMIN]
+  const descriptions = {
+    [Roles.BASIC]: "Basic user",
+    [Roles.POWER]: "Power user",
+    [Roles.ADMIN]: "Can do everything",
+  }
+  const $roles = get(roles)
+  const nodes = builtins
+    .map(roleId => {
+      return {
+        id: roleId,
+        sourcePosition: Position.Right,
+        targetPosition: Position.Left,
+        type: "role",
+        data: {
+          label: $roles.find(x => x._id === roleId)?.name,
+          description: descriptions[roleId],
+        },
+      }
+    })
+    .concat([
+      {
+        id: "management",
+        sourcePosition: Position.Right,
+        targetPosition: Position.Left,
+        type: "role",
+        data: {
+          label: "Management",
+          description: "Custom role",
+          custom: true,
+        },
+      },
+      {
+        id: "approver",
+        sourcePosition: Position.Right,
+        targetPosition: Position.Left,
+        type: "role",
+        data: {
+          label: "Approver",
+          description: "Custom role",
+          custom: true,
+        },
+      },
+      {
+        id: "engineer",
+        sourcePosition: Position.Right,
+        targetPosition: Position.Left,
+        type: "role",
+        data: {
+          label: "Engineer",
+          description: "Custom role",
+          custom: true,
+        },
+      },
+    ])
+
+  let edges = []
+  const link = (source, target) => {
+    edges.push({
+      id: `${source}-${target}`,
+      source,
+      target,
+      animated: true,
+      // markerEnd: {
+      //   type: MarkerType.ArrowClosed,
+      //   width: 16,
+      //   height: 16,
+      // },
+    })
+  }
+
+  link(Roles.BASIC, "engineer")
+  link(Roles.BASIC, "approver")
+
+  link("engineer", Roles.POWER)
+  link("approver", "management")
+
+  link(Roles.POWER, Roles.ADMIN)
+  link("management", Roles.ADMIN)
+
+  return {
+    nodes,
+    edges,
+  }
+}
+
+export const dagreLayout = ({ nodes, edges }) => {
+  const dagreGraph = new dagre.graphlib.Graph()
+  dagreGraph.setDefaultEdgeLabel(() => ({}))
+  dagreGraph.setGraph({
+    rankdir: "LR",
+    ranksep: 100,
+    nodesep: 100,
+  })
+  nodes.forEach(node => {
+    dagreGraph.setNode(node.id, { width: NodeWidth, height: NodeHeight })
+  })
+  edges.forEach(edge => {
+    dagreGraph.setEdge(edge.source, edge.target)
+  })
+  dagre.layout(dagreGraph)
+  nodes.forEach(node => {
+    const nodeWithPosition = dagreGraph.node(node.id)
+    node.targetPosition = Position.Left
+    node.sourcePosition = Position.Right
+    node.position = {
+      x: nodeWithPosition.x - NodeWidth / 2,
+      y: nodeWithPosition.y - NodeHeight / 2,
+    }
+  })
+  return { nodes, edges }
+}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/roles.svelte b/packages/builder/src/pages/builder/app/[application]/data/roles.svelte
new file mode 100644
index 00000000000..bea482d9d2c
--- /dev/null
+++ b/packages/builder/src/pages/builder/app/[application]/data/roles.svelte
@@ -0,0 +1,5 @@
+<script>
+  import RoleEditor from "components/backend/RoleEditor/RoleEditor.svelte"
+</script>
+
+<RoleEditor />

From dd98364a5d20f0b3d4d8421187962f9d801dd5ac Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Sep 2024 16:26:32 +0100
Subject: [PATCH 098/455] Improve RBAC flow chart

---
 .../backend/RoleEditor/Controls.svelte        | 10 +----
 .../backend/RoleEditor/RoleEditor.svelte      | 41 ++++++++++++++++---
 .../backend/RoleEditor/RoleNode.svelte        |  6 ++-
 3 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 07f070f8493..2824500c472 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -2,9 +2,8 @@
   import { Button, Helpers, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow, Position } from "@xyflow/svelte"
   import { getContext } from "svelte"
-  import { dagreLayout } from "./layout"
 
-  const { nodes, edges } = getContext("flow")
+  const { nodes, autoLayout } = getContext("flow")
   const flow = useSvelteFlow()
 
   const addRole = () => {
@@ -25,13 +24,6 @@
     ])
     autoLayout()
   }
-
-  const autoLayout = () => {
-    const layout = dagreLayout({ nodes: $nodes, edges: $edges })
-    nodes.set(layout.nodes)
-    edges.set(layout.edges)
-    flow.fitView({ maxZoom: 1 })
-  }
 </script>
 
 <div class="control top-left">
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 0bc4bd6aef4..5d710dbb43b 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -1,17 +1,48 @@
 <script>
-  import { Heading } from "@budibase/bbui"
-  import { writable } from "svelte/store"
+  import { Heading, Helpers } from "@budibase/bbui"
+  import { derived, writable } from "svelte/store"
   import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
   import { initialLayout, dagreLayout } from "./layout"
-  import { onMount, setContext } from "svelte"
+  import { onMount, setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
+  import { Roles } from "constants/backend"
 
   const nodes = writable([])
-  const edges = writable([])
+  const edgeStore = writable([])
+  const enrichedEdges = derived([edgeStore, nodes], ([$edgeStore, $nodes]) => {
+    let additions = []
+    for (let node of $nodes) {
+      // If a certain node does not inherit anything, make it inherit basic
+      if (
+        !$edgeStore.some(x => x.target === node.id) &&
+        node.id !== Roles.BASIC
+      ) {
+        additions.push({
+          id: Helpers.uuid(),
+          source: Roles.BASIC,
+          target: node.id,
+          animated: true,
+        })
+      }
+    }
+    return [...$edgeStore, ...additions]
+  })
+  const edges = {
+    ...edgeStore,
+    subscribe: enrichedEdges.subscribe,
+  }
 
-  setContext("flow", { nodes, edges })
+  setContext("flow", {
+    nodes,
+    edges,
+    autoLayout: async () => {
+      const layout = dagreLayout({ nodes: $nodes, edges: $edges })
+      nodes.set(layout.nodes)
+      edges.set(layout.edges)
+    },
+  })
 
   onMount(() => {
     const layout = dagreLayout(initialLayout())
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 6261ac6decb..b8d59a19ce2 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -4,20 +4,24 @@
   import { Icon } from "@budibase/bbui"
   import { Roles } from "constants/backend"
   import { NodeWidth, NodeHeight } from "./constants"
+  import { getContext, tick } from "svelte"
 
   export let data
   export let isConnectable
   export let id
 
+  const { autoLayout } = getContext("flow")
   const flow = useSvelteFlow()
   const { label, description, custom } = data
 
   $: color = RoleUtils.getRoleColour(id)
 
-  const deleteNode = () => {
+  const deleteNode = async () => {
     flow.deleteElements({
       nodes: [{ id }],
     })
+    await tick()
+    autoLayout()
   }
 </script>
 

From d8dc8744a90f99ef226003ec045ea49dd9cbf228 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Sep 2024 16:44:50 +0100
Subject: [PATCH 099/455] Add ability to update roles

---
 packages/bbui/src/index.js                    |   1 +
 .../backend/RoleEditor/RoleNode.svelte        |  60 +++++++-
 yarn.lock                                     | 145 ++++++++++++++++++
 3 files changed, 198 insertions(+), 8 deletions(-)

diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js
index 083b4ecfd50..3efafed156e 100644
--- a/packages/bbui/src/index.js
+++ b/packages/bbui/src/index.js
@@ -21,6 +21,7 @@ export { default as EnvDropdown } from "./Form/EnvDropdown.svelte"
 export { default as Multiselect } from "./Form/Multiselect.svelte"
 export { default as Search } from "./Form/Search.svelte"
 export { default as RichTextField } from "./Form/RichTextField.svelte"
+export { default as FieldLabel } from "./Form/FieldLabel.svelte"
 export { default as Slider } from "./Form/Slider.svelte"
 export { default as File } from "./Form/File.svelte"
 
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index b8d59a19ce2..fd7216af772 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -1,10 +1,18 @@
 <script>
   import { RoleUtils } from "@budibase/frontend-core"
   import { Handle, Position, useSvelteFlow } from "@xyflow/svelte"
-  import { Icon } from "@budibase/bbui"
+  import {
+    Icon,
+    Input,
+    ColorPicker,
+    Modal,
+    ModalContent,
+    FieldLabel,
+  } from "@budibase/bbui"
   import { Roles } from "constants/backend"
   import { NodeWidth, NodeHeight } from "./constants"
   import { getContext, tick } from "svelte"
+  import { template } from "lodash"
 
   export let data
   export let isConnectable
@@ -12,9 +20,13 @@
 
   const { autoLayout } = getContext("flow")
   const flow = useSvelteFlow()
-  const { label, description, custom } = data
 
-  $: color = RoleUtils.getRoleColour(id)
+  let anchor
+  let modal
+  let tempLabel
+  let tempColor
+
+  $: color = data.color || RoleUtils.getRoleColour(id)
 
   const deleteNode = async () => {
     flow.deleteElements({
@@ -23,29 +35,43 @@
     await tick()
     autoLayout()
   }
+
+  const openPopover = () => {
+    tempLabel = data.label
+    tempColor = color
+    modal.show()
+  }
+
+  const saveChanges = () => {
+    flow.updateNodeData(id, {
+      label: tempLabel,
+      color: tempColor,
+    })
+  }
 </script>
 
 <div
   class="node"
   class:selected={false}
   style={`--color:${color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
+  bind:this={anchor}
 >
   <div class="color" />
   <div class="content">
     <div class="title">
       <div class="label">
-        {label}
+        {data.label}
       </div>
-      {#if custom}
+      {#if data.custom}
         <div class="buttons">
-          <Icon size="S" name="Edit" hoverable />
+          <Icon size="S" name="Edit" hoverable on:click={openPopover} />
           <Icon size="S" name="Delete" hoverable on:click={deleteNode} />
         </div>
       {/if}
     </div>
-    {#if description}
+    {#if data.description}
       <div class="description">
-        {description}
+        {data.description}
       </div>
     {/if}
   </div>
@@ -57,6 +83,24 @@
   {/if}
 </div>
 
+<Modal bind:this={modal}>
+  <ModalContent
+    title={`Edit ${data.label}`}
+    confirmText="Save"
+    onConfirm={saveChanges}
+  >
+    <Input
+      label="Name"
+      value={tempLabel}
+      on:change={e => (tempLabel = e.detail)}
+    />
+    <div>
+      <FieldLabel label="Color" />
+      <ColorPicker value={tempColor} on:change={e => (tempColor = e.detail)} />
+    </div>
+  </ModalContent>
+</Modal>
+
 <style>
   .node {
     position: relative;
diff --git a/yarn.lock b/yarn.lock
index 655d7d78fcc..a1b6fe57140 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2342,6 +2342,18 @@
     enabled "2.0.x"
     kuler "^2.0.0"
 
+"@dagrejs/dagre@1.1.4":
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/@dagrejs/dagre/-/dagre-1.1.4.tgz#66f9c0e2b558308f2c268f60e2c28f22ee17e339"
+  integrity sha512-QUTc54Cg/wvmlEUxB+uvoPVKFazM1H18kVHBQNmK2NbrDR5ihOCR6CXLnDSZzMcSQKJtabPUWridBOlJM3WkDg==
+  dependencies:
+    "@dagrejs/graphlib" "2.2.4"
+
+"@dagrejs/graphlib@2.2.4":
+  version "2.2.4"
+  resolved "https://registry.yarnpkg.com/@dagrejs/graphlib/-/graphlib-2.2.4.tgz#d77bfa9ff49e2307c0c6e6b8b26b5dd3c05816c4"
+  integrity sha512-mepCf/e9+SKYy1d02/UkvSy6+6MoyXhVxP8lLDfA7BPE1X1d4dR0sZznmbM8/XVJ1GPM+Svnx7Xj6ZweByWUkw==
+
 "@datadog/native-appsec@7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-7.0.0.tgz#a380174dd49aef2d9bb613a0ec8ead6dc7822095"
@@ -5182,6 +5194,11 @@
   resolved "https://registry.yarnpkg.com/@spectrum-css/vars/-/vars-4.3.1.tgz#d333fa41909f691c8750b5c15ad9ba029df2248e"
   integrity sha512-rX6Iasu9BsFMVgEN0vGRPm9dmSxva+IK/uqQAa9HM0lliwqUiFrJxrFXHHpiAgNuux/U4srEJwbSpGzfF+CegQ==
 
+"@svelte-put/shortcut@^3.1.0":
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/@svelte-put/shortcut/-/shortcut-3.1.1.tgz#aba4d7407024d5cff38727e12925c8f81e877079"
+  integrity sha512-2L5EYTZXiaKvbEelVkg5znxqvfZGZai3m97+cAiUBhLZwXnGtviTDpHxOoZBsqz41szlfRMcamW/8o0+fbW3ZQ==
+
 "@sveltejs/vite-plugin-svelte@1.4.0":
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-1.4.0.tgz#412a735de489ca731d0c780c2b410f45dd95b392"
@@ -5540,6 +5557,45 @@
   dependencies:
     "@types/node" "*"
 
+"@types/d3-color@*":
+  version "3.1.3"
+  resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-3.1.3.tgz#368c961a18de721da8200e80bf3943fb53136af2"
+  integrity sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==
+
+"@types/d3-drag@^3.0.7":
+  version "3.0.7"
+  resolved "https://registry.yarnpkg.com/@types/d3-drag/-/d3-drag-3.0.7.tgz#b13aba8b2442b4068c9a9e6d1d82f8bcea77fc02"
+  integrity sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==
+  dependencies:
+    "@types/d3-selection" "*"
+
+"@types/d3-interpolate@*":
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz#412b90e84870285f2ff8a846c6eb60344f12a41c"
+  integrity sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==
+  dependencies:
+    "@types/d3-color" "*"
+
+"@types/d3-selection@*", "@types/d3-selection@^3.0.10":
+  version "3.0.10"
+  resolved "https://registry.yarnpkg.com/@types/d3-selection/-/d3-selection-3.0.10.tgz#98cdcf986d0986de6912b5892e7c015a95ca27fe"
+  integrity sha512-cuHoUgS/V3hLdjJOLTT691+G2QoqAjCVLmr4kJXR4ha56w1Zdu8UUQ5TxLRqudgNjwXeQxKMq4j+lyf9sWuslg==
+
+"@types/d3-transition@^3.0.8":
+  version "3.0.8"
+  resolved "https://registry.yarnpkg.com/@types/d3-transition/-/d3-transition-3.0.8.tgz#677707f5eed5b24c66a1918cde05963021351a8f"
+  integrity sha512-ew63aJfQ/ms7QQ4X7pk5NxQ9fZH/z+i24ZfJ6tJSfqxJMrYLiK01EAs2/Rtw/JreGUsS3pLPNV644qXFGnoZNQ==
+  dependencies:
+    "@types/d3-selection" "*"
+
+"@types/d3-zoom@^3.0.8":
+  version "3.0.8"
+  resolved "https://registry.yarnpkg.com/@types/d3-zoom/-/d3-zoom-3.0.8.tgz#dccb32d1c56b1e1c6e0f1180d994896f038bc40b"
+  integrity sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==
+  dependencies:
+    "@types/d3-interpolate" "*"
+    "@types/d3-selection" "*"
+
 "@types/debug@*":
   version "4.1.7"
   resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82"
@@ -6753,6 +6809,28 @@
     loupe "^3.1.1"
     tinyrainbow "^1.2.0"
 
+"@xyflow/svelte@^0.1.18":
+  version "0.1.18"
+  resolved "https://registry.yarnpkg.com/@xyflow/svelte/-/svelte-0.1.18.tgz#ba2f9f72adc64ff6f71a5ad03cf759af8d7c9748"
+  integrity sha512-P2td3XcvMk36pnhyRUAXtmwfd7sv1KAHVF29YZUNndYlgxG98vwj1UoyyuXwCHIiyu82GgowaTppHCNPXsvNSg==
+  dependencies:
+    "@svelte-put/shortcut" "^3.1.0"
+    "@xyflow/system" "0.0.41"
+    classcat "^5.0.4"
+
+"@xyflow/system@0.0.41":
+  version "0.0.41"
+  resolved "https://registry.yarnpkg.com/@xyflow/system/-/system-0.0.41.tgz#6c314b2bbca594aec4d7cdb56efb003be6727d21"
+  integrity sha512-XAjs8AUA0YMfYD91cT6pLGALwbsPS64s2WBHyULqL1m0gTqXqaUSLK1P7qA/Q8HecN0RFbqlM2tPO8bmZXP0YQ==
+  dependencies:
+    "@types/d3-drag" "^3.0.7"
+    "@types/d3-selection" "^3.0.10"
+    "@types/d3-transition" "^3.0.8"
+    "@types/d3-zoom" "^3.0.8"
+    d3-drag "^3.0.0"
+    d3-selection "^3.0.0"
+    d3-zoom "^3.0.0"
+
 "@yarnpkg/lockfile@^1.1.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31"
@@ -8447,6 +8525,11 @@ cjs-module-lexer@^1.0.0, cjs-module-lexer@^1.2.2:
   resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz#6c370ab19f8a3394e318fe682686ec0ac684d107"
   integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==
 
+classcat@^5.0.4:
+  version "5.0.5"
+  resolved "https://registry.yarnpkg.com/classcat/-/classcat-5.0.5.tgz#8c209f359a93ac302404a10161b501eba9c09c77"
+  integrity sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==
+
 clean-stack@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b"
@@ -9372,6 +9455,68 @@ curlconverter@3.21.0:
     string.prototype.startswith "^1.0.0"
     yamljs "^0.3.0"
 
+"d3-color@1 - 3":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2"
+  integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==
+
+"d3-dispatch@1 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-3.0.1.tgz#5fc75284e9c2375c36c839411a0cf550cbfc4d5e"
+  integrity sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==
+
+"d3-drag@2 - 3", d3-drag@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/d3-drag/-/d3-drag-3.0.0.tgz#994aae9cd23c719f53b5e10e3a0a6108c69607ba"
+  integrity sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==
+  dependencies:
+    d3-dispatch "1 - 3"
+    d3-selection "3"
+
+"d3-ease@1 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-3.0.1.tgz#9658ac38a2140d59d346160f1f6c30fda0bd12f4"
+  integrity sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==
+
+"d3-interpolate@1 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d"
+  integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==
+  dependencies:
+    d3-color "1 - 3"
+
+"d3-selection@2 - 3", d3-selection@3, d3-selection@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/d3-selection/-/d3-selection-3.0.0.tgz#c25338207efa72cc5b9bd1458a1a41901f1e1b31"
+  integrity sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==
+
+"d3-timer@1 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0"
+  integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==
+
+"d3-transition@2 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-transition/-/d3-transition-3.0.1.tgz#6869fdde1448868077fdd5989200cb61b2a1645f"
+  integrity sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==
+  dependencies:
+    d3-color "1 - 3"
+    d3-dispatch "1 - 3"
+    d3-ease "1 - 3"
+    d3-interpolate "1 - 3"
+    d3-timer "1 - 3"
+
+d3-zoom@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/d3-zoom/-/d3-zoom-3.0.0.tgz#d13f4165c73217ffeaa54295cd6969b3e7aee8f3"
+  integrity sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==
+  dependencies:
+    d3-dispatch "1 - 3"
+    d3-drag "2 - 3"
+    d3-interpolate "1 - 3"
+    d3-selection "2 - 3"
+    d3-transition "2 - 3"
+
 dargs@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/dargs/-/dargs-7.0.0.tgz#04015c41de0bcb69ec84050f3d9be0caf8d6d5cc"

From 3295d90b0fe5a04a8397abd2a94a9fc49e8a49f9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 08:43:19 +0100
Subject: [PATCH 100/455] Update mapping of roles to new flow structure

---
 .../backend/RoleEditor/RoleEditor.svelte      |  10 +-
 .../components/backend/RoleEditor/layout.js   | 142 +++++++++---------
 2 files changed, 79 insertions(+), 73 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 5d710dbb43b..0e70d273083 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -4,10 +4,11 @@
   import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
-  import { initialLayout, dagreLayout } from "./layout"
+  import { rolesToLayout, dagreLayout, layoutToRoles } from "./layout"
   import { onMount, setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
   import { Roles } from "constants/backend"
+  import { derivedMemo } from "@budibase/frontend-core"
 
   const nodes = writable([])
   const edgeStore = writable([])
@@ -34,6 +35,11 @@
     subscribe: enrichedEdges.subscribe,
   }
 
+  const roles = derivedMemo([nodes, edges], ([$nodes, $edges]) => {
+    return layoutToRoles({ nodes: $nodes, edges: $edges })
+  })
+  $: console.log("new roles", $roles)
+
   setContext("flow", {
     nodes,
     edges,
@@ -45,7 +51,7 @@
   })
 
   onMount(() => {
-    const layout = dagreLayout(initialLayout())
+    const layout = dagreLayout(rolesToLayout())
     nodes.set(layout.nodes)
     edges.set(layout.edges)
   })
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index da24c0c4d0b..dd1550be510 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -4,87 +4,54 @@ import { Position } from "@xyflow/svelte"
 import { roles } from "stores/builder"
 import { Roles } from "constants/backend"
 import { get } from "svelte/store"
+import { Helpers } from "@budibase/bbui"
 
-export const initialLayout = () => {
-  const builtins = [Roles.BASIC, Roles.POWER, Roles.ADMIN]
+export const rolesToLayout = () => {
+  const ignoredRoles = [Roles.PUBLIC]
+  const $roles = get(roles)
   const descriptions = {
     [Roles.BASIC]: "Basic user",
     [Roles.POWER]: "Power user",
     [Roles.ADMIN]: "Can do everything",
   }
-  const $roles = get(roles)
-  const nodes = builtins
-    .map(roleId => {
-      return {
-        id: roleId,
-        sourcePosition: Position.Right,
-        targetPosition: Position.Left,
-        type: "role",
-        data: {
-          label: $roles.find(x => x._id === roleId)?.name,
-          description: descriptions[roleId],
-        },
-      }
-    })
-    .concat([
-      {
-        id: "management",
-        sourcePosition: Position.Right,
-        targetPosition: Position.Left,
-        type: "role",
-        data: {
-          label: "Management",
-          description: "Custom role",
-          custom: true,
-        },
-      },
-      {
-        id: "approver",
-        sourcePosition: Position.Right,
-        targetPosition: Position.Left,
-        type: "role",
-        data: {
-          label: "Approver",
-          description: "Custom role",
-          custom: true,
-        },
-      },
-      {
-        id: "engineer",
-        sourcePosition: Position.Right,
-        targetPosition: Position.Left,
-        type: "role",
-        data: {
-          label: "Engineer",
-          description: "Custom role",
-          custom: true,
-        },
-      },
-    ])
 
+  let nodes = []
   let edges = []
-  const link = (source, target) => {
-    edges.push({
-      id: `${source}-${target}`,
-      source,
-      target,
-      animated: true,
-      // markerEnd: {
-      //   type: MarkerType.ArrowClosed,
-      //   width: 16,
-      //   height: 16,
-      // },
-    })
-  }
 
-  link(Roles.BASIC, "engineer")
-  link(Roles.BASIC, "approver")
-
-  link("engineer", Roles.POWER)
-  link("approver", "management")
+  for (let role of $roles) {
+    if (ignoredRoles.includes(role._id)) {
+      continue
+    }
+    nodes.push({
+      id: role._id,
+      sourcePosition: Position.Right,
+      targetPosition: Position.Left,
+      type: "role",
+      data: {
+        label: role.name,
+        description: descriptions[role._id] || "Custom role",
+        color: role.color,
+        custom: !role._id.match(/[A-Z]+/),
+      },
+    })
 
-  link(Roles.POWER, Roles.ADMIN)
-  link("management", Roles.ADMIN)
+    let inherits = []
+    if (role.inherits) {
+      inherits = Array.isArray(role.inherits) ? role.inherits : [role.inherits]
+    }
+    for (let sourceRole of inherits) {
+      // Ensure source role exists
+      if (!$roles.some(x => x._id === sourceRole)) {
+        continue
+      }
+      edges.push({
+        id: `${sourceRole}-${role._id}`,
+        source: sourceRole,
+        target: role._id,
+        animated: true,
+      })
+    }
+  }
 
   return {
     nodes,
@@ -92,6 +59,39 @@ export const initialLayout = () => {
   }
 }
 
+export const layoutToRoles = ({ nodes, edges }) => {
+  // Clone and wipe existing inheritance
+  let newRoles = Helpers.cloneDeep(get(roles)).map(role => {
+    return { ...role, inherits: [] }
+  })
+
+  // Copy over names and colours
+  for (let node of nodes) {
+    let role = newRoles.find(x => x._id === node.id)
+    if (role) {
+      role.name = node.data.label
+      role.color = node.data.color
+    } else {
+      // New role
+    }
+  }
+
+  // Build inheritance
+  for (let edge of edges) {
+    let role = newRoles.find(x => x._id === edge.target)
+    if (role) {
+      role.inherits.push(edge.source)
+    } else {
+      // New role
+    }
+  }
+
+  // Ensure basic is correct
+  newRoles.find(x => x._id === Roles.BASIC).inherits = [Roles.BASIC]
+
+  return newRoles
+}
+
 export const dagreLayout = ({ nodes, edges }) => {
   const dagreGraph = new dagre.graphlib.Graph()
   dagreGraph.setDefaultEdgeLabel(() => ({}))

From 5be3fe2722dfd637c0a2ad618bb40fc9c229956a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 08:45:07 +0100
Subject: [PATCH 101/455] Remove screens and automations button from users
 table

---
 .../[application]/data/table/[tableId]/index.svelte    | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 82d27bd4b84..b9b58cbfce8 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -137,12 +137,10 @@
       {/if}
       {#if !isUsersTable}
         <GridRowActionsButton />
-      {/if}
-      <GridScreensButton on:request-generate={() => generateButton?.show()} />
-      <GridAutomationsButton
-        on:request-generate={() => generateButton?.show()}
-      />
-      {#if !isUsersTable}
+        <GridScreensButton on:request-generate={() => generateButton?.show()} />
+        <GridAutomationsButton
+          on:request-generate={() => generateButton?.show()}
+        />
         <GridImportButton />
       {/if}
       <GridExportButton />

From 2d23eaee5e8002cd86cea6f7228b4ecb2b0f8f34 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 08:52:51 +0100
Subject: [PATCH 102/455] Remove empty file

---
 .../data/table/[tableId]/_components/CreateViewModal.svelte     | 2 --
 1 file changed, 2 deletions(-)
 delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
deleted file mode 100644
index 49f5fcd1ff6..00000000000
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewModal.svelte
+++ /dev/null
@@ -1,2 +0,0 @@
-<script>
-</script>

From 9ea74dcb453d348925d6763a0df9ed90f6c5cb48 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 11 Sep 2024 11:56:09 +0100
Subject: [PATCH 103/455] Updates for view tests

---
 .../server/src/api/controllers/row/views.ts   |  19 +--
 .../server/src/sdk/app/rows/queryUtils.ts     |   6 +-
 packages/shared-core/src/filters.ts           | 136 +++++++++++++++++-
 3 files changed, 150 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts
index 8a1e5e08ced..59575318570 100644
--- a/packages/server/src/api/controllers/row/views.ts
+++ b/packages/server/src/api/controllers/row/views.ts
@@ -33,23 +33,26 @@ export async function searchView(
     .map(([key]) => key)
   const { body } = ctx.request
 
+  const sqsEnabled = await features.flags.isEnabled("SQS")
+  const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled
+
   // Enrich saved query with ephemeral query params.
   // We prevent searching on any fields that are saved as part of the query, as
   // that could let users find rows they should not be allowed to access.
-  let query: any = dataFilters.buildQuery(view.query)
+  let query: any = supportsLogicalOperators
+    ? dataFilters.buildQuery(view.query)
+    : dataFilters.buildQueryLegacy(view.query as SearchFilter[])
+
+  delete query?.onEmptyFilter
+
   if (body.query) {
     // Delete extraneous search params that cannot be overridden
     delete body.query.onEmptyFilter
 
-    if (
-      !isExternalTableID(view.tableId) &&
-      !(await features.flags.isEnabled("SQS"))
-    ) {
+    if (!supportsLogicalOperators) {
       // In the unlikely event that a Grouped Filter is in a non-SQS environment
       // It needs to be ignored. Entirely
-      let queryFilters: SearchFilter[] = Array.isArray(view.query)
-        ? view.query
-        : []
+      let queryFilters: SearchFilter[] = Array.isArray(query) ? query : []
 
       // Extract existing fields
       const existingFields =
diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts
index 65f400a1d9c..a73992bcee1 100644
--- a/packages/server/src/sdk/app/rows/queryUtils.ts
+++ b/packages/server/src/sdk/app/rows/queryUtils.ts
@@ -15,7 +15,9 @@ export const removeInvalidFilters = (
   const result = cloneDeep(filters)
 
   validFields = validFields.map(f => f.toLowerCase())
-  for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) {
+  for (const filterKey of Object.keys(
+    result || {}
+  ) as (keyof SearchFilters)[]) {
     const filter = result[filterKey]
     if (!filter || typeof filter !== "object") {
       continue
@@ -24,7 +26,7 @@ export const removeInvalidFilters = (
       const resultingConditions: SearchFilters[] = []
       for (const condition of filter.conditions) {
         const resultingCondition = removeInvalidFilters(condition, validFields)
-        if (Object.keys(resultingCondition).length) {
+        if (Object.keys(resultingCondition || {}).length) {
           resultingConditions.push(resultingCondition)
         }
       }
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 97e1212401b..63d3044da90 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -428,6 +428,126 @@ const builderFilter = (expression: SearchFilter) => {
   return query
 }
 
+export const buildQueryLegacy = (filter: SearchFilter[]) => {
+  //
+  let query: SearchFilters = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
+  }
+
+  if (!Array.isArray(filter)) {
+    return query
+  }
+
+  filter.forEach(expression => {
+    let { operator, field, type, value, externalType, onEmptyFilter } =
+      expression
+    const queryOperator = operator as SearchFilterOperator
+    const isHbs =
+      typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+    // Parse all values into correct types
+    if (operator === "allOr") {
+      query.allOr = true
+      return
+    }
+    if (onEmptyFilter) {
+      query.onEmptyFilter = onEmptyFilter
+      return
+    }
+    if (
+      type === "datetime" &&
+      !isHbs &&
+      queryOperator !== "empty" &&
+      queryOperator !== "notEmpty"
+    ) {
+      // Ensure date value is a valid date and parse into correct format
+      if (!value) {
+        return
+      }
+      try {
+        value = new Date(value).toISOString()
+      } catch (error) {
+        return
+      }
+    }
+    if (type === "number" && typeof value === "string" && !isHbs) {
+      if (queryOperator === "oneOf") {
+        value = value.split(",").map(item => parseFloat(item))
+      } else {
+        value = parseFloat(value)
+      }
+    }
+    if (type === "boolean") {
+      value = `${value}`?.toLowerCase() === "true"
+    }
+    if (
+      ["contains", "notContains", "containsAny"].includes(operator) &&
+      type === "array" &&
+      typeof value === "string"
+    ) {
+      value = value.split(",")
+    }
+    if (operator.startsWith("range") && query.range) {
+      const minint =
+        SqlNumberTypeRangeMap[
+          externalType as keyof typeof SqlNumberTypeRangeMap
+        ]?.min || Number.MIN_SAFE_INTEGER
+      const maxint =
+        SqlNumberTypeRangeMap[
+          externalType as keyof typeof SqlNumberTypeRangeMap
+        ]?.max || Number.MAX_SAFE_INTEGER
+      if (!query.range[field]) {
+        query.range[field] = {
+          low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+          high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+        }
+      }
+      if (operator === "rangeLow" && value != null && value !== "") {
+        query.range[field] = {
+          ...query.range[field],
+          low: value,
+        }
+      } else if (operator === "rangeHigh" && value != null && value !== "") {
+        query.range[field] = {
+          ...query.range[field],
+          high: value,
+        }
+      }
+    } else if (isLogicalSearchOperator(queryOperator)) {
+      // TODO
+    } else if (query[queryOperator] && operator !== "onEmptyFilter") {
+      if (type === "boolean") {
+        // Transform boolean filters to cope with null.
+        // "equals false" needs to be "not equals true"
+        // "not equals false" needs to be "equals true"
+        if (queryOperator === "equal" && value === false) {
+          query.notEqual = query.notEqual || {}
+          query.notEqual[field] = true
+        } else if (queryOperator === "notEqual" && value === false) {
+          query.equal = query.equal || {}
+          query.equal[field] = true
+        } else {
+          query[queryOperator] ??= {}
+          query[queryOperator]![field] = value
+        }
+      } else {
+        query[queryOperator] ??= {}
+        query[queryOperator]![field] = value
+      }
+    }
+  })
+  return query
+}
+
 export const buildQuery = (filter?: SearchFilterGroup | SearchFilter[]) => {
   if (!filter) {
     return
@@ -891,9 +1011,23 @@ export const hasFilters = (query?: SearchFilters) => {
   }
   const queryRoot = query[LogicalOperator.AND] ?? query[LogicalOperator.OR]
 
+  if (!queryRoot) {
+    const skipped = ["allOr", "onEmptyFilter"]
+    for (let [key, value] of Object.entries(query)) {
+      if (skipped.includes(key) || typeof value !== "object") {
+        continue
+      }
+      if (Object.keys(value || {}).length !== 0) {
+        return true
+      }
+    }
+    return false
+  }
+
   return (
     (queryRoot?.conditions || []).reduce((acc, group) => {
-      const groupRoot = group[LogicalOperator.AND] ?? group[LogicalOperator.OR]
+      const groupRoot =
+        group?.[LogicalOperator.AND] ?? group?.[LogicalOperator.OR]
       acc += groupRoot?.conditions?.length || 0
       return acc
     }, 0) > 0

From 9ea3aca5e4776599aa5924917b7d73a2ebbeba4b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 11 Sep 2024 12:21:19 +0100
Subject: [PATCH 104/455] Typing improvements and fixing issue detected by test
 cases (old lucene syntax).

---
 .../server/src/api/controllers/row/views.ts     | 17 ++++++++++-------
 packages/shared-core/src/filters.ts             | 10 ++++++----
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts
index 59575318570..de01386f6e8 100644
--- a/packages/server/src/api/controllers/row/views.ts
+++ b/packages/server/src/api/controllers/row/views.ts
@@ -39,9 +39,9 @@ export async function searchView(
   // Enrich saved query with ephemeral query params.
   // We prevent searching on any fields that are saved as part of the query, as
   // that could let users find rows they should not be allowed to access.
-  let query: any = supportsLogicalOperators
+  let query = supportsLogicalOperators
     ? dataFilters.buildQuery(view.query)
-    : dataFilters.buildQueryLegacy(view.query as SearchFilter[])
+    : dataFilters.buildQueryLegacy(view.query)
 
   delete query?.onEmptyFilter
 
@@ -51,8 +51,10 @@ export async function searchView(
 
     if (!supportsLogicalOperators) {
       // In the unlikely event that a Grouped Filter is in a non-SQS environment
-      // It needs to be ignored. Entirely
-      let queryFilters: SearchFilter[] = Array.isArray(query) ? query : []
+      // It needs to be ignored entirely
+      let queryFilters: SearchFilter[] = Array.isArray(view.query)
+        ? view.query
+        : []
 
       // Extract existing fields
       const existingFields =
@@ -64,15 +66,16 @@ export async function searchView(
       Object.keys(body.query).forEach(key => {
         const operator = key as Exclude<SearchFilterKey, LogicalOperator>
         Object.keys(body.query[operator] || {}).forEach(field => {
-          if (!existingFields.includes(db.removeKeyNumbering(field))) {
+          if (query && !existingFields.includes(db.removeKeyNumbering(field))) {
             query[operator]![field] = body.query[operator]![field]
           }
         })
       })
     } else {
+      const conditions = query ? [query] : []
       query = {
         $and: {
-          conditions: [query, body.query],
+          conditions: [...conditions, body.query],
         },
       }
     }
@@ -80,7 +83,7 @@ export async function searchView(
 
   await context.ensureSnippetContext(true)
 
-  const enrichedQuery = await enrichSearchContext(query, {
+  const enrichedQuery = await enrichSearchContext(query || {}, {
     user: sdk.users.getUserContextBindings(ctx.user),
   })
 
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 63d3044da90..941c1846cab 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -428,8 +428,9 @@ const builderFilter = (expression: SearchFilter) => {
   return query
 }
 
-export const buildQueryLegacy = (filter: SearchFilter[]) => {
-  //
+export const buildQueryLegacy = (
+  filter?: SearchFilterGroup | SearchFilter[]
+): SearchFilters | undefined => {
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -523,7 +524,6 @@ export const buildQueryLegacy = (filter: SearchFilter[]) => {
         }
       }
     } else if (isLogicalSearchOperator(queryOperator)) {
-      // TODO
     } else if (query[queryOperator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {
         // Transform boolean filters to cope with null.
@@ -548,7 +548,9 @@ export const buildQueryLegacy = (filter: SearchFilter[]) => {
   return query
 }
 
-export const buildQuery = (filter?: SearchFilterGroup | SearchFilter[]) => {
+export const buildQuery = (
+  filter?: SearchFilterGroup | SearchFilter[]
+): SearchFilters | undefined => {
   if (!filter) {
     return
   }

From ce73307f0885e0ff5a146dfbeee96d3de9b845c3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 11 Sep 2024 12:31:11 +0100
Subject: [PATCH 105/455] Updating yarn.lock.

---
 yarn.lock | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 61 insertions(+), 3 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 146fa74d5b5..e3a0f725551 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17798,11 +17798,21 @@ periscopic@^3.1.0:
     estree-walker "^3.0.0"
     is-reference "^3.0.0"
 
+pg-cloudflare@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
+  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
+
 pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
   integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
 
+pg-connection-string@^2.6.4:
+  version "2.6.4"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d"
+  integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==
+
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
@@ -17813,11 +17823,21 @@ pg-pool@^3.6.0:
   resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e"
   integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==
 
+pg-pool@^3.6.2:
+  version "3.6.2"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2"
+  integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==
+
 pg-protocol@*, pg-protocol@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
   integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
 
+pg-protocol@^1.6.1:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3"
+  integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==
+
 pg-types@^2.1.0, pg-types@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
@@ -17842,6 +17862,19 @@ pg@8.10.0:
     pg-types "^2.1.0"
     pgpass "1.x"
 
+pg@^8.12.0:
+  version "8.12.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79"
+  integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==
+  dependencies:
+    pg-connection-string "^2.6.4"
+    pg-pool "^3.6.2"
+    pg-protocol "^1.6.1"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+  optionalDependencies:
+    pg-cloudflare "^1.1.1"
+
 pgpass@1.x:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
@@ -20755,7 +20788,16 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
+"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20846,7 +20888,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20860,6 +20902,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22820,7 +22869,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22838,6 +22887,15 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"

From 956a10e1b44f1bffabea1555c2e63dc3d2828b02 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 11 Sep 2024 13:05:14 +0100
Subject: [PATCH 106/455] Lint

---
 packages/shared-core/src/filters.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 1eb77efde0b..431f6836da3 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -530,6 +530,7 @@ export const buildQueryLegacy = (
         }
       }
     } else if (isLogicalSearchOperator(queryOperator)) {
+      // ignore
     } else if (query[queryOperator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {
         // Transform boolean filters to cope with null.

From 9f0c160bfa0f4eec3af33a86761a12da57dd4f52 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 13:06:07 +0100
Subject: [PATCH 107/455] Add deleted permissions file, update auto layout

---
 .../backend/RoleEditor/Controls.svelte        |  41 +++++--
 .../backend/RoleEditor/RoleEditor.svelte      |  61 ++--------
 .../backend/RoleEditor/RoleNode.svelte        |  13 ++-
 .../backend/RoleEditor/constants.js           |   1 +
 .../components/backend/RoleEditor/layout.js   |  44 +++++++-
 .../server/src/sdk/app/permissions/index.ts   | 105 ++++++++++++++++++
 6 files changed, 198 insertions(+), 67 deletions(-)
 create mode 100644 packages/server/src/sdk/app/permissions/index.ts

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 2824500c472..eb7b5c046c5 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -1,12 +1,14 @@
 <script>
   import { Button, Helpers, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow, Position } from "@xyflow/svelte"
-  import { getContext } from "svelte"
+  import { getContext, tick } from "svelte"
+  import { autoLayout } from "./layout"
+  import { ZoomDuration } from "./constants"
 
-  const { nodes, autoLayout } = getContext("flow")
+  const { nodes, edges } = getContext("flow")
   const flow = useSvelteFlow()
 
-  const addRole = () => {
+  const addRole = async () => {
     nodes.update(state => [
       ...state,
       {
@@ -22,19 +24,42 @@
         position: { x: 0, y: 0 },
       },
     ])
-    autoLayout()
+    await doAutoLayout()
+  }
+
+  const doAutoLayout = async () => {
+    const layout = autoLayout({ nodes: $nodes, edges: $edges })
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+    await tick()
+    flow.fitView({
+      maxZoom: 1,
+      duration: ZoomDuration,
+      includeHiddenNodes: true,
+    })
   }
 </script>
 
 <div class="control top-left">
   <div class="group">
-    <ActionButton icon="Add" quiet on:click={flow.zoomIn} />
-    <ActionButton icon="Remove" quiet on:click={flow.zoomOut} />
+    <ActionButton
+      icon="Add"
+      quiet
+      on:click={() => flow.zoomIn({ duration: ZoomDuration })}
+    />
+    <ActionButton
+      icon="Remove"
+      quiet
+      on:click={() => flow.zoomOut({ duration: ZoomDuration })}
+    />
   </div>
-  <Button secondary on:click={() => flow.fitView({ maxZoom: 1 })}>
+  <Button
+    secondary
+    on:click={() => flow.fitView({ maxZoom: 1, duration: ZoomDuration })}
+  >
     Zoom to fit
   </Button>
-  <Button secondary on:click={autoLayout}>Auto layout</Button>
+  <Button secondary on:click={doAutoLayout}>Auto layout</Button>
 </div>
 <div class="control bottom-right">
   <Button icon="Add" cta on:click={addRole}>Add role</Button>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 0e70d273083..42c18934cfc 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -1,57 +1,20 @@
 <script>
-  import { Heading, Helpers } from "@budibase/bbui"
-  import { derived, writable } from "svelte/store"
+  import { Heading } from "@budibase/bbui"
+  import { writable } from "svelte/store"
   import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
-  import { rolesToLayout, dagreLayout, layoutToRoles } from "./layout"
-  import { onMount, setContext, tick } from "svelte"
+  import { defaultLayout, autoLayout } from "./layout"
+  import { onMount, setContext } from "svelte"
   import Controls from "./Controls.svelte"
-  import { Roles } from "constants/backend"
-  import { derivedMemo } from "@budibase/frontend-core"
 
   const nodes = writable([])
-  const edgeStore = writable([])
-  const enrichedEdges = derived([edgeStore, nodes], ([$edgeStore, $nodes]) => {
-    let additions = []
-    for (let node of $nodes) {
-      // If a certain node does not inherit anything, make it inherit basic
-      if (
-        !$edgeStore.some(x => x.target === node.id) &&
-        node.id !== Roles.BASIC
-      ) {
-        additions.push({
-          id: Helpers.uuid(),
-          source: Roles.BASIC,
-          target: node.id,
-          animated: true,
-        })
-      }
-    }
-    return [...$edgeStore, ...additions]
-  })
-  const edges = {
-    ...edgeStore,
-    subscribe: enrichedEdges.subscribe,
-  }
+  const edges = writable([])
 
-  const roles = derivedMemo([nodes, edges], ([$nodes, $edges]) => {
-    return layoutToRoles({ nodes: $nodes, edges: $edges })
-  })
-  $: console.log("new roles", $roles)
-
-  setContext("flow", {
-    nodes,
-    edges,
-    autoLayout: async () => {
-      const layout = dagreLayout({ nodes: $nodes, edges: $edges })
-      nodes.set(layout.nodes)
-      edges.set(layout.edges)
-    },
-  })
+  setContext("flow", { nodes, edges })
 
   onMount(() => {
-    const layout = dagreLayout(rolesToLayout())
+    const layout = autoLayout(defaultLayout())
     nodes.set(layout.nodes)
     edges.set(layout.edges)
   })
@@ -120,15 +83,5 @@
     /* Edges */
     --xy-edge-stroke: var(--edge-color);
     --xy-edge-stroke-selected: var(--selected-color);
-
-    /* Minimap */
-    /* --xy-minimap-background-color-props: var(--node-background);
-    --xy-minimap-mask-background-color-props: var(--translucent-grey); */
   }
-
-  /* Arrow edge markers */
-  /* .flow :global(.svelte-flow__arrowhead > polyline) {
-    fill: var(--edge-color);
-    stroke: var(--edge-color);
-  } */
 </style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index fd7216af772..a515e80de12 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -12,13 +12,13 @@
   import { Roles } from "constants/backend"
   import { NodeWidth, NodeHeight } from "./constants"
   import { getContext, tick } from "svelte"
-  import { template } from "lodash"
+  import { autoLayout } from "./layout"
 
   export let data
   export let isConnectable
   export let id
 
-  const { autoLayout } = getContext("flow")
+  const { nodes, edges } = getContext("flow")
   const flow = useSvelteFlow()
 
   let anchor
@@ -33,7 +33,7 @@
       nodes: [{ id }],
     })
     await tick()
-    autoLayout()
+    doAutoLayout()
   }
 
   const openPopover = () => {
@@ -48,6 +48,13 @@
       color: tempColor,
     })
   }
+
+  const doAutoLayout = () => {
+    const layout = autoLayout({ nodes: $nodes, edges: $edges })
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+    flow.fitView({ maxZoom: 1, duration: 300 })
+  }
 </script>
 
 <div
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index 5c08561a230..76faf284825 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -1,2 +1,3 @@
 export const NodeWidth = 220
 export const NodeHeight = 66
+export const ZoomDuration = 300
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index dd1550be510..84ac34ec033 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -6,7 +6,8 @@ import { Roles } from "constants/backend"
 import { get } from "svelte/store"
 import { Helpers } from "@budibase/bbui"
 
-export const rolesToLayout = () => {
+// Generates a flow compatible structure of nodes and edges from the current roles
+export const defaultLayout = () => {
   const ignoredRoles = [Roles.PUBLIC]
   const $roles = get(roles)
   const descriptions = {
@@ -59,6 +60,7 @@ export const rolesToLayout = () => {
   }
 }
 
+// Converts the flow structure of ndes and edges back into an array of roles
 export const layoutToRoles = ({ nodes, edges }) => {
   // Clone and wipe existing inheritance
   let newRoles = Helpers.cloneDeep(get(roles)).map(role => {
@@ -92,7 +94,8 @@ export const layoutToRoles = ({ nodes, edges }) => {
   return newRoles
 }
 
-export const dagreLayout = ({ nodes, edges }) => {
+// Updates positions of nodes and edges into a nice graph structure
+const dagreLayout = ({ nodes, edges }) => {
   const dagreGraph = new dagre.graphlib.Graph()
   dagreGraph.setDefaultEdgeLabel(() => ({}))
   dagreGraph.setGraph({
@@ -118,3 +121,40 @@ export const dagreLayout = ({ nodes, edges }) => {
   })
   return { nodes, edges }
 }
+
+// Adds additional edges as needed to the flow structure to ensure compatibility with BB role logic
+const sanitiseLayout = ({ nodes, edges }) => {
+  let additions = []
+
+  for (let node of nodes) {
+    // If a node does not inherit anything, let it inherit basic
+    if (!edges.some(x => x.target === node.id) && node.id !== Roles.BASIC) {
+      additions.push({
+        id: Helpers.uuid(),
+        source: Roles.BASIC,
+        target: node.id,
+        animated: true,
+      })
+    }
+
+    // If a node is not inherited by anything, let it be inherited by admin
+    if (!edges.some(x => x.source === node.id) && node.id !== Roles.ADMIN) {
+      additions.push({
+        id: Helpers.uuid(),
+        source: node.id,
+        target: Roles.ADMIN,
+        animated: true,
+      })
+    }
+  }
+
+  return {
+    nodes,
+    edges: [...edges, ...additions],
+  }
+}
+
+// Automatically lays out the graph, sanitising and enriching the structure
+export const autoLayout = ({ nodes, edges }) => {
+  return dagreLayout(sanitiseLayout({ nodes, edges }))
+}
diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts
new file mode 100644
index 00000000000..88058b508da
--- /dev/null
+++ b/packages/server/src/sdk/app/permissions/index.ts
@@ -0,0 +1,105 @@
+import { db, roles } from "@budibase/backend-core"
+import { features } from "@budibase/pro"
+import {
+  DocumentType,
+  PermissionLevel,
+  PermissionSource,
+  PlanType,
+  VirtualDocumentType,
+} from "@budibase/types"
+import { extractViewInfoFromID, isViewID } from "../../../db/utils"
+import {
+  CURRENTLY_SUPPORTED_LEVELS,
+  getBasePermissions,
+} from "../../../utilities/security"
+import sdk from "../../../sdk"
+import { isV2 } from "../views"
+
+type ResourcePermissions = Record<
+  string,
+  { role: string; type: PermissionSource }
+>
+
+export async function getInheritablePermissions(
+  resourceId: string
+): Promise<ResourcePermissions | undefined> {
+  if (isViewID(resourceId)) {
+    return await getResourcePerms(extractViewInfoFromID(resourceId).tableId)
+  }
+}
+
+export async function getResourcePerms(
+  resourceId: string
+): Promise<ResourcePermissions> {
+  const rolesList = await roles.getAllRoles()
+
+  let permissions: ResourcePermissions = {}
+
+  const permsToInherit = await getInheritablePermissions(resourceId)
+
+  for (let level of CURRENTLY_SUPPORTED_LEVELS) {
+    // update the various roleIds in the resource permissions
+    for (let role of rolesList) {
+      const rolePerms = roles.checkForRoleResourceArray(
+        role.permissions || {},
+        resourceId
+      )
+      if (rolePerms[resourceId]?.indexOf(level) > -1) {
+        permissions[level] = {
+          role: roles.getExternalRoleID(role._id!, role.version),
+          type: PermissionSource.EXPLICIT,
+        }
+      } else if (
+        !permissions[level] &&
+        permsToInherit &&
+        permsToInherit[level]
+      ) {
+        permissions[level] = {
+          role: permsToInherit[level].role,
+          type: PermissionSource.INHERITED,
+        }
+      }
+    }
+  }
+
+  const basePermissions = Object.entries(
+    getBasePermissions(resourceId)
+  ).reduce<ResourcePermissions>((p, [level, role]) => {
+    p[level] = { role, type: PermissionSource.BASE }
+    return p
+  }, {})
+  const result = Object.assign(basePermissions, permissions)
+  return result
+}
+
+export async function getDependantResources(
+  resourceId: string
+): Promise<Record<string, number> | undefined> {
+  if (db.isTableId(resourceId)) {
+    const dependants: Record<string, Set<string>> = {}
+
+    const table = await sdk.tables.getTable(resourceId)
+    const views = Object.values(table.views || {})
+
+    for (const view of views) {
+      if (!isV2(view)) {
+        continue
+      }
+
+      const permissions = await getResourcePerms(view.id)
+      for (const [, roleInfo] of Object.entries(permissions)) {
+        if (roleInfo.type === PermissionSource.INHERITED) {
+          dependants[VirtualDocumentType.VIEW] ??= new Set()
+          dependants[VirtualDocumentType.VIEW].add(view.id)
+        }
+      }
+    }
+
+    return Object.entries(dependants).reduce((p, [type, resources]) => {
+      p[type] = resources.size
+      return p
+    }, {} as Record<string, number>)
+  }
+
+  return
+}

From a5520a973cddb672cfeb806cc918cb00df867d00 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 13:11:41 +0100
Subject: [PATCH 108/455] Use display names and allow descriptions to be edited

---
 .../backend/RoleEditor/Controls.svelte        |  3 ++-
 .../backend/RoleEditor/RoleNode.svelte        | 26 ++++++++++++-------
 .../components/backend/RoleEditor/layout.js   |  2 +-
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index eb7b5c046c5..aa0b0112d8f 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -17,9 +17,10 @@
         targetPosition: Position.Left,
         type: "role",
         data: {
-          label: "New role",
+          displayName: "New role",
           description: "Custom role",
           custom: true,
+          color: "var(--spectrum-global-color-gray-700)",
         },
         position: { x: 0, y: 0 },
       },
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index a515e80de12..aaf228638dc 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -23,7 +23,8 @@
 
   let anchor
   let modal
-  let tempLabel
+  let tempDisplayName
+  let tempDescription
   let tempColor
 
   $: color = data.color || RoleUtils.getRoleColour(id)
@@ -37,14 +38,16 @@
   }
 
   const openPopover = () => {
-    tempLabel = data.label
+    tempDisplayName = data.displayName
+    tempDescription = data.description
     tempColor = color
     modal.show()
   }
 
   const saveChanges = () => {
     flow.updateNodeData(id, {
-      label: tempLabel,
+      displayName: tempDisplayName,
+      description: tempDescription,
       color: tempColor,
     })
   }
@@ -66,8 +69,8 @@
   <div class="color" />
   <div class="content">
     <div class="title">
-      <div class="label">
-        {data.label}
+      <div class="name">
+        {data.displayName}
       </div>
       {#if data.custom}
         <div class="buttons">
@@ -92,14 +95,19 @@
 
 <Modal bind:this={modal}>
   <ModalContent
-    title={`Edit ${data.label}`}
+    title={`Edit ${data.displayName}`}
     confirmText="Save"
     onConfirm={saveChanges}
   >
     <Input
       label="Name"
-      value={tempLabel}
-      on:change={e => (tempLabel = e.detail)}
+      value={tempDisplayName}
+      on:change={e => (tempDisplayName = e.detail)}
+    />
+    <Input
+      label="Description"
+      value={tempDescription}
+      on:change={e => (tempDescription = e.detail)}
     />
     <div>
       <FieldLabel label="Color" />
@@ -160,7 +168,7 @@
   .title :global(.spectrum-Icon) {
     color: var(--spectrum-global-color-gray-600);
   }
-  .label {
+  .name {
     white-space: nowrap;
     text-overflow: ellipsis;
     overflow: hidden;
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index 84ac34ec033..4bb6d1bb056 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -29,7 +29,7 @@ export const defaultLayout = () => {
       targetPosition: Position.Left,
       type: "role",
       data: {
-        label: role.name,
+        displayName: role.displayName || role.name || "",
         description: descriptions[role._id] || "Custom role",
         color: role.color,
         custom: !role._id.match(/[A-Z]+/),

From a81d9c6dd1bb9b63da683c9e171abf737b1a1afa Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 15:42:05 +0100
Subject: [PATCH 109/455] Add display name, color and descriptions to roles.
 Allow row CRUD via new UI

---
 packages/backend-core/src/security/roles.ts   | 44 ++++++++--
 .../backend/RoleEditor/Controls.svelte        | 34 ++++----
 .../backend/RoleEditor/RoleEditor.svelte      |  4 +-
 .../backend/RoleEditor/RoleNode.svelte        | 13 +--
 .../components/backend/RoleEditor/layout.js   | 83 +++++++------------
 .../src/components/common/RoleIcon.svelte     |  5 +-
 .../src/components/common/RoleSelect.svelte   |  2 +-
 .../settings/controls/RoleSelect.svelte       |  3 +-
 .../ScreenList/RoleIndicator.svelte           |  2 +-
 .../_components/AppRoleTableRenderer.svelte   |  8 +-
 packages/builder/src/stores/builder/roles.js  | 27 ++----
 packages/frontend-core/src/utils/roles.js     | 13 ---
 packages/server/src/api/controllers/role.ts   | 19 ++++-
 .../server/src/api/routes/utils/validators.ts |  3 +
 packages/types/src/api/web/role.ts            |  3 +
 packages/types/src/documents/app/role.ts      |  3 +
 16 files changed, 141 insertions(+), 125 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index a64be6b3197..7fb4eefc8ba 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -45,10 +45,22 @@ export class Role implements RoleDoc {
   inherits?: string
   version?: string
   permissions = {}
+  displayName?: string
+  color?: string
+  description?: string
 
-  constructor(id: string, name: string, permissionId: string) {
+  constructor(
+    id: string,
+    displayName: string,
+    description: string,
+    color: string,
+    permissionId: string
+  ) {
     this._id = id
-    this.name = name
+    this.name = id
+    this.displayName = displayName
+    this.color = color
+    this.description = description
     this.permissionId = permissionId
     // version for managing the ID - removing the role_ when responding
     this.version = RoleIDVersion.NAME
@@ -63,21 +75,39 @@ export class Role implements RoleDoc {
 const BUILTIN_ROLES = {
   ADMIN: new Role(
     BUILTIN_IDS.ADMIN,
-    "Admin",
+    "App admin",
+    "Can do everything",
+    "var(--spectrum-global-color-static-red-400)",
     BuiltinPermissionID.ADMIN
   ).addInheritance(BUILTIN_IDS.POWER),
   POWER: new Role(
     BUILTIN_IDS.POWER,
-    "Power",
+    "App power user",
+    "An app user with more access",
+    "var(--spectrum-global-color-static-orange-400)",
     BuiltinPermissionID.POWER
   ).addInheritance(BUILTIN_IDS.BASIC),
   BASIC: new Role(
     BUILTIN_IDS.BASIC,
-    "Basic",
+    "App user",
+    "Any logged in user",
+    "var(--spectrum-global-color-static-green-400)",
     BuiltinPermissionID.WRITE
   ).addInheritance(BUILTIN_IDS.PUBLIC),
-  PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public", BuiltinPermissionID.PUBLIC),
-  BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder", BuiltinPermissionID.ADMIN),
+  PUBLIC: new Role(
+    BUILTIN_IDS.PUBLIC,
+    "Public user",
+    "Accessible to anyone",
+    "var(--spectrum-global-color-static-blue-400)",
+    BuiltinPermissionID.PUBLIC
+  ),
+  BUILDER: new Role(
+    BUILTIN_IDS.BUILDER,
+    "Builder user",
+    "Users that can edit this app",
+    "var(--spectrum-global-color-static-magenta-600)",
+    BuiltinPermissionID.ADMIN
+  ),
 }
 
 export function getBuiltinRoles(): { [key: string]: RoleDoc } {
diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index aa0b0112d8f..94c82dcd65b 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -2,29 +2,28 @@
   import { Button, Helpers, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow, Position } from "@xyflow/svelte"
   import { getContext, tick } from "svelte"
-  import { autoLayout } from "./layout"
+  import { autoLayout, roleToNode } from "./layout"
   import { ZoomDuration } from "./constants"
+  import { getSequentialName } from "helpers/duplicate"
+  import { roles } from "stores/builder"
+  import { Roles } from "constants/backend"
 
   const { nodes, edges } = getContext("flow")
   const flow = useSvelteFlow()
 
   const addRole = async () => {
-    nodes.update(state => [
-      ...state,
-      {
-        id: Helpers.uuid(),
-        sourcePosition: Position.Right,
-        targetPosition: Position.Left,
-        type: "role",
-        data: {
-          displayName: "New role",
-          description: "Custom role",
-          custom: true,
-          color: "var(--spectrum-global-color-gray-700)",
-        },
-        position: { x: 0, y: 0 },
-      },
-    ])
+    const role = {
+      name: Helpers.uuid(),
+      displayName: getSequentialName($nodes, "New role ", {
+        getName: x => x.data.displayName,
+      }),
+      color: "var(--spectrum-global-color-gray-700)",
+      description: "Custom role",
+      permissionId: "write",
+      inherits: Roles.BASIC,
+    }
+    const savedRole = await roles.save(role)
+    nodes.update(state => [...state, roleToNode(savedRole)])
     await doAutoLayout()
   }
 
@@ -36,7 +35,6 @@
     flow.fitView({
       maxZoom: 1,
       duration: ZoomDuration,
-      includeHiddenNodes: true,
     })
   }
 </script>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 42c18934cfc..9904a8877b8 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -4,7 +4,7 @@
   import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
-  import { defaultLayout, autoLayout } from "./layout"
+  import { rolesToNodes, autoLayout } from "./layout"
   import { onMount, setContext } from "svelte"
   import Controls from "./Controls.svelte"
 
@@ -14,7 +14,7 @@
   setContext("flow", { nodes, edges })
 
   onMount(() => {
-    const layout = autoLayout(defaultLayout())
+    const layout = autoLayout(rolesToNodes())
     nodes.set(layout.nodes)
     edges.set(layout.edges)
   })
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index aaf228638dc..42a107ee2e6 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -12,7 +12,8 @@
   import { Roles } from "constants/backend"
   import { NodeWidth, NodeHeight } from "./constants"
   import { getContext, tick } from "svelte"
-  import { autoLayout } from "./layout"
+  import { autoLayout, nodeToRole } from "./layout"
+  import { roles } from "stores/builder"
 
   export let data
   export let isConnectable
@@ -27,9 +28,8 @@
   let tempDescription
   let tempColor
 
-  $: color = data.color || RoleUtils.getRoleColour(id)
-
   const deleteNode = async () => {
+    await roles.delete(nodeToRole({ id, data }))
     flow.deleteElements({
       nodes: [{ id }],
     })
@@ -40,11 +40,12 @@
   const openPopover = () => {
     tempDisplayName = data.displayName
     tempDescription = data.description
-    tempColor = color
+    tempColor = data.color
     modal.show()
   }
 
-  const saveChanges = () => {
+  const saveChanges = async () => {
+    await roles.save(nodeToRole({ id, data }))
     flow.updateNodeData(id, {
       displayName: tempDisplayName,
       description: tempDescription,
@@ -63,7 +64,7 @@
 <div
   class="node"
   class:selected={false}
-  style={`--color:${color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
+  style={`--color:${data.color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
   bind:this={anchor}
 >
   <div class="color" />
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index 4bb6d1bb056..b8a5ae7d5a7 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -6,15 +6,36 @@ import { Roles } from "constants/backend"
 import { get } from "svelte/store"
 import { Helpers } from "@budibase/bbui"
 
+// Converts a role doc into a node structure
+export const roleToNode = role => ({
+  id: role._id,
+  sourcePosition: Position.Right,
+  targetPosition: Position.Left,
+  type: "role",
+  position: { x: 0, y: 0 },
+  data: {
+    displayName: role.displayName || role.name,
+    description: role.description || "Custom role",
+    color: role.color || "var(--spectrum-global-color-static-magenta-400)",
+    custom: !role._id.match(/[A-Z]+/),
+  },
+})
+
+// Converts a node structure back into a role doc
+export const nodeToRole = node => {
+  const role = get(roles).find(x => x._id === node.id)
+  return {
+    ...role,
+    displayName: node.data.displayName,
+    color: node.data.color,
+    description: node.data.description,
+  }
+}
+
 // Generates a flow compatible structure of nodes and edges from the current roles
-export const defaultLayout = () => {
+export const rolesToNodes = () => {
   const ignoredRoles = [Roles.PUBLIC]
   const $roles = get(roles)
-  const descriptions = {
-    [Roles.BASIC]: "Basic user",
-    [Roles.POWER]: "Power user",
-    [Roles.ADMIN]: "Can do everything",
-  }
 
   let nodes = []
   let edges = []
@@ -23,19 +44,11 @@ export const defaultLayout = () => {
     if (ignoredRoles.includes(role._id)) {
       continue
     }
-    nodes.push({
-      id: role._id,
-      sourcePosition: Position.Right,
-      targetPosition: Position.Left,
-      type: "role",
-      data: {
-        displayName: role.displayName || role.name || "",
-        description: descriptions[role._id] || "Custom role",
-        color: role.color,
-        custom: !role._id.match(/[A-Z]+/),
-      },
-    })
 
+    // Add node for this role
+    nodes.push(roleToNode(role))
+
+    // Add edges for this role
     let inherits = []
     if (role.inherits) {
       inherits = Array.isArray(role.inherits) ? role.inherits : [role.inherits]
@@ -60,40 +73,6 @@ export const defaultLayout = () => {
   }
 }
 
-// Converts the flow structure of ndes and edges back into an array of roles
-export const layoutToRoles = ({ nodes, edges }) => {
-  // Clone and wipe existing inheritance
-  let newRoles = Helpers.cloneDeep(get(roles)).map(role => {
-    return { ...role, inherits: [] }
-  })
-
-  // Copy over names and colours
-  for (let node of nodes) {
-    let role = newRoles.find(x => x._id === node.id)
-    if (role) {
-      role.name = node.data.label
-      role.color = node.data.color
-    } else {
-      // New role
-    }
-  }
-
-  // Build inheritance
-  for (let edge of edges) {
-    let role = newRoles.find(x => x._id === edge.target)
-    if (role) {
-      role.inherits.push(edge.source)
-    } else {
-      // New role
-    }
-  }
-
-  // Ensure basic is correct
-  newRoles.find(x => x._id === Roles.BASIC).inherits = [Roles.BASIC]
-
-  return newRoles
-}
-
 // Updates positions of nodes and edges into a nice graph structure
 const dagreLayout = ({ nodes, edges }) => {
   const dagreGraph = new dagre.graphlib.Graph()
diff --git a/packages/builder/src/components/common/RoleIcon.svelte b/packages/builder/src/components/common/RoleIcon.svelte
index 1bd6ba49bcc..567a7c0b5da 100644
--- a/packages/builder/src/components/common/RoleIcon.svelte
+++ b/packages/builder/src/components/common/RoleIcon.svelte
@@ -1,12 +1,15 @@
 <script>
   import { RoleUtils } from "@budibase/frontend-core"
   import { StatusLight } from "@budibase/bbui"
+  import { roles } from "stores/builder"
 
   export let id
   export let size = "M"
   export let disabled = false
 
-  $: color = RoleUtils.getRoleColour(id)
+  $: color =
+    $roles.find(x => x._id === id)?.color ||
+    "var(--spectrum-global-color-static-magenta-400)"
 </script>
 
 <StatusLight square {disabled} {size} {color} />
diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 4605b0c182d..72495fe3c95 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -99,7 +99,7 @@
     if (role._id === Constants.Roles.CREATOR || role._id === RemoveID) {
       return null
     }
-    return RoleUtils.getRoleColour(role._id)
+    return role.color || "var(--spectrum-global-color-static-magenta-400)"
   }
 
   const getIcon = role => {
diff --git a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
index 5b5daac1830..973a239bd8f 100644
--- a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
@@ -14,7 +14,8 @@
   options={$roles}
   getOptionLabel={role => role.name}
   getOptionValue={role => role._id}
-  getOptionColour={role => RoleUtils.getRoleColour(role._id)}
+  getOptionColour={role =>
+    role.color || "var(--spectrum-global-color-static-magenta-400)"}
   {placeholder}
   {error}
 />
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
index 4b7f26709c8..994a56e833c 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
@@ -8,7 +8,7 @@
 
   let showTooltip = false
 
-  $: color = RoleUtils.getRoleColour(roleId)
+  $: color = role.color || "var(--spectrum-global-color-static-magenta-400)"
   $: role = $roles.find(role => role._id === roleId)
   $: tooltip =
     roleId === Roles.PUBLIC
diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
index 0f19bb3e1f9..85d94ba3508 100644
--- a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
@@ -6,8 +6,9 @@
 
   export let value
 
+  $: role = $roles.find(x => x._id === roleId)
+
   const getRoleLabel = roleId => {
-    const role = $roles.find(x => x._id === roleId)
     return roleId === Constants.Roles.CREATOR
       ? capitalise(Constants.Roles.CREATOR.toLowerCase())
       : role?.name || "Custom role"
@@ -17,7 +18,10 @@
 {#if value === Constants.Roles.CREATOR}
   Can edit
 {:else}
-  <StatusLight square color={RoleUtils.getRoleColour(value)}>
+  <StatusLight
+    square
+    color={role?.color || "var(--spectrum-global-color-static-magenta-400)"}
+  >
     Can use as {getRoleLabel(value)}
   </StatusLight>
 {/if}
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index 8f1bfdc177a..a438abac567 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -1,14 +1,6 @@
 import { writable } from "svelte/store"
 import { API } from "api"
 import { RoleUtils } from "@budibase/frontend-core"
-import { Roles } from "constants/backend"
-
-const ROLE_NAMES = {
-  [Roles.ADMIN]: "App admin",
-  [Roles.POWER]: "App power user",
-  [Roles.BASIC]: "App user",
-  [Roles.PUBLIC]: "Public user",
-}
 
 export function createRolesStore() {
   const { subscribe, update, set } = writable([])
@@ -25,16 +17,7 @@ export function createRolesStore() {
 
   const actions = {
     fetch: async () => {
-      let roles = await API.getRoles()
-
-      // Update labels
-      for (let [roleId, name] of Object.entries(ROLE_NAMES)) {
-        const idx = roles.findIndex(x => x._id === roleId)
-        if (idx !== -1) {
-          roles[idx].name = name
-        }
-      }
-
+      const roles = await API.getRoles()
       setRoles(roles)
     },
     fetchByAppId: async appId => {
@@ -51,7 +34,13 @@ export function createRolesStore() {
     save: async role => {
       const savedRole = await API.saveRole(role)
       await actions.fetch()
-      return savedRole
+
+      // When saving a role we get back an _id prefixed by role_, but the API does not want this
+      // in future requests
+      return {
+        ...savedRole,
+        _id: savedRole._id.replace("role_", ""),
+      }
     },
   }
 
diff --git a/packages/frontend-core/src/utils/roles.js b/packages/frontend-core/src/utils/roles.js
index 1ae9d3ac142..913d452e7cd 100644
--- a/packages/frontend-core/src/utils/roles.js
+++ b/packages/frontend-core/src/utils/roles.js
@@ -7,20 +7,7 @@ const RolePriorities = {
   [Roles.BASIC]: 2,
   [Roles.PUBLIC]: 1,
 }
-const RoleColours = {
-  [Roles.ADMIN]: "var(--spectrum-global-color-static-red-400)",
-  [Roles.CREATOR]: "var(--spectrum-global-color-static-magenta-600)",
-  [Roles.POWER]: "var(--spectrum-global-color-static-orange-400)",
-  [Roles.BASIC]: "var(--spectrum-global-color-static-green-400)",
-  [Roles.PUBLIC]: "var(--spectrum-global-color-static-blue-400)",
-}
 
 export const getRolePriority = role => {
   return RolePriorities[role] ?? 0
 }
-
-export const getRoleColour = roleId => {
-  return (
-    RoleColours[roleId] ?? "var(--spectrum-global-color-static-magenta-400)"
-  )
-}
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 3398c8102ce..9c7fb2859ab 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -62,7 +62,16 @@ export async function find(ctx: UserCtx<void, FindRoleResponse>) {
 
 export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   const db = context.getAppDB()
-  let { _id, name, inherits, permissionId, version } = ctx.request.body
+  let {
+    _id,
+    name,
+    displayName,
+    description,
+    color,
+    inherits,
+    permissionId,
+    version,
+  } = ctx.request.body
   let isCreate = false
   const isNewVersion = version === roles.RoleIDVersion.NAME
 
@@ -88,7 +97,13 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     ctx.throw(400, "Cannot change custom role name")
   }
 
-  const role = new roles.Role(_id, name, permissionId).addInheritance(inherits)
+  const role = new roles.Role(
+    _id,
+    displayName || name,
+    description || "Custom role",
+    color || "var(--spectrum-global-color-static-magenta-400)",
+    permissionId
+  ).addInheritance(inherits)
   if (ctx.request.body._rev) {
     role._rev = ctx.request.body._rev
   }
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index 5e2a585b4af..dd943f12825 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -208,6 +208,9 @@ export function roleValidator() {
       name: Joi.string()
         .regex(/^[a-zA-Z0-9_]*$/)
         .required(),
+      displayName: Joi.string().optional(),
+      color: Joi.string().optional(),
+      description: Joi.string().optional(),
       // this is the base permission ID (for now a built in)
       permissionId: Joi.string()
         .valid(...Object.values(permissions.BuiltinPermissionID))
diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts
index c37dee60e0f..7782c332156 100644
--- a/packages/types/src/api/web/role.ts
+++ b/packages/types/src/api/web/role.ts
@@ -4,6 +4,9 @@ export interface SaveRoleRequest {
   _id?: string
   _rev?: string
   name: string
+  displayName?: string
+  color?: string
+  description?: string
   inherits: string
   permissionId: string
   version: string
diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts
index f32ba810b00..0169da90c2a 100644
--- a/packages/types/src/documents/app/role.ts
+++ b/packages/types/src/documents/app/role.ts
@@ -6,4 +6,7 @@ export interface Role extends Document {
   permissions: { [key: string]: string[] }
   version?: string
   name: string
+  displayName?: string
+  color?: string
+  description?: string
 }

From de7604f8fd786176ea66b1bb94d09cfbedcea46b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 15:55:48 +0100
Subject: [PATCH 110/455] Fix role CRUD

---
 .../builder/src/components/backend/RoleEditor/RoleNode.svelte | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 42a107ee2e6..d7db4b72ba9 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -29,12 +29,12 @@
   let tempColor
 
   const deleteNode = async () => {
-    await roles.delete(nodeToRole({ id, data }))
     flow.deleteElements({
       nodes: [{ id }],
     })
     await tick()
     doAutoLayout()
+    await roles.delete(nodeToRole({ id, data }))
   }
 
   const openPopover = () => {
@@ -45,12 +45,12 @@
   }
 
   const saveChanges = async () => {
-    await roles.save(nodeToRole({ id, data }))
     flow.updateNodeData(id, {
       displayName: tempDisplayName,
       description: tempDescription,
       color: tempColor,
     })
+    await roles.save(nodeToRole({ id, data }))
   }
 
   const doAutoLayout = () => {

From 0fd38927e2311d9581a1be749244be3842ee6e42 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 15:56:04 +0100
Subject: [PATCH 111/455] Remove log

---
 .../builder/src/components/backend/RoleEditor/RoleEditor.svelte  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 9904a8877b8..783dc5a108b 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -36,7 +36,6 @@
     proOptions={{ hideAttribution: true }}
     fitViewOptions={{ maxZoom: 1 }}
     defaultEdgeOptions={{ type: "bezier", animated: true }}
-    on:nodeclick={event => console.log("on node click", event.detail.node)}
   >
     <Background variant={BackgroundVariant.Dots} />
     <Controls />

From e47d25cb48d739660dce34cf5646ab40afdbfd72 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 16:20:23 +0100
Subject: [PATCH 112/455] Fix role updating and add custom role type to grid

---
 .../backend/RoleEditor/RoleNode.svelte        | 12 ++---
 .../data/table/[tableId]/index.svelte         | 15 ++++---
 .../src/components/grid/cells/DataCell.svelte |  2 +-
 .../src/components/grid/cells/RoleCell.svelte | 44 +++++++++++++++++++
 .../src/components/grid/lib/renderers.js      |  4 ++
 .../src/components/grid/stores/columns.js     |  1 +
 6 files changed, 67 insertions(+), 11 deletions(-)
 create mode 100644 packages/frontend-core/src/components/grid/cells/RoleCell.svelte

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index d7db4b72ba9..eda225b604f 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -45,12 +45,13 @@
   }
 
   const saveChanges = async () => {
-    flow.updateNodeData(id, {
+    const newData = {
       displayName: tempDisplayName,
       description: tempDescription,
       color: tempColor,
-    })
-    await roles.save(nodeToRole({ id, data }))
+    }
+    flow.updateNodeData(id, newData)
+    await roles.save(nodeToRole({ id, data: newData }))
   }
 
   const doAutoLayout = () => {
@@ -81,7 +82,7 @@
       {/if}
     </div>
     {#if data.description}
-      <div class="description">
+      <div class="description" title={data.description}>
         {data.description}
       </div>
     {/if}
@@ -169,7 +170,8 @@
   .title :global(.spectrum-Icon) {
     color: var(--spectrum-global-color-gray-600);
   }
-  .name {
+  .name,
+  .description {
     white-space: nowrap;
     text-overflow: ellipsis;
     overflow: hidden;
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index b9b58cbfce8..93ec3d07648 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -6,6 +6,7 @@
     integrations,
     appStore,
     rowActions,
+    roles,
   } from "stores/builder"
   import { themeStore, admin } from "stores/portal"
   import { TableNames } from "constants"
@@ -26,16 +27,20 @@
   import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
   import { DB_TYPE_EXTERNAL } from "constants/backend"
 
-  const userSchemaOverrides = {
+  let generateButton
+
+  $: userSchemaOverrides = {
     firstName: { displayName: "First name", disabled: true },
     lastName: { displayName: "Last name", disabled: true },
     email: { displayName: "Email", disabled: true },
-    roleId: { displayName: "Role", disabled: true },
     status: { displayName: "Status", disabled: true },
+    roleId: {
+      displayName: "Role",
+      type: "role",
+      disabled: true,
+      roles: $roles,
+    },
   }
-
-  let generateButton
-
   $: autoColumnStatus = verifyAutocolumns($tables?.selected)
   $: duplicates = Object.values(autoColumnStatus).reduce((acc, status) => {
     if (status.length > 1) {
diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
index 5d5f06872d0..cd6b61af800 100644
--- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
@@ -66,7 +66,7 @@
     focus: () => api?.focus?.(),
     blur: () => api?.blur?.(),
     isActive: () => api?.isActive?.() ?? false,
-    onKeyDown: (...params) => api?.onKeyDown(...params),
+    onKeyDown: (...params) => api?.onKeyDown?.(...params),
     isReadonly: () => readonly,
     getType: () => column.schema.type,
     getValue: () => row[column.name],
diff --git a/packages/frontend-core/src/components/grid/cells/RoleCell.svelte b/packages/frontend-core/src/components/grid/cells/RoleCell.svelte
new file mode 100644
index 00000000000..b638d0e0e3f
--- /dev/null
+++ b/packages/frontend-core/src/components/grid/cells/RoleCell.svelte
@@ -0,0 +1,44 @@
+<script>
+  import { StatusLight } from "@budibase/bbui"
+
+  export let value
+  export let schema
+
+  $: role = schema.roles?.find(x => x._id === value)
+</script>
+
+<div class="role-cell">
+  <div class="light">
+    <StatusLight
+      square
+      size="L"
+      color={role?.color || "var(--spectrum-global-color-magenta-400)"}
+    />
+  </div>
+  <div class="value">
+    {role?.displayName || role?.name || value}
+  </div>
+</div>
+
+<style>
+  .role-cell {
+    flex: 1 1 auto;
+    padding: var(--cell-padding);
+    align-self: stretch;
+    display: flex;
+    align-items: flex-start;
+    overflow: hidden;
+    gap: var(--cell-padding);
+  }
+  .light {
+    height: 20px;
+    display: grid;
+    place-items: center;
+  }
+  .value {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    line-height: 20px;
+  }
+</style>
diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index 2e22ee18711..607240b7a2c 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -15,6 +15,7 @@ import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
 import BBReferenceCell from "../cells/BBReferenceCell.svelte"
 import SignatureCell from "../cells/SignatureCell.svelte"
 import BBReferenceSingleCell from "../cells/BBReferenceSingleCell.svelte"
+import RoleCell from "../cells/RoleCell.svelte"
 
 const TypeComponentMap = {
   [FieldType.STRING]: TextCell,
@@ -33,6 +34,9 @@ const TypeComponentMap = {
   [FieldType.JSON]: JSONCell,
   [FieldType.BB_REFERENCE]: BBReferenceCell,
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
+
+  // Custom types for UI only
+  role: RoleCell,
 }
 export const getCellRenderer = column => {
   return TypeComponentMap[column?.schema?.type] || TextCell
diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 0073754a5d3..61d1f41fc8a 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -145,6 +145,7 @@ export const initialise = context => {
             readonly: fieldSchema.readonly,
             order: fieldSchema.order ?? oldColumn?.order,
             conditions: fieldSchema.conditions,
+            enrichValue: fieldSchema.enrichValue,
           }
           // Override a few properties for primary display
           if (field === primaryDisplay) {

From 7e6f1407fe142cc26808fd5c41bc71706f5570f1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 16:30:17 +0100
Subject: [PATCH 113/455] Ensure roles always have new metadta and update
 access popover

---
 packages/bbui/src/List/ListItem.svelte        |  5 +-
 .../buttons/ManageAccessButton.svelte         | 57 +++++++++----------
 .../components/backend/RoleEditor/layout.js   |  6 +-
 packages/builder/src/stores/builder/roles.js  | 18 ++++--
 4 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index abcdf2cc6e7..64f646cae8e 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -1,5 +1,6 @@
 <script>
   import Icon from "../Icon/Icon.svelte"
+  import StatusLight from "../StatusLight/StatusLight.svelte"
 
   export let icon = null
   export let iconColor = null
@@ -19,7 +20,9 @@
   class:selected
 >
   <div class="left">
-    {#if icon}
+    {#if icon === "StatusLight"}
+      <StatusLight square size="L" color={iconColor} />
+    {:else if icon}
       <Icon name={icon} color={iconColor} />
     {/if}
     <div class="list-item__text">
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 5c8ccddf592..4e52a606385 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -30,12 +30,21 @@
   $: roleMismatch = checkRoleMismatch(permissions)
   $: selectedRole = roleMismatch ? null : permissions?.[0]?.value
   $: readableRole = selectedRole
-    ? $roles.find(x => x._id === selectedRole)?.name
+    ? $roles.find(x => x._id === selectedRole)?.displayName
     : null
   $: buttonLabel = readableRole ? `Access: ${readableRole}` : "Access"
-  $: customRoles = $roles.filter(x => !builtins.includes(x._id))
   $: highlight = roleMismatch || selectedRole === Roles.PUBLIC
 
+  $: builtInRoles = builtins.map(roleId => $roles.find(x => x._id === roleId))
+  $: customRoles = $roles
+    .filter(x => !builtins.includes(x._id))
+    .slice()
+    .toSorted((a, b) => {
+      const aName = a.displayName || a.name
+      const bName = b.displayName || b.name
+      return aName < bName ? -1 : 1
+    })
+
   const fetchPermissions = async id => {
     const res = await permissionsStore.forResourceDetailed(id)
     permissions = Object.entries(res?.permissions || {}).map(([perm, info]) => {
@@ -149,39 +158,25 @@
   {/if}
 
   <List>
-    <ListItem
-      title="App admin"
-      subtitle="Only app admins can access this data"
-      hoverable
-      selected={selectedRole === Roles.ADMIN}
-      on:click={() => changePermission(Roles.ADMIN)}
-    />
-    <ListItem
-      title="App power user"
-      subtitle="Only app power users can access this data"
-      hoverable
-      selected={selectedRole === Roles.POWER}
-      on:click={() => changePermission(Roles.POWER)}
-    />
-    <ListItem
-      title="App user"
-      subtitle="Only logged in users can access this data"
-      hoverable
-      selected={selectedRole === Roles.BASIC}
-      on:click={() => changePermission(Roles.BASIC)}
-    />
-    <ListItem
-      title="Public user"
-      subtitle="Users are not required to log in to access this data"
-      hoverable
-      selected={selectedRole === Roles.PUBLIC}
-      on:click={() => changePermission(Roles.PUBLIC)}
-    />
+    {#each builtInRoles as role}
+      <ListItem
+        title={role.displayName}
+        subtitle={role.description}
+        hoverable
+        selected={selectedRole === role._id}
+        icon="StatusLight"
+        iconColor={role.color}
+        on:click={() => changePermission(role._id)}
+      />
+    {/each}
     {#each customRoles as role}
       <ListItem
-        title={role.name}
+        title={role.displayName}
+        subtitle={role.description}
         hoverable
         selected={selectedRole === role._id}
+        icon="StatusLight"
+        iconColor={role.color}
         on:click={() => changePermission(role._id)}
       />
     {/each}
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index b8a5ae7d5a7..5dae9a3b5c3 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -14,9 +14,9 @@ export const roleToNode = role => ({
   type: "role",
   position: { x: 0, y: 0 },
   data: {
-    displayName: role.displayName || role.name,
-    description: role.description || "Custom role",
-    color: role.color || "var(--spectrum-global-color-static-magenta-400)",
+    displayName: role.displayName,
+    description: role.description,
+    color: role.color,
     custom: !role._id.match(/[A-Z]+/),
   },
 })
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index a438abac567..bee0bd9372d 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -1,12 +1,22 @@
-import { writable } from "svelte/store"
+import { derived, writable } from "svelte/store"
 import { API } from "api"
 import { RoleUtils } from "@budibase/frontend-core"
 
 export function createRolesStore() {
-  const { subscribe, update, set } = writable([])
+  const store = writable([])
+  const enriched = derived(store, $store => {
+    return $store.map(role => ({
+      ...role,
+
+      // Ensure we have new metadata for all roles
+      displayName: role.displayName || role.name,
+      color: role.color || "var(--spectrum-global-color-magenta-400)",
+      description: role.description || "Custom role",
+    }))
+  })
 
   function setRoles(roles) {
-    set(
+    store.set(
       roles.sort((a, b) => {
         const priorityA = RoleUtils.getRolePriority(a._id)
         const priorityB = RoleUtils.getRolePriority(b._id)
@@ -45,7 +55,7 @@ export function createRolesStore() {
   }
 
   return {
-    subscribe,
+    subscribe: enriched.subscribe,
     ...actions,
   }
 }

From aa75fca3775231815bf26ba53afdc25a7edd6abd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 16:33:19 +0100
Subject: [PATCH 114/455] Update user side panel to account for new role
 metadata

---
 packages/builder/src/components/common/RoleSelect.svelte | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 72495fe3c95..1b4c24b706a 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -48,7 +48,8 @@
       let options = roles
         .filter(role => allowedRoles.includes(role._id))
         .map(role => ({
-          name: enrichLabel(role.name),
+          color: role.color,
+          name: enrichLabel(role.displayName),
           _id: role._id,
         }))
       if (allowedRoles.includes(Constants.Roles.CREATOR)) {
@@ -63,7 +64,8 @@
 
     // Allow all core roles
     let options = roles.map(role => ({
-      name: enrichLabel(role.name),
+      color: role.color,
+      name: enrichLabel(role.displayName),
       _id: role._id,
     }))
 

From dc0f93e314de76404d0e2c7048b7ed8ff9e86052 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 16:36:03 +0100
Subject: [PATCH 115/455] Update design section to use new role metadata

---
 .../components/design/settings/controls/RoleSelect.svelte   | 6 ++----
 .../[screenId]/_components/ScreenList/RoleIndicator.svelte  | 3 +--
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
index 973a239bd8f..773abd69b9e 100644
--- a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
@@ -1,7 +1,6 @@
 <script>
   import { Select } from "@budibase/bbui"
   import { roles } from "stores/builder"
-  import { RoleUtils } from "@budibase/frontend-core"
 
   export let value
   export let error
@@ -12,10 +11,9 @@
   bind:value
   on:change
   options={$roles}
-  getOptionLabel={role => role.name}
+  getOptionLabel={role => role.displayName}
   getOptionValue={role => role._id}
-  getOptionColour={role =>
-    role.color || "var(--spectrum-global-color-static-magenta-400)"}
+  getOptionColour={role => role.color}
   {placeholder}
   {error}
 />
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
index 994a56e833c..4ee5d5dec58 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
@@ -1,5 +1,4 @@
 <script>
-  import { RoleUtils } from "@budibase/frontend-core"
   import { Tooltip, StatusLight } from "@budibase/bbui"
   import { roles } from "stores/builder"
   import { Roles } from "constants/backend"
@@ -13,7 +12,7 @@
   $: tooltip =
     roleId === Roles.PUBLIC
       ? "Open to the public"
-      : `Requires ${role?.name} access`
+      : `Requires ${role?.displayName} access`
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->

From 641982948490f28ff6468d7b208df86ddb1121ea Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 16:51:36 +0100
Subject: [PATCH 116/455] Update other usages of roles to use new metadata

---
 .../buttons/ManageAccessButton.svelte         |   7 -
 .../DataTable/modals/CreateEditUser.svelte    |   2 +-
 .../backend/DataTable/modals/EditRoles.svelte | 177 ------------------
 packages/builder/src/dataBinding.js           |   4 +-
 4 files changed, 3 insertions(+), 187 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 4e52a606385..fe24f47cf1b 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -185,13 +185,6 @@
   {#if dependantsInfoMessage}
     <InfoDisplay info body={dependantsInfoMessage} />
   {/if}
-
-  <div class="buttons">
-    <EditRolesButton
-      on:show={() => (showPopover = false)}
-      on:hide={() => (showPopover = true)}
-    />
-  </div>
 </DetailPopover>
 
 <style>
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte
index a9990d0c2b3..befe5ec4846 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte
@@ -125,7 +125,7 @@
     label="Role"
     bind:value={row.roleId}
     options={$roles}
-    getOptionLabel={role => role.name}
+    getOptionLabel={role => role.displayName}
     getOptionValue={role => role._id}
     disabled={!creating}
   />
diff --git a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte
deleted file mode 100644
index e46a2aa3dc7..00000000000
--- a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte
+++ /dev/null
@@ -1,177 +0,0 @@
-<script>
-  import {
-    keepOpen,
-    ModalContent,
-    Select,
-    Input,
-    Button,
-    notifications,
-  } from "@budibase/bbui"
-  import { onMount } from "svelte"
-  import { API } from "api"
-  import ErrorsBox from "components/common/ErrorsBox.svelte"
-  import { roles } from "stores/builder"
-
-  const BASE_ROLE = { _id: "", inherits: "BASIC", permissionId: "write" }
-
-  let basePermissions = []
-  let selectedRole = BASE_ROLE
-  let errors = []
-  let builtInRoles = ["App admin", "App power user", "App user", "Public user"]
-  let validRegex = /^[a-zA-Z0-9_]*$/
-  // Don't allow editing of public role
-  $: editableRoles = $roles.filter(role => role._id !== "PUBLIC")
-  $: selectedRoleId = selectedRole._id
-  $: otherRoles = editableRoles.filter(role => role._id !== selectedRoleId)
-  $: isCreating = selectedRoleId == null || selectedRoleId === ""
-
-  $: roleNameError = getRoleNameError(selectedRole.name)
-
-  $: valid =
-    selectedRole.name &&
-    selectedRole.inherits &&
-    selectedRole.permissionId &&
-    !builtInRoles.includes(selectedRole.name)
-
-  $: shouldDisableRoleInput =
-    builtInRoles.includes(selectedRole.name) &&
-    selectedRole.name?.toLowerCase() === selectedRoleId?.toLowerCase()
-
-  const fetchBasePermissions = async () => {
-    try {
-      basePermissions = await API.getBasePermissions()
-    } catch (error) {
-      notifications.error("Error fetching base permission options")
-      basePermissions = []
-    }
-  }
-
-  // Changes the selected role
-  const changeRole = event => {
-    const id = event?.detail
-    const role = $roles.find(role => role._id === id)
-    if (role) {
-      selectedRole = {
-        ...role,
-        inherits: role.inherits ?? "",
-        permissionId: role.permissionId ?? "",
-      }
-    } else {
-      selectedRole = BASE_ROLE
-    }
-    errors = []
-  }
-
-  // Saves or creates the selected role
-  const saveRole = async () => {
-    errors = []
-
-    // Clean up empty strings
-    const keys = ["_id", "inherits", "permissionId"]
-    keys.forEach(key => {
-      if (selectedRole[key] === "") {
-        delete selectedRole[key]
-      }
-    })
-
-    // Validation
-    if (!selectedRole.name || selectedRole.name.trim() === "") {
-      errors.push({ message: "Please enter a role name" })
-    }
-    if (!selectedRole.permissionId) {
-      errors.push({ message: "Please choose permissions" })
-    }
-    if (errors.length) {
-      return keepOpen
-    }
-
-    // Save/create the role
-    try {
-      await roles.save(selectedRole)
-      notifications.success("Role saved successfully")
-    } catch (error) {
-      notifications.error(`Error saving role - ${error.message}`)
-      return keepOpen
-    }
-  }
-
-  // Deletes the selected role
-  const deleteRole = async () => {
-    try {
-      await roles.delete(selectedRole)
-      changeRole()
-      notifications.success("Role deleted successfully")
-    } catch (error) {
-      notifications.error(`Error deleting role - ${error.message}`)
-      return false
-    }
-  }
-
-  const getRoleNameError = name => {
-    if (builtInRoles.includes(name)) {
-      return null
-    }
-    const hasUniqueRoleName = !otherRoles
-      ?.map(role => role.name)
-      ?.includes(name)
-    const invalidRoleName = !validRegex.test(name)
-    if (!hasUniqueRoleName) {
-      return "Select a unique role name."
-    } else if (invalidRoleName) {
-      return "Please enter a role name consisting of only alphanumeric symbols and underscores"
-    }
-  }
-
-  onMount(fetchBasePermissions)
-</script>
-
-<ModalContent
-  title="Edit Roles"
-  confirmText={isCreating ? "Create" : "Save"}
-  onConfirm={saveRole}
-  disabled={!valid || roleNameError}
->
-  {#if errors.length}
-    <ErrorsBox {errors} />
-  {/if}
-  <Select
-    thin
-    secondary
-    label="Role"
-    value={selectedRoleId}
-    on:change={changeRole}
-    options={editableRoles}
-    placeholder="Create new role"
-    getOptionValue={role => role._id}
-    getOptionLabel={role => role.name}
-  />
-  {#if selectedRole}
-    <Input
-      label="Name"
-      bind:value={selectedRole.name}
-      disabled={!!selectedRoleId}
-      error={roleNameError}
-    />
-    <Select
-      label="Inherits Role"
-      bind:value={selectedRole.inherits}
-      options={selectedRole._id === "BASIC" ? $roles : otherRoles}
-      getOptionValue={role => role._id}
-      getOptionLabel={role => role.name}
-      disabled={shouldDisableRoleInput}
-    />
-    <Select
-      label="Base Permissions"
-      bind:value={selectedRole.permissionId}
-      options={basePermissions}
-      getOptionValue={x => x._id}
-      getOptionLabel={x => x.name}
-      disabled={shouldDisableRoleInput}
-    />
-  {/if}
-  <div slot="footer">
-    {#if !isCreating && !builtInRoles.includes(selectedRole.name)}
-      <Button warning on:click={deleteRole}>Delete</Button>
-    {/if}
-  </div>
-</ModalContent>
diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js
index 3eefb373ca5..09ff8e7a812 100644
--- a/packages/builder/src/dataBinding.js
+++ b/packages/builder/src/dataBinding.js
@@ -725,10 +725,10 @@ const getRoleBindings = () => {
     return {
       type: "context",
       runtimeBinding: `'${role._id}'`,
-      readableBinding: `Role.${role.name}`,
+      readableBinding: `Role.${role.displayName}`,
       category: "Role",
       icon: "UserGroup",
-      display: { type: "string", name: role.name },
+      display: { type: "string", name: role.displayName },
     }
   })
 }

From 8a7af78c48820633d26fe93fa4d94688661b5551 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 16:52:20 +0100
Subject: [PATCH 117/455] Remove legacy stuff

---
 .../DataTable/buttons/EditRolesButton.svelte        | 13 -------------
 .../DataTable/buttons/ManageAccessButton.svelte     |  1 -
 2 files changed, 14 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
deleted file mode 100644
index bf9606ff973..00000000000
--- a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte
+++ /dev/null
@@ -1,13 +0,0 @@
-<script>
-  import { Button, Modal } from "@budibase/bbui"
-  import EditRolesModal from "../modals/EditRoles.svelte"
-
-  let modal
-</script>
-
-<Button secondary icon="UsersLock" on:click on:click={modal.show}>
-  Edit roles
-</Button>
-<Modal bind:this={modal} on:show on:hide>
-  <EditRolesModal />
-</Modal>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index fe24f47cf1b..1725740a7c0 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -10,7 +10,6 @@
   } from "@budibase/bbui"
   import { permissions as permissionsStore, roles } from "stores/builder"
   import DetailPopover from "components/common/DetailPopover.svelte"
-  import EditRolesButton from "./EditRolesButton.svelte"
   import { PermissionSource } from "@budibase/types"
   import { capitalise } from "helpers"
   import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"

From f6852d5331c54f97b5331528b3456bf87bfbdd2e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 17:00:41 +0100
Subject: [PATCH 118/455] Update some other portal components to use new role
 metadata

---
 .../_components/AppRoleTableRenderer.svelte   |  6 +-
 .../_components/RolesTagsTableRenderer.svelte |  9 ---
 .../_components/TagsTableRenderer.svelte      | 35 ---------
 .../users/_components/UpdateRolesModal.svelte | 74 -------------------
 4 files changed, 3 insertions(+), 121 deletions(-)
 delete mode 100644 packages/builder/src/pages/builder/portal/users/users/_components/RolesTagsTableRenderer.svelte
 delete mode 100644 packages/builder/src/pages/builder/portal/users/users/_components/TagsTableRenderer.svelte
 delete mode 100644 packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte

diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
index 85d94ba3508..e63ee2493ed 100644
--- a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
@@ -1,17 +1,17 @@
 <script>
   import { StatusLight } from "@budibase/bbui"
-  import { RoleUtils, Constants } from "@budibase/frontend-core"
+  import { Constants } from "@budibase/frontend-core"
   import { roles } from "stores/builder"
   import { capitalise } from "helpers"
 
   export let value
 
-  $: role = $roles.find(x => x._id === roleId)
+  $: role = $roles.find(x => x._id === value)
 
   const getRoleLabel = roleId => {
     return roleId === Constants.Roles.CREATOR
       ? capitalise(Constants.Roles.CREATOR.toLowerCase())
-      : role?.name || "Custom role"
+      : role?.displayName || role?.name || "Custom role"
   }
 </script>
 
diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/RolesTagsTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/RolesTagsTableRenderer.svelte
deleted file mode 100644
index 402344dedd3..00000000000
--- a/packages/builder/src/pages/builder/portal/users/users/_components/RolesTagsTableRenderer.svelte
+++ /dev/null
@@ -1,9 +0,0 @@
-<script>
-  import TagsTableRenderer from "./TagsTableRenderer.svelte"
-
-  export let value
-
-  $: roles = value?.filter(role => role != null).map(role => role.name) ?? []
-</script>
-
-<TagsTableRenderer value={roles} />
diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/TagsTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/TagsTableRenderer.svelte
deleted file mode 100644
index 8b2fd3c522f..00000000000
--- a/packages/builder/src/pages/builder/portal/users/users/_components/TagsTableRenderer.svelte
+++ /dev/null
@@ -1,35 +0,0 @@
-<script>
-  import { Tag, Tags } from "@budibase/bbui"
-
-  export let value
-
-  const displayLimit = 5
-
-  $: values = value?.filter(value => value != null) ?? []
-  $: tags = values.slice(0, displayLimit)
-  $: leftover = values.length - tags.length
-</script>
-
-<div class="tag-renderer">
-  <Tags>
-    {#each tags as tag}
-      <Tag>
-        {tag}
-      </Tag>
-    {/each}
-    {#if leftover}
-      <Tag>+{leftover} more</Tag>
-    {/if}
-  </Tags>
-</div>
-
-<style>
-  .tag-renderer :global(.spectrum-Tags-item:hover) {
-    color: var(--spectrum-alias-label-text-color);
-    border-color: var(--spectrum-alias-border-color-darker-default);
-    cursor: pointer;
-  }
-  .tag-renderer :global(.spectrum-Tags-itemLabel) {
-    cursor: pointer;
-  }
-</style>
diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte
deleted file mode 100644
index 9ad41ad6528..00000000000
--- a/packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte
+++ /dev/null
@@ -1,74 +0,0 @@
-<script>
-  import { createEventDispatcher } from "svelte"
-  import { Body, Select, ModalContent, notifications } from "@budibase/bbui"
-  import { users } from "stores/portal"
-  import { sdk } from "@budibase/shared-core"
-
-  export let app
-  export let user
-
-  const NO_ACCESS = "NO_ACCESS"
-
-  const dispatch = createEventDispatcher()
-
-  const roles = app.roles
-  let options = roles
-    .filter(role => role._id !== "PUBLIC")
-    .map(role => ({ value: role._id, label: role.name }))
-
-  if (!sdk.users.isBuilder(user, app?.appId)) {
-    options.push({ value: NO_ACCESS, label: "No Access" })
-  }
-  let selectedRole = user?.roles?.[app?._id]
-
-  async function updateUserRoles() {
-    try {
-      if (selectedRole === NO_ACCESS) {
-        // Remove the user role
-        const filteredRoles = { ...user.roles }
-        delete filteredRoles[app?._id]
-        await users.save({
-          ...user,
-          roles: {
-            ...filteredRoles,
-          },
-        })
-      } else {
-        // Add the user role
-        await users.save({
-          ...user,
-          roles: {
-            ...user.roles,
-            [app._id]: selectedRole,
-          },
-        })
-      }
-      notifications.success("Role updated")
-      dispatch("update")
-    } catch (error) {
-      notifications.error("Failed to update role")
-    }
-  }
-</script>
-
-<ModalContent
-  onConfirm={updateUserRoles}
-  title="Update App Role"
-  confirmText="Update role"
-  cancelText="Cancel"
-  size="M"
-  showCloseIcon={false}
->
-  <Body>
-    Update {user.email}'s role for <strong>{app.name}</strong>.
-  </Body>
-  <Select
-    placeholder={null}
-    bind:value={selectedRole}
-    on:change
-    {options}
-    label="Role"
-    getOptionLabel={role => role.label}
-    getOptionValue={role => role.value}
-  />
-</ModalContent>

From bc03f1e4730c3eba4f60a83daf6ed94f8e5927b7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 11 Sep 2024 18:06:05 +0100
Subject: [PATCH 119/455] Fixing some role typing issues, as well as fixing an
 issue with the validator not allowing the structure that roles are expected
 to have.

---
 packages/backend-core/src/security/roles.ts     |  6 +++---
 .../server/src/api/controllers/permission.ts    |  6 ++++--
 packages/server/src/api/controllers/role.ts     | 17 +++++++++++------
 .../server/src/api/routes/utils/validators.ts   | 10 ++++++++--
 .../server/src/sdk/app/permissions/index.ts     |  2 +-
 packages/types/src/documents/app/role.ts        |  3 ++-
 packages/types/src/documents/global/user.ts     |  5 ++---
 7 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 7fb4eefc8ba..097d6e84da7 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -44,7 +44,7 @@ export class Role implements RoleDoc {
   permissionId: string
   inherits?: string
   version?: string
-  permissions = {}
+  permissions: Record<string, PermissionLevel[]> = {}
   displayName?: string
   color?: string
   description?: string
@@ -274,9 +274,9 @@ export async function getUserRoleHierarchy(
 // some templates/older apps will use a simple string instead of array for roles
 // convert the string to an array using the theory that write is higher than read
 export function checkForRoleResourceArray(
-  rolePerms: { [key: string]: string[] },
+  rolePerms: Record<string, PermissionLevel[]>,
   resourceId: string
-) {
+): Record<string, PermissionLevel[]> {
   if (rolePerms && !Array.isArray(rolePerms[resourceId])) {
     const permLevel = rolePerms[resourceId] as any
     rolePerms[resourceId] = [permLevel]
diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index 14a450a4de7..0629ebc967c 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -63,7 +63,9 @@ async function updatePermissionOnRole(
   // resource from another role and then adding to the new role
   for (let role of dbRoles) {
     let updated = false
-    const rolePermissions = role.permissions ? role.permissions : {}
+    const rolePermissions: Record<string, PermissionLevel[]> = role.permissions
+      ? role.permissions
+      : {}
     // make sure its an array, also handle migrating
     if (
       !rolePermissions[resourceId] ||
@@ -71,7 +73,7 @@ async function updatePermissionOnRole(
     ) {
       rolePermissions[resourceId] =
         typeof rolePermissions[resourceId] === "string"
-          ? [rolePermissions[resourceId] as unknown as string]
+          ? [rolePermissions[resourceId] as unknown as PermissionLevel]
           : []
     }
     // handle the removal/updating the role which has this permission first
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 9c7fb2859ab..28e9cb27792 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -17,7 +17,7 @@ import {
   SaveRoleResponse,
   UserCtx,
   UserMetadata,
-  UserRoles,
+  DocumentType,
 } from "@budibase/types"
 import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
@@ -89,9 +89,9 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     _id = dbCore.prefixRoleID(_id)
   }
 
-  let dbRole
-  if (!isCreate) {
-    dbRole = await db.get<UserRoles>(_id)
+  let dbRole: Role | undefined
+  if (!isCreate && _id?.startsWith(DocumentType.ROLE)) {
+    dbRole = await db.get<Role>(_id)
   }
   if (dbRole && dbRole.name !== name && isNewVersion) {
     ctx.throw(400, "Cannot change custom role name")
@@ -104,8 +104,13 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     color || "var(--spectrum-global-color-static-magenta-400)",
     permissionId
   ).addInheritance(inherits)
-  if (ctx.request.body._rev) {
-    role._rev = ctx.request.body._rev
+
+  if (dbRole?.permissions && !role.permissions) {
+    role.permissions = dbRole.permissions
+  }
+  const foundRev = ctx.request.body._rev || dbRole?._rev
+  if (foundRev) {
+    role._rev = foundRev
   }
   const result = await db.put(role)
   if (isCreate) {
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index dd943f12825..2b50f818680 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -200,7 +200,7 @@ export function webhookValidator() {
 
 export function roleValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-
+  const permissionString = Joi.string().valid(...permLevelArray)
   return auth.joiValidator.body(
     Joi.object({
       _id: OPTIONAL_STRING,
@@ -216,7 +216,13 @@ export function roleValidator() {
         .valid(...Object.values(permissions.BuiltinPermissionID))
         .required(),
       permissions: Joi.object()
-        .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+        .pattern(
+          /.*/,
+          Joi.alternatives().try(
+            Joi.array().items(permissionString),
+            permissionString
+          )
+        )
         .optional(),
       inherits: OPTIONAL_STRING,
     }).unknown(true)
diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts
index 88058b508da..3ed2fe0a832 100644
--- a/packages/server/src/sdk/app/permissions/index.ts
+++ b/packages/server/src/sdk/app/permissions/index.ts
@@ -44,7 +44,7 @@ export async function getResourcePerms(
         role.permissions || {},
         resourceId
       )
-      if (rolePerms[resourceId]?.indexOf(level) > -1) {
+      if (rolePerms[resourceId]?.indexOf(level as PermissionLevel) > -1) {
         permissions[level] = {
           role: roles.getExternalRoleID(role._id!, role.version),
           type: PermissionSource.EXPLICIT,
diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts
index 0169da90c2a..29687ce6e12 100644
--- a/packages/types/src/documents/app/role.ts
+++ b/packages/types/src/documents/app/role.ts
@@ -1,9 +1,10 @@
 import { Document } from "../document"
+import { PermissionLevel } from "../../sdk"
 
 export interface Role extends Document {
   permissionId: string
   inherits?: string
-  permissions: { [key: string]: string[] }
+  permissions: Record<string, PermissionLevel[]>
   version?: string
   name: string
   displayName?: string
diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts
index af5c11374de..99e01fedf06 100644
--- a/packages/types/src/documents/global/user.ts
+++ b/packages/types/src/documents/global/user.ts
@@ -74,9 +74,8 @@ export enum UserStatus {
   INACTIVE = "inactive",
 }
 
-export interface UserRoles {
-  [key: string]: string
-}
+// specifies a map of app ID to role ID
+export type UserRoles = Record<string, string>
 
 // UTILITY TYPES
 

From 0a627e6cf5b3fcb2d3093be5a53b3f1d397ba024 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Sep 2024 19:24:22 +0100
Subject: [PATCH 120/455] Add validation to editing roles

---
 .../backend/RoleEditor/RoleNode.svelte        | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index eda225b604f..e07ab70ff6c 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -28,6 +28,27 @@
   let tempDescription
   let tempColor
 
+  $: nameError = validateName(tempDisplayName, $roles)
+  $: descriptionError = validateDescription(tempDescription)
+  $: invalid = nameError || descriptionError
+
+  const validateName = (name, roles) => {
+    if (!name?.length) {
+      return "Please enter a name"
+    }
+    if (roles.some(x => x.displayName === name && x._id !== id)) {
+      return "That name is already used by another role"
+    }
+    return null
+  }
+
+  const validateDescription = description => {
+    if (!description?.length) {
+      return "Please enter a name"
+    }
+    return null
+  }
+
   const deleteNode = async () => {
     flow.deleteElements({
       nodes: [{ id }],
@@ -100,15 +121,18 @@
     title={`Edit ${data.displayName}`}
     confirmText="Save"
     onConfirm={saveChanges}
+    disabled={invalid}
   >
     <Input
       label="Name"
       value={tempDisplayName}
+      error={nameError}
       on:change={e => (tempDisplayName = e.detail)}
     />
     <Input
       label="Description"
       value={tempDescription}
+      error={descriptionError}
       on:change={e => (tempDescription = e.detail)}
     />
     <div>

From 35bdc998ca77b6035de5a389afb8472195a4f1f0 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 10:33:30 +0100
Subject: [PATCH 121/455] Fix roles store update

---
 packages/builder/src/stores/builder/roles.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index bee0bd9372d..ef4480c6d24 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -39,7 +39,7 @@ export function createRolesStore() {
         roleId: role?._id,
         roleRev: role?._rev,
       })
-      update(state => state.filter(existing => existing._id !== role._id))
+      store.update(state => state.filter(existing => existing._id !== role._id))
     },
     save: async role => {
       const savedRole = await API.saveRole(role)

From 6f9175168bf377f9f861e3f9da01e86160561831 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 12:23:27 +0100
Subject: [PATCH 122/455] Add custom RBAC edges with inline deletion icon

---
 .../backend/RoleEditor/RoleEdge.svelte        | 114 ++++++++++++++++++
 .../backend/RoleEditor/RoleEditor.svelte      |  11 +-
 .../backend/RoleEditor/RoleNode.svelte        |  20 ++-
 .../components/backend/RoleEditor/layout.js   |   4 +-
 4 files changed, 139 insertions(+), 10 deletions(-)
 create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
new file mode 100644
index 00000000000..b4076e22810
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -0,0 +1,114 @@
+<script>
+  import {
+    getBezierPath,
+    BaseEdge,
+    EdgeLabelRenderer,
+    useSvelteFlow,
+  } from "@xyflow/svelte"
+  import { Icon } from "@budibase/bbui"
+  import { onMount } from "svelte"
+
+  export let sourceX
+  export let sourceY
+  export let sourcePosition
+  export let targetX
+  export let targetY
+  export let targetPosition
+  export let id
+
+  const flow = useSvelteFlow()
+
+  let edgeHovered = false
+  let labelHovered = false
+
+  $: hovered = edgeHovered || labelHovered
+  $: edgeClasses = getEdgeClasses(hovered, labelHovered)
+  $: [edgePath, labelX, labelY] = getBezierPath({
+    sourceX,
+    sourceY,
+    sourcePosition,
+    targetX,
+    targetY,
+    targetPosition,
+  })
+
+  const getEdgeClasses = (hovered, labelHovered) => {
+    let classes = ""
+    if (hovered) classes += `hovered `
+    if (labelHovered) classes += `delete `
+    return classes
+  }
+
+  const onEdgeMouseOver = e => {
+    edgeHovered = true
+  }
+
+  const onEdgeMouseOut = e => {
+    edgeHovered = false
+  }
+
+  const deleteEdge = () => {
+    flow.deleteElements({
+      edges: [{ id }],
+    })
+  }
+
+  onMount(() => {
+    const edge = document.querySelector(`.svelte-flow__edge[data-id="${id}"]`)
+    if (edge) {
+      edge.addEventListener("mouseover", onEdgeMouseOver)
+      edge.addEventListener("mouseout", onEdgeMouseOut)
+    }
+    return () => {
+      if (edge) {
+        edge.removeEventListener("mouseover", onEdgeMouseOver)
+        edge.removeEventListener("mouseout", onEdgeMouseOut)
+      }
+    }
+  })
+</script>
+
+<BaseEdge path={edgePath} class={edgeClasses} />
+<EdgeLabelRenderer>
+  <!-- svelte-ignore a11y-click-events-have-key-events -->
+  <!-- svelte-ignore a11y-no-static-element-interactions -->
+  <!-- svelte-ignore a11y-mouse-events-have-key-events -->
+  <div
+    style:transform="translate(-50%, -50%) translate({labelX}px,{labelY}px)"
+    class="edge-label nodrag nopan"
+    class:hovered
+    on:click={deleteEdge}
+    on:mouseover={() => (labelHovered = true)}
+    on:mouseout={() => (labelHovered = false)}
+  >
+    <Icon name="Delete" />
+  </div>
+</EdgeLabelRenderer>
+
+<style>
+  .edge-label {
+    position: absolute;
+    padding: 8px;
+    opacity: 0;
+    pointer-events: all;
+  }
+  .edge-label:hover,
+  .edge-label.hovered {
+    opacity: 1;
+    cursor: pointer;
+  }
+  .edge-label:hover :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-red-400);
+  }
+  .edge-label :global(.spectrum-Icon) {
+    background: var(--background-color);
+    color: var(--spectrum-global-color-gray-600);
+  }
+  :global(.svelte-flow__edge:hover .svelte-flow__edge-path),
+  :global(.svelte-flow__edge-path.hovered) {
+    stroke: var(--spectrum-global-color-blue-400);
+  }
+  :global(.svelte-flow__edge-path.hovered.delete) {
+    stroke: var(--spectrum-global-color-red-400);
+  }
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 783dc5a108b..d24810f468e 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -4,14 +4,16 @@
   import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
+  import RoleEdge from "./RoleEdge.svelte"
   import { rolesToNodes, autoLayout } from "./layout"
   import { onMount, setContext } from "svelte"
   import Controls from "./Controls.svelte"
 
   const nodes = writable([])
   const edges = writable([])
+  const dragging = writable(false)
 
-  setContext("flow", { nodes, edges })
+  setContext("flow", { nodes, edges, dragging })
 
   onMount(() => {
     const layout = autoLayout(rolesToNodes())
@@ -33,9 +35,12 @@
     {edges}
     snapGrid={[25, 25]}
     nodeTypes={{ role: RoleNode }}
+    edgeTypes={{ role: RoleEdge }}
     proOptions={{ hideAttribution: true }}
     fitViewOptions={{ maxZoom: 1 }}
-    defaultEdgeOptions={{ type: "bezier", animated: true }}
+    defaultEdgeOptions={{ type: "role", animated: true, selectable: false }}
+    onconnectstart={() => dragging.set(true)}
+    onconnectend={() => dragging.set(false)}
   >
     <Background variant={BackgroundVariant.Dots} />
     <Controls />
@@ -81,6 +86,6 @@
 
     /* Edges */
     --xy-edge-stroke: var(--edge-color);
-    --xy-edge-stroke-selected: var(--selected-color);
+    --xy-edge-stroke-selected: var(--edge-color);
   }
 </style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index e07ab70ff6c..2614d609f61 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -16,10 +16,9 @@
   import { roles } from "stores/builder"
 
   export let data
-  export let isConnectable
   export let id
 
-  const { nodes, edges } = getContext("flow")
+  const { nodes, edges, dragging } = getContext("flow")
   const flow = useSvelteFlow()
 
   let anchor
@@ -31,6 +30,7 @@
   $: nameError = validateName(tempDisplayName, $roles)
   $: descriptionError = validateDescription(tempDescription)
   $: invalid = nameError || descriptionError
+  $: targetClasses = `target${$dragging ? "" : " hidden"}`
 
   const validateName = (name, roles) => {
     if (!name?.length) {
@@ -109,10 +109,15 @@
     {/if}
   </div>
   {#if id !== Roles.BASIC}
-    <Handle type="target" position={Position.Left} {isConnectable} />
+    <Handle
+      type="target"
+      position={Position.Left}
+      class={targetClasses}
+      isConnectable={$dragging}
+    />
   {/if}
   {#if id !== Roles.ADMIN}
-    <Handle type="source" position={Position.Right} {isConnectable} />
+    <Handle type="source" position={Position.Right} />
   {/if}
 </div>
 
@@ -207,4 +212,11 @@
   .node:hover .buttons {
     display: flex;
   }
+  .node :global(.svelte-flow__handle.target) {
+    background: var(--background-color);
+  }
+  .node :global(.svelte-flow__handle.hidden) {
+    opacity: 0;
+    pointer-events: none;
+  }
 </style>
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index 5dae9a3b5c3..d721549d48b 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -62,7 +62,6 @@ export const rolesToNodes = () => {
         id: `${sourceRole}-${role._id}`,
         source: sourceRole,
         target: role._id,
-        animated: true,
       })
     }
   }
@@ -79,7 +78,7 @@ const dagreLayout = ({ nodes, edges }) => {
   dagreGraph.setDefaultEdgeLabel(() => ({}))
   dagreGraph.setGraph({
     rankdir: "LR",
-    ranksep: 100,
+    ranksep: 200,
     nodesep: 100,
   })
   nodes.forEach(node => {
@@ -122,7 +121,6 @@ const sanitiseLayout = ({ nodes, edges }) => {
         id: Helpers.uuid(),
         source: node.id,
         target: Roles.ADMIN,
-        animated: true,
       })
     }
   }

From d80d38196fb4e2be36f47ec8a50701a6ce36af95 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 15:53:25 +0100
Subject: [PATCH 123/455] Update styles of handles and edges

---
 .../src/components/backend/RoleEditor/RoleEdge.svelte        | 5 ++++-
 .../src/components/backend/RoleEditor/RoleEditor.svelte      | 3 +++
 .../src/components/backend/RoleEditor/RoleNode.svelte        | 5 +++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index b4076e22810..243e4b988bf 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -5,7 +5,7 @@
     EdgeLabelRenderer,
     useSvelteFlow,
   } from "@xyflow/svelte"
-  import { Icon } from "@budibase/bbui"
+  import { Icon, ActionButton } from "@budibase/bbui"
   import { onMount } from "svelte"
 
   export let sourceX
@@ -104,6 +104,9 @@
     background: var(--background-color);
     color: var(--spectrum-global-color-gray-600);
   }
+  .edge-label :global(svg) {
+    padding: 8px;
+  }
   :global(.svelte-flow__edge:hover .svelte-flow__edge-path),
   :global(.svelte-flow__edge-path.hovered) {
     stroke: var(--spectrum-global-color-blue-400);
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index d24810f468e..e64ac7d9353 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -70,6 +70,8 @@
     --handle-color: var(--spectrum-global-color-gray-600);
     --selected-color: var(--spectrum-global-color-blue-400);
   }
+
+  /* Customise svelte-flow theme */
   .flow :global(.svelte-flow) {
     /* Panel */
     --xy-background-color: var(--background-color);
@@ -87,5 +89,6 @@
     /* Edges */
     --xy-edge-stroke: var(--edge-color);
     --xy-edge-stroke-selected: var(--edge-color);
+    --xy-edge-stroke-width: 2px;
   }
 </style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 2614d609f61..97ded3c2ea2 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -212,6 +212,11 @@
   .node:hover .buttons {
     display: flex;
   }
+  .node :global(.svelte-flow__handle) {
+    width: 6px;
+    height: 6px;
+    border-width: 2px;
+  }
   .node :global(.svelte-flow__handle.target) {
     background: var(--background-color);
   }

From d4451a487cb8afab1cca561c4f4c089fb965f982 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 16:09:37 +0100
Subject: [PATCH 124/455] Increase max auto zoom and add more constants

---
 .../src/components/backend/RoleEditor/Controls.svelte      | 7 ++++---
 .../src/components/backend/RoleEditor/RoleNode.svelte      | 4 ++--
 .../builder/src/components/backend/RoleEditor/constants.js | 1 +
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 94c82dcd65b..8ac8ce26686 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -3,7 +3,7 @@
   import { useSvelteFlow, Position } from "@xyflow/svelte"
   import { getContext, tick } from "svelte"
   import { autoLayout, roleToNode } from "./layout"
-  import { ZoomDuration } from "./constants"
+  import { MaxAutoZoom, ZoomDuration } from "./constants"
   import { getSequentialName } from "helpers/duplicate"
   import { roles } from "stores/builder"
   import { Roles } from "constants/backend"
@@ -33,7 +33,7 @@
     edges.set(layout.edges)
     await tick()
     flow.fitView({
-      maxZoom: 1,
+      maxZoom: MaxAutoZoom,
       duration: ZoomDuration,
     })
   }
@@ -54,7 +54,8 @@
   </div>
   <Button
     secondary
-    on:click={() => flow.fitView({ maxZoom: 1, duration: ZoomDuration })}
+    on:click={() =>
+      flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })}
   >
     Zoom to fit
   </Button>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 97ded3c2ea2..69d75d9a578 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -10,7 +10,7 @@
     FieldLabel,
   } from "@budibase/bbui"
   import { Roles } from "constants/backend"
-  import { NodeWidth, NodeHeight } from "./constants"
+  import { NodeWidth, NodeHeight, MaxAutoZoom, ZoomDuration } from "./constants"
   import { getContext, tick } from "svelte"
   import { autoLayout, nodeToRole } from "./layout"
   import { roles } from "stores/builder"
@@ -79,7 +79,7 @@
     const layout = autoLayout({ nodes: $nodes, edges: $edges })
     nodes.set(layout.nodes)
     edges.set(layout.edges)
-    flow.fitView({ maxZoom: 1, duration: 300 })
+    flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })
   }
 </script>
 
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index 76faf284825..d2fb813830e 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -1,3 +1,4 @@
 export const NodeWidth = 220
 export const NodeHeight = 66
 export const ZoomDuration = 300
+export const MaxAutoZoom = 1.2

From 87206b1c4320f09906192b728d77c7476b253634 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 16:11:37 +0100
Subject: [PATCH 125/455] Update node spacing

---
 .../src/components/backend/RoleEditor/RoleEditor.svelte        | 3 ++-
 packages/builder/src/components/backend/RoleEditor/layout.js   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index e64ac7d9353..01f87b6dea0 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -8,6 +8,7 @@
   import { rolesToNodes, autoLayout } from "./layout"
   import { onMount, setContext } from "svelte"
   import Controls from "./Controls.svelte"
+  import { MaxAutoZoom } from "./constants"
 
   const nodes = writable([])
   const edges = writable([])
@@ -37,7 +38,7 @@
     nodeTypes={{ role: RoleNode }}
     edgeTypes={{ role: RoleEdge }}
     proOptions={{ hideAttribution: true }}
-    fitViewOptions={{ maxZoom: 1 }}
+    fitViewOptions={{ maxZoom: MaxAutoZoom }}
     defaultEdgeOptions={{ type: "role", animated: true, selectable: false }}
     onconnectstart={() => dragging.set(true)}
     onconnectend={() => dragging.set(false)}
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index d721549d48b..c702adb821e 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -79,7 +79,7 @@ const dagreLayout = ({ nodes, edges }) => {
   dagreGraph.setGraph({
     rankdir: "LR",
     ranksep: 200,
-    nodesep: 100,
+    nodesep: 50,
   })
   nodes.forEach(node => {
     dagreGraph.setNode(node.id, { width: NodeWidth, height: NodeHeight })

From dded09ca9724b594f1950544dc51b539abc9979b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 16:51:36 +0100
Subject: [PATCH 126/455] Update RBAC editor to use a grid and make all
 dimensions consistent with grid

---
 .../backend/RoleEditor/RoleEditor.svelte      |  5 ++--
 .../backend/RoleEditor/RoleNode.svelte        | 23 ++++++++++++-------
 .../backend/RoleEditor/constants.js           |  5 ++--
 .../components/backend/RoleEditor/layout.js   | 12 +++++-----
 4 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 01f87b6dea0..c1772c32bb4 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -8,7 +8,7 @@
   import { rolesToNodes, autoLayout } from "./layout"
   import { onMount, setContext } from "svelte"
   import Controls from "./Controls.svelte"
-  import { MaxAutoZoom } from "./constants"
+  import { GridResolution, MaxAutoZoom } from "./constants"
 
   const nodes = writable([])
   const edges = writable([])
@@ -34,7 +34,7 @@
     fitView
     {nodes}
     {edges}
-    snapGrid={[25, 25]}
+    snapGrid={[GridResolution, GridResolution]}
     nodeTypes={{ role: RoleNode }}
     edgeTypes={{ role: RoleEdge }}
     proOptions={{ hideAttribution: true }}
@@ -76,7 +76,6 @@
   .flow :global(.svelte-flow) {
     /* Panel */
     --xy-background-color: var(--background-color);
-    --xy-background-pattern-color-props: transparent;
 
     /* Controls */
     --xy-controls-button-background-color: var(--node-background);
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 69d75d9a578..b4a3cafc582 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -156,28 +156,30 @@
     width: var(--width);
     height: var(--height);
     display: flex;
-    flex-direction: column;
+    flex-direction: row;
+    box-sizing: border-box;
   }
   .node.selected {
     background: var(--spectrum-global-color-blue-100);
   }
   .color {
     border-top-left-radius: 4px;
-    border-top-right-radius: 4px;
-    height: 8px;
-    width: 100%;
+    border-bottom-left-radius: 4px;
+    height: 100%;
+    width: 10px;
+    flex: 0 0 10px;
     background: var(--color);
   }
   .content {
     flex: 1 1 auto;
-    padding: 0 14px 0 14px;
+    padding: 0 12px;
     display: flex;
     flex-direction: column;
     justify-content: center;
     align-items: stretch;
     gap: 2px;
     border: 1px solid var(--border-color);
-    border-bottom-left-radius: 4px;
+    border-top-right-radius: 4px;
     border-bottom-right-radius: 4px;
   }
   .node.selected .content {
@@ -199,8 +201,7 @@
   .title :global(.spectrum-Icon) {
     color: var(--spectrum-global-color-gray-600);
   }
-  .name,
-  .description {
+  .name {
     white-space: nowrap;
     text-overflow: ellipsis;
     overflow: hidden;
@@ -208,6 +209,12 @@
   .description {
     color: var(--spectrum-global-color-gray-600);
     font-size: 12px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    display: -webkit-box;
+    -webkit-line-clamp: 1;
+    line-clamp: 1;
+    -webkit-box-orient: vertical;
   }
   .node:hover .buttons {
     display: flex;
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index d2fb813830e..d22a228a877 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -1,4 +1,5 @@
-export const NodeWidth = 220
-export const NodeHeight = 66
 export const ZoomDuration = 300
 export const MaxAutoZoom = 1.2
+export const GridResolution = 20
+export const NodeHeight = GridResolution * 3
+export const NodeWidth = GridResolution * 12
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index c702adb821e..fb1f8e795a1 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -1,5 +1,5 @@
 import dagre from "@dagrejs/dagre"
-import { NodeWidth, NodeHeight } from "./constants"
+import { NodeWidth, NodeHeight, GridResolution } from "./constants"
 import { Position } from "@xyflow/svelte"
 import { roles } from "stores/builder"
 import { Roles } from "constants/backend"
@@ -78,8 +78,8 @@ const dagreLayout = ({ nodes, edges }) => {
   dagreGraph.setDefaultEdgeLabel(() => ({}))
   dagreGraph.setGraph({
     rankdir: "LR",
-    ranksep: 200,
-    nodesep: 50,
+    ranksep: GridResolution * 6,
+    nodesep: GridResolution * 2,
   })
   nodes.forEach(node => {
     dagreGraph.setNode(node.id, { width: NodeWidth, height: NodeHeight })
@@ -89,12 +89,12 @@ const dagreLayout = ({ nodes, edges }) => {
   })
   dagre.layout(dagreGraph)
   nodes.forEach(node => {
-    const nodeWithPosition = dagreGraph.node(node.id)
+    const pos = dagreGraph.node(node.id)
     node.targetPosition = Position.Left
     node.sourcePosition = Position.Right
     node.position = {
-      x: nodeWithPosition.x - NodeWidth / 2,
-      y: nodeWithPosition.y - NodeHeight / 2,
+      x: Math.round((pos.x - NodeWidth / 2) / GridResolution) * GridResolution,
+      y: Math.round((pos.y - NodeHeight / 2) / GridResolution) * GridResolution,
     }
   })
   return { nodes, edges }

From 63dd73f7b6541807533dadb22cdb4f9d67cf5136 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Sep 2024 16:56:25 +0100
Subject: [PATCH 127/455] Update RBAC editor edges to explain what action the
 delete icon will take

---
 .../components/backend/RoleEditor/RoleEdge.svelte   | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 243e4b988bf..5dc21d6f85c 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -5,8 +5,9 @@
     EdgeLabelRenderer,
     useSvelteFlow,
   } from "@xyflow/svelte"
-  import { Icon, ActionButton } from "@budibase/bbui"
+  import { Icon, TooltipPosition } from "@budibase/bbui"
   import { onMount } from "svelte"
+  import { roles } from "stores/builder"
 
   export let sourceX
   export let sourceY
@@ -15,6 +16,8 @@
   export let targetY
   export let targetPosition
   export let id
+  export let source
+  export let target
 
   const flow = useSvelteFlow()
 
@@ -31,6 +34,12 @@
     targetY,
     targetPosition,
   })
+  $: sourceRole = $roles.find(x => x._id === source)
+  $: targetRole = $roles.find(x => x._id === target)
+  $: tooltip =
+    sourceRole && targetRole
+      ? `Stop ${targetRole.displayName} from inheriting ${sourceRole.displayName}`
+      : null
 
   const getEdgeClasses = (hovered, labelHovered) => {
     let classes = ""
@@ -81,7 +90,7 @@
     on:mouseover={() => (labelHovered = true)}
     on:mouseout={() => (labelHovered = false)}
   >
-    <Icon name="Delete" />
+    <Icon name="Delete" {tooltip} tooltipPosition={TooltipPosition.Top} />
   </div>
 </EdgeLabelRenderer>
 

From 040405089783741815081a57f0824a8195e7b7c6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 13 Sep 2024 10:47:56 +0100
Subject: [PATCH 128/455] Reset server changes to master

---
 packages/backend-core/src/security/roles.ts   | 81 +++++++------------
 packages/server/src/api/controllers/role.ts   | 27 ++-----
 .../src/api/routes/tests/permissions.spec.ts  | 36 +--------
 .../server/src/api/routes/utils/validators.ts |  8 +-
 packages/types/src/api/web/role.ts            |  3 -
 packages/types/src/documents/app/role.ts      | 10 ++-
 6 files changed, 54 insertions(+), 111 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 097d6e84da7..a7210ec2b8e 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -7,8 +7,9 @@ import {
   doWithDB,
 } from "../db"
 import { getAppDB } from "../context"
-import { Screen, Role as RoleDoc } from "@budibase/types"
+import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types"
 import cloneDeep from "lodash/fp/cloneDeep"
+import { RoleColor } from "@budibase/shared-core"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",
@@ -45,22 +46,12 @@ export class Role implements RoleDoc {
   inherits?: string
   version?: string
   permissions: Record<string, PermissionLevel[]> = {}
-  displayName?: string
-  color?: string
-  description?: string
+  uiMetadata?: RoleUIMetadata
 
-  constructor(
-    id: string,
-    displayName: string,
-    description: string,
-    color: string,
-    permissionId: string
-  ) {
+  constructor(id: string, permissionId: string, uiMetadata?: RoleUIMetadata) {
     this._id = id
-    this.name = id
-    this.displayName = displayName
-    this.color = color
-    this.description = description
+    this.name = uiMetadata?.displayName || id
+    this.uiMetadata = uiMetadata
     this.permissionId = permissionId
     // version for managing the ID - removing the role_ when responding
     this.version = RoleIDVersion.NAME
@@ -73,41 +64,31 @@ export class Role implements RoleDoc {
 }
 
 const BUILTIN_ROLES = {
-  ADMIN: new Role(
-    BUILTIN_IDS.ADMIN,
-    "App admin",
-    "Can do everything",
-    "var(--spectrum-global-color-static-red-400)",
-    BuiltinPermissionID.ADMIN
-  ).addInheritance(BUILTIN_IDS.POWER),
-  POWER: new Role(
-    BUILTIN_IDS.POWER,
-    "App power user",
-    "An app user with more access",
-    "var(--spectrum-global-color-static-orange-400)",
-    BuiltinPermissionID.POWER
-  ).addInheritance(BUILTIN_IDS.BASIC),
-  BASIC: new Role(
-    BUILTIN_IDS.BASIC,
-    "App user",
-    "Any logged in user",
-    "var(--spectrum-global-color-static-green-400)",
-    BuiltinPermissionID.WRITE
-  ).addInheritance(BUILTIN_IDS.PUBLIC),
-  PUBLIC: new Role(
-    BUILTIN_IDS.PUBLIC,
-    "Public user",
-    "Accessible to anyone",
-    "var(--spectrum-global-color-static-blue-400)",
-    BuiltinPermissionID.PUBLIC
-  ),
-  BUILDER: new Role(
-    BUILTIN_IDS.BUILDER,
-    "Builder user",
-    "Users that can edit this app",
-    "var(--spectrum-global-color-static-magenta-600)",
-    BuiltinPermissionID.ADMIN
-  ),
+  ADMIN: new Role(BUILTIN_IDS.ADMIN, BuiltinPermissionID.ADMIN, {
+    displayName: "App admin",
+    description: "Can do everything",
+    color: RoleColor.ADMIN,
+  }).addInheritance(BUILTIN_IDS.POWER),
+  POWER: new Role(BUILTIN_IDS.POWER, BuiltinPermissionID.POWER, {
+    displayName: "App power user",
+    description: "An app user with more access",
+    color: RoleColor.POWER,
+  }).addInheritance(BUILTIN_IDS.BASIC),
+  BASIC: new Role(BUILTIN_IDS.BASIC, BuiltinPermissionID.WRITE, {
+    displayName: "App user",
+    description: "Any logged in user",
+    color: RoleColor.BASIC,
+  }).addInheritance(BUILTIN_IDS.PUBLIC),
+  PUBLIC: new Role(BUILTIN_IDS.PUBLIC, BuiltinPermissionID.PUBLIC, {
+    displayName: "Public user",
+    description: "Accessible to anyone",
+    color: RoleColor.PUBLIC,
+  }),
+  BUILDER: new Role(BUILTIN_IDS.BUILDER, BuiltinPermissionID.ADMIN, {
+    displayName: "Builder user",
+    description: "Users that can edit this app",
+    color: RoleColor.BUILDER,
+  }),
 }
 
 export function getBuiltinRoles(): { [key: string]: RoleDoc } {
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 28e9cb27792..ee1c2239524 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -19,7 +19,7 @@ import {
   UserMetadata,
   DocumentType,
 } from "@budibase/types"
-import { sdk as sharedSdk } from "@budibase/shared-core"
+import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
 const UpdateRolesOptions = {
@@ -62,16 +62,8 @@ export async function find(ctx: UserCtx<void, FindRoleResponse>) {
 
 export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   const db = context.getAppDB()
-  let {
-    _id,
-    name,
-    displayName,
-    description,
-    color,
-    inherits,
-    permissionId,
-    version,
-  } = ctx.request.body
+  let { _id, name, inherits, permissionId, version, uiMetadata } =
+    ctx.request.body
   let isCreate = false
   const isNewVersion = version === roles.RoleIDVersion.NAME
 
@@ -97,14 +89,11 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     ctx.throw(400, "Cannot change custom role name")
   }
 
-  const role = new roles.Role(
-    _id,
-    displayName || name,
-    description || "Custom role",
-    color || "var(--spectrum-global-color-static-magenta-400)",
-    permissionId
-  ).addInheritance(inherits)
-
+  const role = new roles.Role(_id, permissionId, {
+    displayName: uiMetadata?.displayName || name,
+    description: uiMetadata?.description || "Custom role",
+    color: uiMetadata?.color || RoleColor.DEFAULT_CUSTOM,
+  }).addInheritance(inherits)
   if (dbRole?.permissions && !role.permissions) {
     role.permissions = dbRole.permissions
   }
diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 0fba930144c..0f059998ae1 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -1,16 +1,5 @@
-const mockedSdk = sdk.permissions as jest.Mocked<typeof sdk.permissions>
-
-import sdk from "../../../sdk"
-
 import { roles } from "@budibase/backend-core"
-import {
-  Document,
-  DocumentType,
-  PermissionLevel,
-  Row,
-  Table,
-  ViewV2,
-} from "@budibase/types"
+import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
@@ -36,6 +25,7 @@ describe("/permission", () => {
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
+
     table = (await config.createTable()) as typeof table
     row = await config.createRow()
     view = await config.api.viewV2.create({
@@ -154,27 +144,7 @@ describe("/permission", () => {
       await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
     })
 
-    it("should ignore the view permissions if the flag is not on", async () => {
-      await config.api.permission.add({
-        roleId: STD_ROLE_ID,
-        resourceId: view.id,
-        level: PermissionLevel.READ,
-      })
-      await config.api.permission.revoke({
-        roleId: STD_ROLE_ID,
-        resourceId: table._id,
-        level: PermissionLevel.READ,
-      })
-      // replicate changes before checking permissions
-      await config.publish()
-
-      await config.api.viewV2.publicSearch(view.id, undefined, {
-        status: 401,
-      })
-    })
-
-    it("should use the view permissions if the flag is on", async () => {
-      mocks.licenses.useViewPermissions()
+    it("should use the view permissions", async () => {
       await config.api.permission.add({
         roleId: STD_ROLE_ID,
         resourceId: view.id,
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index 2b50f818680..b589d44b31c 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -208,9 +208,11 @@ export function roleValidator() {
       name: Joi.string()
         .regex(/^[a-zA-Z0-9_]*$/)
         .required(),
-      displayName: Joi.string().optional(),
-      color: Joi.string().optional(),
-      description: Joi.string().optional(),
+      uiMetadata: Joi.object({
+        displayName: OPTIONAL_STRING,
+        color: OPTIONAL_STRING,
+        description: OPTIONAL_STRING,
+      }).optional(),
       // this is the base permission ID (for now a built in)
       permissionId: Joi.string()
         .valid(...Object.values(permissions.BuiltinPermissionID))
diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts
index df63fbaa779..644222b8f94 100644
--- a/packages/types/src/api/web/role.ts
+++ b/packages/types/src/api/web/role.ts
@@ -4,9 +4,6 @@ export interface SaveRoleRequest {
   _id?: string
   _rev?: string
   name: string
-  displayName?: string
-  color?: string
-  description?: string
   inherits: string
   permissionId: string
   version: string
diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts
index 29687ce6e12..6557b7e19da 100644
--- a/packages/types/src/documents/app/role.ts
+++ b/packages/types/src/documents/app/role.ts
@@ -1,13 +1,17 @@
 import { Document } from "../document"
 import { PermissionLevel } from "../../sdk"
 
+export interface RoleUIMetadata {
+  displayName?: string
+  color?: string
+  description?: string
+}
+
 export interface Role extends Document {
   permissionId: string
   inherits?: string
   permissions: Record<string, PermissionLevel[]>
   version?: string
   name: string
-  displayName?: string
-  color?: string
-  description?: string
+  uiMetadata?: RoleUIMetadata
 }

From 2849a7a4ff59973c743926b9f062ba2b52dafc81 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 13 Sep 2024 14:03:21 +0100
Subject: [PATCH 129/455] Add uiMetdata prefix to roles everywhere

---
 .../buttons/ManageAccessButton.svelte          | 18 +++++++++---------
 .../DataTable/modals/CreateEditUser.svelte     |  2 +-
 .../backend/RoleEditor/Controls.svelte         | 12 +++++++-----
 .../backend/RoleEditor/RoleEdge.svelte         |  2 +-
 .../backend/RoleEditor/RoleNode.svelte         | 12 ++++--------
 .../components/backend/RoleEditor/layout.js    | 12 ++++++------
 .../src/components/common/RoleSelect.svelte    |  8 ++++----
 .../design/settings/controls/RoleSelect.svelte |  4 ++--
 packages/builder/src/dataBinding.js            |  4 ++--
 .../ScreenList/RoleIndicator.svelte            |  5 +++--
 .../_components/AppRoleTableRenderer.svelte    |  2 +-
 packages/builder/src/stores/builder/roles.js   |  9 ++++++---
 .../src/components/grid/cells/RoleCell.svelte  |  5 +++--
 13 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 1725740a7c0..2826d8d9860 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -29,7 +29,7 @@
   $: roleMismatch = checkRoleMismatch(permissions)
   $: selectedRole = roleMismatch ? null : permissions?.[0]?.value
   $: readableRole = selectedRole
-    ? $roles.find(x => x._id === selectedRole)?.displayName
+    ? $roles.find(x => x._id === selectedRole)?.uiMetadata.displayName
     : null
   $: buttonLabel = readableRole ? `Access: ${readableRole}` : "Access"
   $: highlight = roleMismatch || selectedRole === Roles.PUBLIC
@@ -39,8 +39,8 @@
     .filter(x => !builtins.includes(x._id))
     .slice()
     .toSorted((a, b) => {
-      const aName = a.displayName || a.name
-      const bName = b.displayName || b.name
+      const aName = a.uiMetadata.displayName || a.name
+      const bName = b.uiMetadata.displayName || b.name
       return aName < bName ? -1 : 1
     })
 
@@ -159,23 +159,23 @@
   <List>
     {#each builtInRoles as role}
       <ListItem
-        title={role.displayName}
-        subtitle={role.description}
+        title={role.uiMetadata.displayName}
+        subtitle={role.uiMetadata.description}
         hoverable
         selected={selectedRole === role._id}
         icon="StatusLight"
-        iconColor={role.color}
+        iconColor={role.uiMetadata.color}
         on:click={() => changePermission(role._id)}
       />
     {/each}
     {#each customRoles as role}
       <ListItem
-        title={role.displayName}
-        subtitle={role.description}
+        title={role.uiMetadata.displayName}
+        subtitle={role.uiMetadata.description}
         hoverable
         selected={selectedRole === role._id}
         icon="StatusLight"
-        iconColor={role.color}
+        iconColor={role.uiMetadata.color}
         on:click={() => changePermission(role._id)}
       />
     {/each}
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte
index befe5ec4846..5ac2beab65c 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte
@@ -125,7 +125,7 @@
     label="Role"
     bind:value={row.roleId}
     options={$roles}
-    getOptionLabel={role => role.displayName}
+    getOptionLabel={role => role.uiMetadata.displayName}
     getOptionValue={role => role._id}
     disabled={!creating}
   />
diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 8ac8ce26686..a5c601f2dd7 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -14,11 +14,13 @@
   const addRole = async () => {
     const role = {
       name: Helpers.uuid(),
-      displayName: getSequentialName($nodes, "New role ", {
-        getName: x => x.data.displayName,
-      }),
-      color: "var(--spectrum-global-color-gray-700)",
-      description: "Custom role",
+      uiMetadata: {
+        displayName: getSequentialName($roles, "New role ", {
+          getName: x => x.uiMetadata.displayName,
+        }),
+        color: "var(--spectrum-global-color-gray-700)",
+        description: "Custom role",
+      },
       permissionId: "write",
       inherits: Roles.BASIC,
     }
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 5dc21d6f85c..4e25294d21d 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -38,7 +38,7 @@
   $: targetRole = $roles.find(x => x._id === target)
   $: tooltip =
     sourceRole && targetRole
-      ? `Stop ${targetRole.displayName} from inheriting ${sourceRole.displayName}`
+      ? `Stop ${targetRole.uiMetadata.displayName} from inheriting ${sourceRole.uiMetadata.displayName}`
       : null
 
   const getEdgeClasses = (hovered, labelHovered) => {
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index b4a3cafc582..1a387de13e7 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -36,7 +36,7 @@
     if (!name?.length) {
       return "Please enter a name"
     }
-    if (roles.some(x => x.displayName === name && x._id !== id)) {
+    if (roles.some(x => x.uiMetadata.displayName === name && x._id !== id)) {
       return "That name is already used by another role"
     }
     return null
@@ -171,6 +171,7 @@
     background: var(--color);
   }
   .content {
+    width: 0;
     flex: 1 1 auto;
     padding: 0 12px;
     display: flex;
@@ -201,7 +202,8 @@
   .title :global(.spectrum-Icon) {
     color: var(--spectrum-global-color-gray-600);
   }
-  .name {
+  .name,
+  .description {
     white-space: nowrap;
     text-overflow: ellipsis;
     overflow: hidden;
@@ -209,12 +211,6 @@
   .description {
     color: var(--spectrum-global-color-gray-600);
     font-size: 12px;
-    overflow: hidden;
-    text-overflow: ellipsis;
-    display: -webkit-box;
-    -webkit-line-clamp: 1;
-    line-clamp: 1;
-    -webkit-box-orient: vertical;
   }
   .node:hover .buttons {
     display: flex;
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index fb1f8e795a1..3671cb6782f 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -14,9 +14,7 @@ export const roleToNode = role => ({
   type: "role",
   position: { x: 0, y: 0 },
   data: {
-    displayName: role.displayName,
-    description: role.description,
-    color: role.color,
+    ...role.uiMetadata,
     custom: !role._id.match(/[A-Z]+/),
   },
 })
@@ -26,9 +24,11 @@ export const nodeToRole = node => {
   const role = get(roles).find(x => x._id === node.id)
   return {
     ...role,
-    displayName: node.data.displayName,
-    color: node.data.color,
-    description: node.data.description,
+    uiMetadata: {
+      displayName: node.data.displayName,
+      color: node.data.color,
+      description: node.data.description,
+    },
   }
 }
 
diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 152b752fd00..c59dd8fc605 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -49,8 +49,8 @@
       let options = roles
         .filter(role => allowedRoles.includes(role._id))
         .map(role => ({
-          color: role.color,
-          name: enrichLabel(role.displayName),
+          color: role.uiMetadata.color,
+          name: enrichLabel(role.uiMetadata.displayName),
           _id: role._id,
         }))
       if (allowedRoles.includes(Constants.Roles.CREATOR)) {
@@ -65,8 +65,8 @@
 
     // Allow all core roles
     let options = roles.map(role => ({
-      color: role.color,
-      name: enrichLabel(role.displayName),
+      color: role.uiMetadata.color,
+      name: enrichLabel(role.uiMetadata.displayName),
       _id: role._id,
     }))
 
diff --git a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
index 773abd69b9e..ad466cbce76 100644
--- a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
@@ -11,9 +11,9 @@
   bind:value
   on:change
   options={$roles}
-  getOptionLabel={role => role.displayName}
+  getOptionLabel={role => role.uiMetadata.displayName}
   getOptionValue={role => role._id}
-  getOptionColour={role => role.color}
+  getOptionColour={role => role.uiMetadata.color}
   {placeholder}
   {error}
 />
diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js
index 09ff8e7a812..a59ce16e5a2 100644
--- a/packages/builder/src/dataBinding.js
+++ b/packages/builder/src/dataBinding.js
@@ -725,10 +725,10 @@ const getRoleBindings = () => {
     return {
       type: "context",
       runtimeBinding: `'${role._id}'`,
-      readableBinding: `Role.${role.displayName}`,
+      readableBinding: `Role.${role.uiMetadata.displayName}`,
       category: "Role",
       icon: "UserGroup",
-      display: { type: "string", name: role.displayName },
+      display: { type: "string", name: role.uiMetadata.displayName },
     }
   })
 }
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
index 4ee5d5dec58..09e29d806ab 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/RoleIndicator.svelte
@@ -7,12 +7,13 @@
 
   let showTooltip = false
 
-  $: color = role.color || "var(--spectrum-global-color-static-magenta-400)"
   $: role = $roles.find(role => role._id === roleId)
+  $: color =
+    role?.uiMetadata.color || "var(--spectrum-global-color-static-magenta-400)"
   $: tooltip =
     roleId === Roles.PUBLIC
       ? "Open to the public"
-      : `Requires ${role?.displayName} access`
+      : `Requires ${role?.uiMetadata.displayName || "Unknown role"} access`
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
index e63ee2493ed..7ed36eb3a4e 100644
--- a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
@@ -11,7 +11,7 @@
   const getRoleLabel = roleId => {
     return roleId === Constants.Roles.CREATOR
       ? capitalise(Constants.Roles.CREATOR.toLowerCase())
-      : role?.displayName || role?.name || "Custom role"
+      : role?.uiMetadata.displayName || role?.name || "Custom role"
   }
 </script>
 
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index ef4480c6d24..0b322104b54 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -9,9 +9,12 @@ export function createRolesStore() {
       ...role,
 
       // Ensure we have new metadata for all roles
-      displayName: role.displayName || role.name,
-      color: role.color || "var(--spectrum-global-color-magenta-400)",
-      description: role.description || "Custom role",
+      uiMetadata: {
+        displayName: role.uiMetadata?.displayName || role.name,
+        color:
+          role.uiMetadata?.color || "var(--spectrum-global-color-magenta-400)",
+        description: role.uiMetadata?.description || "Custom role",
+      },
     }))
   })
 
diff --git a/packages/frontend-core/src/components/grid/cells/RoleCell.svelte b/packages/frontend-core/src/components/grid/cells/RoleCell.svelte
index b638d0e0e3f..82d1e26aa7d 100644
--- a/packages/frontend-core/src/components/grid/cells/RoleCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RoleCell.svelte
@@ -12,11 +12,12 @@
     <StatusLight
       square
       size="L"
-      color={role?.color || "var(--spectrum-global-color-magenta-400)"}
+      color={role?.uiMetadata?.color ||
+        "var(--spectrum-global-color-static-magenta-400)"}
     />
   </div>
   <div class="value">
-    {role?.displayName || role?.name || value}
+    {role?.uiMetadata?.displayName || role?.name || "Unknown role"}
   </div>
 </div>
 

From bf10b4cd9dd92485c78c4fd80b31378659b302fd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 09:26:31 +0100
Subject: [PATCH 130/455] Lint

---
 .../builder/src/components/backend/RoleEditor/Controls.svelte | 2 +-
 .../builder/src/components/backend/RoleEditor/RoleEdge.svelte | 4 ++--
 .../builder/src/components/backend/RoleEditor/RoleNode.svelte | 1 -
 packages/builder/src/components/common/RoleIcon.svelte        | 1 -
 packages/builder/src/components/common/RoleSelect.svelte      | 2 +-
 5 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index a5c601f2dd7..1ddb856b86e 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -1,6 +1,6 @@
 <script>
   import { Button, Helpers, ActionButton } from "@budibase/bbui"
-  import { useSvelteFlow, Position } from "@xyflow/svelte"
+  import { useSvelteFlow } from "@xyflow/svelte"
   import { getContext, tick } from "svelte"
   import { autoLayout, roleToNode } from "./layout"
   import { MaxAutoZoom, ZoomDuration } from "./constants"
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 4e25294d21d..73c43490178 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -48,11 +48,11 @@
     return classes
   }
 
-  const onEdgeMouseOver = e => {
+  const onEdgeMouseOver = () => {
     edgeHovered = true
   }
 
-  const onEdgeMouseOut = e => {
+  const onEdgeMouseOut = () => {
     edgeHovered = false
   }
 
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 1a387de13e7..9a4f5f54fdb 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -1,5 +1,4 @@
 <script>
-  import { RoleUtils } from "@budibase/frontend-core"
   import { Handle, Position, useSvelteFlow } from "@xyflow/svelte"
   import {
     Icon,
diff --git a/packages/builder/src/components/common/RoleIcon.svelte b/packages/builder/src/components/common/RoleIcon.svelte
index 567a7c0b5da..3b48935e0c5 100644
--- a/packages/builder/src/components/common/RoleIcon.svelte
+++ b/packages/builder/src/components/common/RoleIcon.svelte
@@ -1,5 +1,4 @@
 <script>
-  import { RoleUtils } from "@budibase/frontend-core"
   import { StatusLight } from "@budibase/bbui"
   import { roles } from "stores/builder"
 
diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index c59dd8fc605..38b84e964d9 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -3,7 +3,7 @@
   import { roles } from "stores/builder"
   import { licensing } from "stores/portal"
 
-  import { Constants, RoleUtils } from "@budibase/frontend-core"
+  import { Constants } from "@budibase/frontend-core"
   import { createEventDispatcher } from "svelte"
   import { capitalise } from "helpers"
 

From a120a9c97a828b38c7b6e27675fe5e210a249928 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 09:40:52 +0100
Subject: [PATCH 131/455] Add multi user sync for roles

---
 packages/builder/src/stores/builder/roles.js  | 28 +++++++++++++++++++
 .../builder/src/stores/builder/websocket.js   | 11 ++++++--
 packages/server/src/api/controllers/role.ts   |  3 ++
 packages/server/src/websockets/builder.ts     | 15 ++++++++++
 packages/shared-core/src/constants/index.ts   |  1 +
 5 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index 0b322104b54..fcbceab1ce2 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -55,6 +55,34 @@ export function createRolesStore() {
         _id: savedRole._id.replace("role_", ""),
       }
     },
+    replace: (roleId, role) => {
+      console.log("replace", roleId, role)
+
+      // Handles external updates of roles
+      if (!roleId) {
+        return
+      }
+
+      // Handle deletion
+      if (!role) {
+        store.update(state => state.filter(x => x._id !== roleId))
+        return
+      }
+
+      // Add new role
+      const index = get(store).findIndex(x => x._id === role._id)
+      if (index === -1) {
+        store.update(state => [...state, role])
+      }
+
+      // Update existing role
+      else if (role) {
+        store.update(state => {
+          state[index] = role
+          return [...state]
+        })
+      }
+    },
   }
 
   return {
diff --git a/packages/builder/src/stores/builder/websocket.js b/packages/builder/src/stores/builder/websocket.js
index 7df5ab9adba..553d4cdd76e 100644
--- a/packages/builder/src/stores/builder/websocket.js
+++ b/packages/builder/src/stores/builder/websocket.js
@@ -9,6 +9,7 @@ import {
   snippets,
   datasources,
   tables,
+  roles,
 } from "stores/builder"
 import { get } from "svelte/store"
 import { auth, appsStore } from "stores/portal"
@@ -56,12 +57,18 @@ export const createBuilderWebsocket = appId => {
     datasources.replaceDatasource(id, datasource)
   })
 
+  // Role events
+  socket.onOther(BuilderSocketEvent.RoleChange, ({ id, role }) => {
+    roles.replaceRole(id, role)
+  })
+
   // Design section events
   socket.onOther(BuilderSocketEvent.ScreenChange, ({ id, screen }) => {
     screenStore.replace(id, screen)
   })
+
+  // App events
   socket.onOther(BuilderSocketEvent.AppMetadataChange, ({ metadata }) => {
-    //Sync app metadata across the stores
     appStore.syncMetadata(metadata)
     themeStore.syncMetadata(metadata)
     navigationStore.syncMetadata(metadata)
@@ -79,7 +86,7 @@ export const createBuilderWebsocket = appId => {
     }
   )
 
-  // Automations
+  // Automation events
   socket.onOther(BuilderSocketEvent.AutomationChange, ({ id, automation }) => {
     automationStore.actions.replace(id, automation)
   })
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index b6b9ac1a291..9e78aeadb2b 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -21,6 +21,7 @@ import {
 } from "@budibase/types"
 import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
+import { builderSocket } from "src/websockets"
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -130,6 +131,7 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
       },
     })
   }
+  builderSocket?.emitRoleUpdate(ctx, role)
 }
 
 export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {
@@ -165,6 +167,7 @@ export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {
   )
   ctx.message = `Role ${ctx.params.roleId} deleted successfully`
   ctx.status = 200
+  builderSocket?.emitRoleDeletion(ctx, role)
 }
 
 export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index 702a9410956..cf92d68ef3b 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -11,6 +11,7 @@ import {
   Screen,
   App,
   Automation,
+  Role,
 } from "@budibase/types"
 import { gridSocket } from "./index"
 import { clearLock, updateLock } from "../utilities/redis"
@@ -100,6 +101,20 @@ export default class BuilderSocket extends BaseSocket {
     })
   }
 
+  emitRoleUpdate(ctx: any, role: Role) {
+    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.RoleChange, {
+      id: role._id,
+      role,
+    })
+  }
+
+  emitRoleDeletion(ctx: any, role: Role) {
+    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.RoleChange, {
+      id: role._id,
+      role: null,
+    })
+  }
+
   emitTableUpdate(ctx: any, table: Table, options?: EmitOptions) {
     if (table.sourceId == null || table.sourceId === "") {
       throw new Error("Table sourceId is not set")
diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts
index 5a42fc56772..48667eed4ae 100644
--- a/packages/shared-core/src/constants/index.ts
+++ b/packages/shared-core/src/constants/index.ts
@@ -96,6 +96,7 @@ export enum BuilderSocketEvent {
   SelectResource = "SelectResource",
   AppPublishChange = "AppPublishChange",
   AutomationChange = "AutomationChange",
+  RoleChange = "RoleChange",
 }
 
 export const SocketSessionTTL = 60

From c634cfdeac9ecc3b0aba7460daf11af15c5c7d5f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 12:15:17 +0100
Subject: [PATCH 132/455] Simplify and rewrite some flow logic

---
 .../DatasourceNavigator.svelte                |   1 +
 .../backend/RoleEditor/Controls.svelte        |  25 +--
 .../backend/RoleEditor/RoleEdge.svelte        |   6 +-
 .../backend/RoleEditor/RoleEditor.svelte      | 151 ++++++++++++++++--
 .../backend/RoleEditor/RoleNode.svelte        |  13 +-
 .../components/backend/RoleEditor/layout.js   |  70 +-------
 .../app/[application]/data/roles.svelte       |   3 +
 packages/builder/src/stores/builder/roles.js  |   9 +-
 .../builder/src/stores/builder/websocket.js   |   2 +-
 9 files changed, 168 insertions(+), 112 deletions(-)

diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
index 621b8073a6e..d7a98c564af 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
@@ -81,6 +81,7 @@
     text="Manage roles"
     selected={$isActive("./roles")}
     on:click={() => $goto("./roles")}
+    selectedBy={$userSelectedResourceMap.roles}
   />
   {#each enrichedDataSources.filter(ds => ds.show) as datasource}
     <DatasourceNavItem
diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 1ddb856b86e..5e5b7bc6fc9 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -1,32 +1,15 @@
 <script>
-  import { Button, Helpers, ActionButton } from "@budibase/bbui"
+  import { Button, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow } from "@xyflow/svelte"
   import { getContext, tick } from "svelte"
-  import { autoLayout, roleToNode } from "./layout"
+  import { autoLayout } from "./layout"
   import { MaxAutoZoom, ZoomDuration } from "./constants"
-  import { getSequentialName } from "helpers/duplicate"
-  import { roles } from "stores/builder"
-  import { Roles } from "constants/backend"
 
-  const { nodes, edges } = getContext("flow")
+  const { nodes, edges, createRole } = getContext("flow")
   const flow = useSvelteFlow()
 
   const addRole = async () => {
-    const role = {
-      name: Helpers.uuid(),
-      uiMetadata: {
-        displayName: getSequentialName($roles, "New role ", {
-          getName: x => x.uiMetadata.displayName,
-        }),
-        color: "var(--spectrum-global-color-gray-700)",
-        description: "Custom role",
-      },
-      permissionId: "write",
-      inherits: Roles.BASIC,
-    }
-    const savedRole = await roles.save(role)
-    nodes.update(state => [...state, roleToNode(savedRole)])
-    await doAutoLayout()
+    await createRole()
   }
 
   const doAutoLayout = async () => {
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 73c43490178..f361f257ef6 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -6,7 +6,7 @@
     useSvelteFlow,
   } from "@xyflow/svelte"
   import { Icon, TooltipPosition } from "@budibase/bbui"
-  import { onMount } from "svelte"
+  import { onMount, getContext } from "svelte"
   import { roles } from "stores/builder"
 
   export let sourceX
@@ -20,6 +20,7 @@
   export let target
 
   const flow = useSvelteFlow()
+  const { updateRole } = getContext("flow")
 
   let edgeHovered = false
   let labelHovered = false
@@ -56,10 +57,11 @@
     edgeHovered = false
   }
 
-  const deleteEdge = () => {
+  const deleteEdge = async () => {
     flow.deleteElements({
       edges: [{ id }],
     })
+    await updateRole(target)
   }
 
   onMount(() => {
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index c1772c32bb4..0f5f55b1bfb 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -1,25 +1,157 @@
 <script>
-  import { Heading } from "@budibase/bbui"
+  import { Heading, Helpers } from "@budibase/bbui"
   import { writable } from "svelte/store"
-  import { SvelteFlow, Background, BackgroundVariant } from "@xyflow/svelte"
+  import {
+    SvelteFlow,
+    Background,
+    BackgroundVariant,
+    Position,
+  } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
   import RoleEdge from "./RoleEdge.svelte"
-  import { rolesToNodes, autoLayout } from "./layout"
-  import { onMount, setContext } from "svelte"
+  import { autoLayout } from "./layout"
+  import { setContext } from "svelte"
   import Controls from "./Controls.svelte"
   import { GridResolution, MaxAutoZoom } from "./constants"
+  import { roles } from "stores/builder"
+  import { Roles } from "constants/backend"
+  import { getSequentialName } from "helpers/duplicate"
 
   const nodes = writable([])
   const edges = writable([])
   const dragging = writable(false)
 
-  setContext("flow", { nodes, edges, dragging })
+  // Ensure role changes are synced with nodes and edges
+  $: handleExternalRoleChanges($roles)
 
-  onMount(() => {
-    const layout = autoLayout(rolesToNodes())
-    nodes.set(layout.nodes)
-    edges.set(layout.edges)
+  // Converts a role doc into a node structure
+  const roleToNode = role => ({
+    id: role._id,
+    sourcePosition: Position.Right,
+    targetPosition: Position.Left,
+    type: "role",
+    position: { x: 0, y: 0 },
+    data: {
+      ...role.uiMetadata,
+      custom: !role._id.match(/[A-Z]+/),
+    },
+  })
+
+  // Converts a node structure back into a role doc
+  const nodeToRole = node => {
+    const role = $roles.find(x => x._id === node.id)
+    const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
+    console.log(inherits)
+    return {
+      ...role,
+      // inherits,
+      uiMetadata: {
+        displayName: node.data.displayName,
+        color: node.data.color,
+        description: node.data.description,
+      },
+    }
+  }
+
+  // Builds a layout from an array of roles
+  const rolesToLayout = roles => {
+    let nodes = []
+    let edges = []
+    for (let role of roles.filter(role => role._id !== Roles.PUBLIC)) {
+      // Add node for this role
+      nodes.push(roleToNode(role))
+
+      // Add edges for this role
+      let inherits = []
+      if (role.inherits) {
+        inherits = Array.isArray(role.inherits)
+          ? role.inherits
+          : [role.inherits]
+      }
+      for (let sourceRole of inherits) {
+        if (!roles.some(x => x._id === sourceRole)) {
+          continue
+        }
+        edges.push({
+          id: `${sourceRole}-${role._id}`,
+          source: sourceRole,
+          target: role._id,
+        })
+      }
+    }
+    return {
+      nodes,
+      edges,
+    }
+  }
+
+  // Updates nodes and edges based on external changes to roles
+  const handleExternalRoleChanges = roles => {
+    const currentNodes = $nodes
+    const newLayout = autoLayout(rolesToLayout(roles))
+
+    // For roles we want to persist their current positions
+    nodes.set(
+      newLayout.nodes.map(node => {
+        const position = currentNodes.find(x => x.id === node.id)?.position
+        if (!position) {
+          return node
+        }
+        return { ...node, position }
+      })
+    )
+
+    // Edges can always be updated
+    edges.set(newLayout.edges)
+  }
+
+  // Creates a new role
+  const createRole = async () => {
+    const role = {
+      name: Helpers.uuid(),
+      uiMetadata: {
+        displayName: getSequentialName($roles, "New role ", {
+          getName: x => x.uiMetadata.displayName,
+        }),
+        color: "var(--spectrum-global-color-gray-700)",
+        description: "Custom role",
+      },
+      permissionId: "write",
+      inherits: Roles.BASIC,
+    }
+    await roles.save(role)
+  }
+
+  // Updates a role based on the latest flow data
+  const updateRole = async roleId => {
+    const node = $nodes.find(x => x.id === roleId)
+    if (node) {
+      const role = nodeToRole(node)
+      await roles.save(role)
+    }
+  }
+
+  // Deletes a role
+  const deleteRole = async roleId => {
+    const role = $roles.find(x => x._id === roleId)
+    if (role) {
+      roles.delete(role)
+    }
+  }
+
+  // Saves a new connection
+  const onConnect = async connection => {
+    await saveRole(connection.target)
+  }
+
+  setContext("flow", {
+    nodes,
+    edges,
+    dragging,
+    createRole,
+    updateRole,
+    deleteRole,
   })
 </script>
 
@@ -42,6 +174,7 @@
     defaultEdgeOptions={{ type: "role", animated: true, selectable: false }}
     onconnectstart={() => dragging.set(true)}
     onconnectend={() => dragging.set(false)}
+    onconnect={onConnect}
   >
     <Background variant={BackgroundVariant.Dots} />
     <Controls />
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 9a4f5f54fdb..e0297492375 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -11,13 +11,13 @@
   import { Roles } from "constants/backend"
   import { NodeWidth, NodeHeight, MaxAutoZoom, ZoomDuration } from "./constants"
   import { getContext, tick } from "svelte"
-  import { autoLayout, nodeToRole } from "./layout"
+  import { autoLayout } from "./layout"
   import { roles } from "stores/builder"
 
   export let data
   export let id
 
-  const { nodes, edges, dragging } = getContext("flow")
+  const { nodes, edges, dragging, updateRole, deleteRole } = getContext("flow")
   const flow = useSvelteFlow()
 
   let anchor
@@ -49,12 +49,7 @@
   }
 
   const deleteNode = async () => {
-    flow.deleteElements({
-      nodes: [{ id }],
-    })
-    await tick()
-    doAutoLayout()
-    await roles.delete(nodeToRole({ id, data }))
+    await deleteRole(id)
   }
 
   const openPopover = () => {
@@ -71,7 +66,7 @@
       color: tempColor,
     }
     flow.updateNodeData(id, newData)
-    await roles.save(nodeToRole({ id, data: newData }))
+    await updateRole(id)
   }
 
   const doAutoLayout = () => {
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index 3671cb6782f..d41a2b04e88 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -1,79 +1,11 @@
 import dagre from "@dagrejs/dagre"
 import { NodeWidth, NodeHeight, GridResolution } from "./constants"
 import { Position } from "@xyflow/svelte"
-import { roles } from "stores/builder"
 import { Roles } from "constants/backend"
-import { get } from "svelte/store"
 import { Helpers } from "@budibase/bbui"
 
-// Converts a role doc into a node structure
-export const roleToNode = role => ({
-  id: role._id,
-  sourcePosition: Position.Right,
-  targetPosition: Position.Left,
-  type: "role",
-  position: { x: 0, y: 0 },
-  data: {
-    ...role.uiMetadata,
-    custom: !role._id.match(/[A-Z]+/),
-  },
-})
-
-// Converts a node structure back into a role doc
-export const nodeToRole = node => {
-  const role = get(roles).find(x => x._id === node.id)
-  return {
-    ...role,
-    uiMetadata: {
-      displayName: node.data.displayName,
-      color: node.data.color,
-      description: node.data.description,
-    },
-  }
-}
-
-// Generates a flow compatible structure of nodes and edges from the current roles
-export const rolesToNodes = () => {
-  const ignoredRoles = [Roles.PUBLIC]
-  const $roles = get(roles)
-
-  let nodes = []
-  let edges = []
-
-  for (let role of $roles) {
-    if (ignoredRoles.includes(role._id)) {
-      continue
-    }
-
-    // Add node for this role
-    nodes.push(roleToNode(role))
-
-    // Add edges for this role
-    let inherits = []
-    if (role.inherits) {
-      inherits = Array.isArray(role.inherits) ? role.inherits : [role.inherits]
-    }
-    for (let sourceRole of inherits) {
-      // Ensure source role exists
-      if (!$roles.some(x => x._id === sourceRole)) {
-        continue
-      }
-      edges.push({
-        id: `${sourceRole}-${role._id}`,
-        source: sourceRole,
-        target: role._id,
-      })
-    }
-  }
-
-  return {
-    nodes,
-    edges,
-  }
-}
-
 // Updates positions of nodes and edges into a nice graph structure
-const dagreLayout = ({ nodes, edges }) => {
+export const dagreLayout = ({ nodes, edges }) => {
   const dagreGraph = new dagre.graphlib.Graph()
   dagreGraph.setDefaultEdgeLabel(() => ({}))
   dagreGraph.setGraph({
diff --git a/packages/builder/src/pages/builder/app/[application]/data/roles.svelte b/packages/builder/src/pages/builder/app/[application]/data/roles.svelte
index bea482d9d2c..01775777303 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/roles.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/roles.svelte
@@ -1,5 +1,8 @@
 <script>
   import RoleEditor from "components/backend/RoleEditor/RoleEditor.svelte"
+  import { builderStore } from "stores/builder"
+
+  builderStore.selectResource("roles")
 </script>
 
 <RoleEditor />
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index fcbceab1ce2..61f0c2b53db 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -1,4 +1,4 @@
-import { derived, writable } from "svelte/store"
+import { derived, writable, get } from "svelte/store"
 import { API } from "api"
 import { RoleUtils } from "@budibase/frontend-core"
 
@@ -56,6 +56,13 @@ export function createRolesStore() {
       }
     },
     replace: (roleId, role) => {
+      // Remove role_ prefix
+      if (roleId?.startsWith("role_")) {
+        roleId = roleId.replace("role_", "")
+      }
+      if (role?._id.startsWith("role_")) {
+        role._id = role._id.replace("role_", "")
+      }
       console.log("replace", roleId, role)
 
       // Handles external updates of roles
diff --git a/packages/builder/src/stores/builder/websocket.js b/packages/builder/src/stores/builder/websocket.js
index 553d4cdd76e..8a0d83abc10 100644
--- a/packages/builder/src/stores/builder/websocket.js
+++ b/packages/builder/src/stores/builder/websocket.js
@@ -59,7 +59,7 @@ export const createBuilderWebsocket = appId => {
 
   // Role events
   socket.onOther(BuilderSocketEvent.RoleChange, ({ id, role }) => {
-    roles.replaceRole(id, role)
+    roles.replace(id, role)
   })
 
   // Design section events

From d23d4156c34b4b8c391376a2603f8efb1c64025c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 12:17:12 +0100
Subject: [PATCH 133/455] Lint

---
 .../components/backend/RoleEditor/Controls.svelte   |  6 +-----
 .../components/backend/RoleEditor/RoleNode.svelte   | 13 +++++++------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 5e5b7bc6fc9..6a580726a65 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -8,10 +8,6 @@
   const { nodes, edges, createRole } = getContext("flow")
   const flow = useSvelteFlow()
 
-  const addRole = async () => {
-    await createRole()
-  }
-
   const doAutoLayout = async () => {
     const layout = autoLayout({ nodes: $nodes, edges: $edges })
     nodes.set(layout.nodes)
@@ -47,7 +43,7 @@
   <Button secondary on:click={doAutoLayout}>Auto layout</Button>
 </div>
 <div class="control bottom-right">
-  <Button icon="Add" cta on:click={addRole}>Add role</Button>
+  <Button icon="Add" cta on:click={createRole}>Add role</Button>
 </div>
 
 <style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index e0297492375..03890ac9b7f 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -10,7 +10,7 @@
   } from "@budibase/bbui"
   import { Roles } from "constants/backend"
   import { NodeWidth, NodeHeight, MaxAutoZoom, ZoomDuration } from "./constants"
-  import { getContext, tick } from "svelte"
+  import { getContext } from "svelte"
   import { autoLayout } from "./layout"
   import { roles } from "stores/builder"
 
@@ -48,10 +48,6 @@
     return null
   }
 
-  const deleteNode = async () => {
-    await deleteRole(id)
-  }
-
   const openPopover = () => {
     tempDisplayName = data.displayName
     tempDescription = data.description
@@ -92,7 +88,12 @@
       {#if data.custom}
         <div class="buttons">
           <Icon size="S" name="Edit" hoverable on:click={openPopover} />
-          <Icon size="S" name="Delete" hoverable on:click={deleteNode} />
+          <Icon
+            size="S"
+            name="Delete"
+            hoverable
+            on:click={() => deleteRole(id)}
+          />
         </div>
       {/if}
     </div>

From e0d2c706119d198b8b4cdcd4480a41ac38a1a696 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 16 Sep 2024 12:46:21 +0100
Subject: [PATCH 134/455] PR feedback

---
 .../builder/src/stores/builder/components.js  |  16 ++-
 .../src/components/app/DataProvider.svelte    |  11 +-
 .../src/components/CoreFilterBuilder.svelte   | 116 ++++++++++--------
 .../src/components/FilterField.svelte         |  19 +--
 .../frontend-core/src/fetch/CustomFetch.js    |   8 --
 packages/frontend-core/src/fetch/DataFetch.js |   2 +-
 packages/shared-core/src/filters.ts           |  50 +++++---
 packages/shared-core/src/utils.ts             |   6 +-
 8 files changed, 127 insertions(+), 101 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 0250ed6556e..6c2c438f0c7 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -206,9 +206,15 @@ export class ComponentStore extends BudiStore {
       setting?.type?.startsWith("filter")
     )
     for (let setting of filterableTypes || []) {
-      enrichedComponent[setting.key] = utils.processSearchFilters(
-        enrichedComponent[setting.key]
-      )
+      const isLegacy = Array.isArray(enrichedComponent[setting.key])
+
+      if (isLegacy) {
+        const processedSetting = utils.processSearchFilters(
+          enrichedComponent[setting.key]
+        )
+        enrichedComponent[setting.key] = processedSetting
+        migrated = true
+      }
     }
     return migrated
   }
@@ -575,9 +581,7 @@ export class ComponentStore extends BudiStore {
       const patchResult = patchFn(component, screen)
 
       // Post processing
-      const migrated = null
-
-      this.migrateSettings(component)
+      const migrated = this.migrateSettings(component)
 
       // Returning an explicit false signifies that we should skip this
       // update. If we migrated something, ensure we never skip.
diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index 7b90432bbf9..a2ba674bebb 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -19,9 +19,7 @@
   let queryExtensions = {}
   let localFilters
 
-  $: if (filter) {
-    localFilters = Helpers.cloneDeep(filter)
-  }
+  $: localFilters = filter ? Helpers.cloneDeep(filter) : null
 
   $: defaultQuery = QueryUtils.buildQuery(localFilters)
 
@@ -131,7 +129,7 @@
   }
 
   const extendQuery = (defaultQuery, extensions) => {
-    return {
+    const extended = {
       [LogicalOperator.AND]: {
         conditions: [
           ...(defaultQuery ? [defaultQuery] : []),
@@ -140,6 +138,11 @@
       },
       onEmptyFilter: EmptyFilterOption.RETURN_NONE,
     }
+
+    // If there are no conditions applied at all, clear the request.
+    return extended[LogicalOperator.AND]?.conditions?.length > 0
+      ? extended
+      : null
   }
 
   const setUpAutoRefresh = autoRefresh => {
diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index 405373e0a7a..c711a57e1c5 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -58,8 +58,13 @@
     return { value: entry, label: Helpers.capitalise(entry) }
   })
 
+  const onEmptyLabelling = {
+    [OnEmptyFilter.RETURN_ALL]: "All rows",
+    [OnEmptyFilter.RETURN_NONE]: "No rows",
+  }
+
   const onEmptyOptions = Object.values(OnEmptyFilter).map(entry => {
-    return { value: entry, label: Helpers.capitalise(entry) }
+    return { value: entry, label: onEmptyLabelling[entry] }
   })
 
   const context = getContext("context")
@@ -151,7 +156,7 @@
 
   const getGroupPrefix = groupIdx => {
     if (groupIdx == 0) {
-      return "Where"
+      return "When"
     }
     const operatorMapping = {
       [FilterOperator.ANY]: "or",
@@ -191,16 +196,17 @@
     if (targetFilter) {
       if (deleteFilter) {
         targetGroup.filters.splice(filterIdx, 1)
+
+        // Clear the group entirely if no valid filters remain
+        if (targetGroup.filters.length === 0) {
+          editable.groups.splice(groupIdx, 1)
+        }
       } else if (filter) {
         targetGroup.filters[filterIdx] = filter
       }
     } else if (targetGroup) {
       if (deleteGroup) {
-        if (editable.groups.length > 1) {
-          editable.groups.splice(groupIdx, 1)
-        } else {
-          editable = {}
-        }
+        editable.groups.splice(groupIdx, 1)
       } else if (addFilter) {
         targetGroup.filters.push({
           valueType: FilterValueType.VALUE,
@@ -239,6 +245,9 @@
       }
     }
 
+    // Set the request to null if the groups are emptied
+    editable = editable.groups.length ? editable : null
+
     dispatch("change", editable)
   }
 </script>
@@ -294,7 +303,7 @@
                       placeholder={false}
                     />
                   </span>
-                  <span>of the following filters are met:</span>
+                  <span>of the following filters are matched:</span>
                 </div>
                 <div class="group-actions">
                   <Icon
@@ -386,49 +395,51 @@
       {/if}
 
       <div class="filters-footer">
-        {#if behaviourFilters && editableFilters?.groups?.length}
-          <div class="empty-filter">
-            <span>Return</span>
-            <span class="empty-filter-picker">
-              <Select
-                value={editableFilters?.onEmptyFilter}
-                options={onEmptyOptions}
-                getOptionLabel={opt => opt.label}
-                getOptionValue={opt => opt.value}
-                on:change={e => {
-                  handleFilterChange({
-                    onEmptyFilter: e.detail,
-                  })
-                }}
-                placeholder={false}
+        <Layout noPadding>
+          {#if behaviourFilters && editableFilters?.groups?.length}
+            <div class="empty-filter">
+              <span>Return</span>
+              <span class="empty-filter-picker">
+                <Select
+                  value={editableFilters?.onEmptyFilter}
+                  options={onEmptyOptions}
+                  getOptionLabel={opt => opt.label}
+                  getOptionValue={opt => opt.value}
+                  on:change={e => {
+                    handleFilterChange({
+                      onEmptyFilter: e.detail,
+                    })
+                  }}
+                  placeholder={false}
+                />
+              </span>
+              <span>when all filters are empty</span>
+            </div>
+          {/if}
+          <div class="add-group">
+            <Button
+              icon="AddCircle"
+              size="M"
+              secondary
+              on:click={() => {
+                handleFilterChange({
+                  addGroup: true,
+                })
+              }}
+            >
+              Add filter group
+            </Button>
+            <a
+              href="https://docs.budibase.com/docs/searchfilter-data"
+              target="_blank"
+            >
+              <Icon
+                name="HelpOutline"
+                color="var(--spectrum-global-color-gray-600)"
               />
-            </span>
-            <span>when all filters are empty</span>
+            </a>
           </div>
-        {/if}
-        <div class="add-group">
-          <Button
-            icon="AddCircle"
-            size="M"
-            secondary
-            on:click={() => {
-              handleFilterChange({
-                addGroup: true,
-              })
-            }}
-          >
-            Add filter group
-          </Button>
-          <a
-            href="https://docs.budibase.com/docs/searchfilter-data"
-            target="_blank"
-          >
-            <Icon
-              name="HelpOutline"
-              color="var(--spectrum-global-color-gray-600)"
-            />
-          </a>
-        </div>
+        </Layout>
       </div>
     {:else}
       <Body size="S">None of the table column can be used for filtering.</Body>
@@ -446,7 +457,7 @@
   .empty-filter,
   .group-options {
     display: flex;
-    gap: var(--spacing-xl);
+    gap: var(--spacing-m);
     align-items: center;
   }
 
@@ -461,14 +472,13 @@
   }
 
   .empty-filter-picker {
-    width: 80px;
+    width: 92px;
   }
 
   .filter-groups {
     display: flex;
     flex-direction: column;
     gap: var(--spacing-xl);
-    /* overflow-x: scroll; */
   }
 
   .group {
@@ -489,7 +499,7 @@
   .filter {
     display: grid;
     gap: var(--spacing-l);
-    grid-template-columns: minmax(150px, 1fr) 170px 200px 40px 40px;
+    grid-template-columns: minmax(150px, 1fr) 170px minmax(200px, 1fr) 40px 40px;
   }
 
   .filters-footer {
diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte
index 03a09c9db0c..c763194d693 100644
--- a/packages/frontend-core/src/components/FilterField.svelte
+++ b/packages/frontend-core/src/components/FilterField.svelte
@@ -20,12 +20,9 @@
   export let bindings = []
   export let allowBindings = false
   export let schemaFields
-
-  // Export these to another field type
   export let panel
   export let toReadable
   export let toRuntime
-  // Only required if you're in the builder, which we aint.
 
   const dispatch = createEventDispatcher()
   const { OperatorOptions, FilterValueType } = Constants
@@ -48,17 +45,21 @@
     return schemaFields.find(field => field.name === filter.field)
   }
 
+  const drawerOnChange = e => {
+    drawerValue = e.detail
+  }
+
   const onChange = e => {
     fieldValue = e.detail
     dispatch("change", {
-      value: toRuntime(bindings, fieldValue),
+      value: toRuntime ? toRuntime(bindings, fieldValue) : fieldValue,
     })
   }
 
-  const onConfirm = () => {
+  const onConfirmBinding = () => {
     dispatch("change", {
-      value: fieldValue,
-      valueType: fieldValue ? FilterValueType.BINDING : FilterValueType.VALUE,
+      value: toRuntime ? toRuntime(bindings, drawerValue) : drawerValue,
+      valueType: drawerValue ? FilterValueType.BINDING : FilterValueType.VALUE,
     })
   }
 
@@ -139,7 +140,7 @@
       cta
       slot="buttons"
       on:click={() => {
-        onConfirm()
+        onConfirmBinding()
         bindingDrawer.hide()
       }}
     >
@@ -153,7 +154,7 @@
       allowJS
       allowHelpers
       allowHBS
-      on:change={onChange}
+      on:change={drawerOnChange}
       {bindings}
     />
   </Drawer>
diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index a8b14a0809e..fc62d790e2a 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -1,14 +1,6 @@
 import DataFetch from "./DataFetch.js"
 
 export default class CustomFetch extends DataFetch {
-  determineFeatureFlags() {
-    return {
-      supportsSearch: false,
-      supportsSort: false,
-      supportsPagination: false,
-    }
-  }
-
   // Gets the correct Budibase type for a JS value
   getType(value) {
     if (value == null) {
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index 18dc72bf24d..4f51cdad71e 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -183,7 +183,7 @@ export default class DataFetch {
     }
 
     // Build the query
-    let query = this.options.query || null
+    let query = this.options.query
 
     if (!query && this.features.supportsSearch) {
       query = buildQuery(filter || defaultQuery)
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 431f6836da3..360d3ae5127 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -307,11 +307,11 @@ export class ColumnSplitter {
 }
 
 /**
- * Builds a JSON query from the filter structure generated in the builder
+ * Builds a JSON query from the filter a SearchFilter definition
  * @param filter the builder filter structure
  */
 
-const builderFilter = (expression: SearchFilter) => {
+const buildCondition = (expression: SearchFilter) => {
   // Filter body
   let query: SearchFilters = {
     string: {},
@@ -378,13 +378,15 @@ const builderFilter = (expression: SearchFilter) => {
     value = `${value}`?.toLowerCase() === "true"
   }
   if (
-    ["contains", "notContains", "containsAny"].includes(operator) &&
+    ["contains", "notContains", "containsAny"].includes(
+      operator.toLocaleString()
+    ) &&
     type === "array" &&
     typeof value === "string"
   ) {
     value = value.split(",")
   }
-  if (operator.startsWith("range") && query.range) {
+  if (operator.toLocaleString().startsWith("range") && query.range) {
     const minint =
       SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
         ?.min || Number.MIN_SAFE_INTEGER
@@ -497,13 +499,15 @@ export const buildQueryLegacy = (
       value = `${value}`?.toLowerCase() === "true"
     }
     if (
-      ["contains", "notContains", "containsAny"].includes(operator) &&
+      ["contains", "notContains", "containsAny"].includes(
+        operator.toLocaleString()
+      ) &&
       type === "array" &&
       typeof value === "string"
     ) {
       value = value.split(",")
     }
-    if (operator.startsWith("range") && query.range) {
+    if (operator.toLocaleString().startsWith("range") && query.range) {
       const minint =
         SqlNumberTypeRangeMap[
           externalType as keyof typeof SqlNumberTypeRangeMap
@@ -555,28 +559,40 @@ export const buildQueryLegacy = (
   return query
 }
 
+/**
+ * Converts a **SearchFilterGroup** filter definition into a grouped
+ * search query of type **SearchFilters**
+ *
+ * Legacy support remains for the old **SearchFilter[]** format.
+ * These will be migrated to an appropriate **SearchFilters** object, if encountered
+ *
+ * @param filter
+ *
+ * @returns {SearchFilters}
+ */
+
 export const buildQuery = (
   filter?: SearchFilterGroup | SearchFilter[]
 ): SearchFilters | undefined => {
-  if (!filter) {
-    return
-  }
-
-  const parsedFilter = processSearchFilters(filter)
+  const parsedFilter: SearchFilterGroup | undefined =
+    processSearchFilters(filter)
 
   if (!parsedFilter) {
     return
   }
 
-  const operatorMap = {
-    [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
-    [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
-  }
+  const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
+    {
+      [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
+      [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
+    }
 
   const globalOnEmpty = parsedFilter.onEmptyFilter
     ? parsedFilter.onEmptyFilter
     : null
-  const globalOperator = operatorMap[parsedFilter.logicalOperator]
+
+  const globalOperator: LogicalOperator =
+    operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
 
   const coreRequest: SearchFilters = {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
@@ -585,7 +601,7 @@ export const buildQuery = (
         return {
           [operatorMap[group.logicalOperator]]: {
             conditions: group.filters
-              ?.map(x => builderFilter(x))
+              ?.map(x => buildCondition(x))
               .filter(filter => filter),
           },
         }
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 027db8e914e..f37d50d1720 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -97,8 +97,8 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
  * @param {SearchFilter[] | SearchFilterGroup} filters
  */
 export const processSearchFilters = (
-  filters: SearchFilter[] | SearchFilterGroup
-) => {
+  filters: SearchFilter[] | SearchFilterGroup | undefined
+): SearchFilterGroup | undefined => {
   if (!filters) {
     return
   }
@@ -182,7 +182,7 @@ export const processSearchFilters = (
 
     return migratedSetting
   } else if (!filters?.groups) {
-    return null
+    return
   }
   return filters
 }

From 22eafa892c9a8f11512058f54f19ba9780f8f1a0 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 16 Sep 2024 13:02:35 +0100
Subject: [PATCH 135/455] PR feedback

---
 packages/client/src/components/app/DataProvider.svelte       | 5 +----
 .../src/components/app/dynamic-filter/DynamicFilter.svelte   | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index a2ba674bebb..c05d4de74e0 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -17,11 +17,8 @@
 
   let interval
   let queryExtensions = {}
-  let localFilters
 
-  $: localFilters = filter ? Helpers.cloneDeep(filter) : null
-
-  $: defaultQuery = QueryUtils.buildQuery(localFilters)
+  $: defaultQuery = QueryUtils.buildQuery(filter)
 
   // We need to manage our lucene query manually as we want to allow components
   // to extend it
diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
index a1b64b181a9..4ab01de17d4 100644
--- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
+++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
@@ -95,7 +95,7 @@
   }
 
   const updateQuery = () => {
-    filters = Helpers.cloneDeep(editableFilters)
+    filters = editableFilters
   }
 
   onDestroy(() => {

From d61594d74e6d8ad1cbcbbac8bba7d1190c69bc2e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 13:31:36 +0100
Subject: [PATCH 136/455] Refactor RBAC flow and use selected states more

---
 .../backend/RoleEditor/Controls.svelte        |  14 +-
 .../backend/RoleEditor/RoleEdge.svelte        |  56 ++--
 .../backend/RoleEditor/RoleEditor.svelte      | 229 +--------------
 .../backend/RoleEditor/RoleFlow.svelte        | 271 ++++++++++++++++++
 .../backend/RoleEditor/RoleNode.svelte        |  37 ++-
 packages/builder/src/stores/builder/roles.js  |   7 +-
 6 files changed, 321 insertions(+), 293 deletions(-)
 create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 6a580726a65..9cadbb1be13 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -8,15 +8,15 @@
   const { nodes, edges, createRole } = getContext("flow")
   const flow = useSvelteFlow()
 
-  const doAutoLayout = async () => {
+  const addRole = async () => {
+    await createRole()
+    doAutoLayout()
+  }
+
+  const doAutoLayout = () => {
     const layout = autoLayout({ nodes: $nodes, edges: $edges })
     nodes.set(layout.nodes)
     edges.set(layout.edges)
-    await tick()
-    flow.fitView({
-      maxZoom: MaxAutoZoom,
-      duration: ZoomDuration,
-    })
   }
 </script>
 
@@ -43,7 +43,7 @@
   <Button secondary on:click={doAutoLayout}>Auto layout</Button>
 </div>
 <div class="control bottom-right">
-  <Button icon="Add" cta on:click={createRole}>Add role</Button>
+  <Button icon="Add" cta on:click={addRole}>Add role</Button>
 </div>
 
 <style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index f361f257ef6..67e19c48e9a 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -20,13 +20,12 @@
   export let target
 
   const flow = useSvelteFlow()
-  const { updateRole } = getContext("flow")
+  const { updateRole, selectedNode } = getContext("flow")
 
-  let edgeHovered = false
-  let labelHovered = false
+  let iconHovered = false
 
-  $: hovered = edgeHovered || labelHovered
-  $: edgeClasses = getEdgeClasses(hovered, labelHovered)
+  $: active = source === $selectedNode || target === $selectedNode
+  $: edgeClasses = getEdgeClasses(active, iconHovered)
   $: [edgePath, labelX, labelY] = getBezierPath({
     sourceX,
     sourceY,
@@ -42,41 +41,19 @@
       ? `Stop ${targetRole.uiMetadata.displayName} from inheriting ${sourceRole.uiMetadata.displayName}`
       : null
 
-  const getEdgeClasses = (hovered, labelHovered) => {
+  const getEdgeClasses = (active, iconHovered) => {
     let classes = ""
-    if (hovered) classes += `hovered `
-    if (labelHovered) classes += `delete `
+    if (active) classes += `active `
+    if (iconHovered) classes += `delete `
     return classes
   }
 
-  const onEdgeMouseOver = () => {
-    edgeHovered = true
-  }
-
-  const onEdgeMouseOut = () => {
-    edgeHovered = false
-  }
-
   const deleteEdge = async () => {
     flow.deleteElements({
       edges: [{ id }],
     })
     await updateRole(target)
   }
-
-  onMount(() => {
-    const edge = document.querySelector(`.svelte-flow__edge[data-id="${id}"]`)
-    if (edge) {
-      edge.addEventListener("mouseover", onEdgeMouseOver)
-      edge.addEventListener("mouseout", onEdgeMouseOut)
-    }
-    return () => {
-      if (edge) {
-        edge.removeEventListener("mouseover", onEdgeMouseOver)
-        edge.removeEventListener("mouseout", onEdgeMouseOut)
-      }
-    }
-  })
 </script>
 
 <BaseEdge path={edgePath} class={edgeClasses} />
@@ -87,10 +64,10 @@
   <div
     style:transform="translate(-50%, -50%) translate({labelX}px,{labelY}px)"
     class="edge-label nodrag nopan"
-    class:hovered
+    class:active
     on:click={deleteEdge}
-    on:mouseover={() => (labelHovered = true)}
-    on:mouseout={() => (labelHovered = false)}
+    on:mouseover={() => (iconHovered = true)}
+    on:mouseout={() => (iconHovered = false)}
   >
     <Icon name="Delete" {tooltip} tooltipPosition={TooltipPosition.Top} />
   </div>
@@ -101,11 +78,11 @@
     position: absolute;
     padding: 8px;
     opacity: 0;
-    pointer-events: all;
+    pointer-events: none;
   }
-  .edge-label:hover,
-  .edge-label.hovered {
+  .edge-label.active {
     opacity: 1;
+    pointer-events: all;
     cursor: pointer;
   }
   .edge-label:hover :global(.spectrum-Icon) {
@@ -116,13 +93,12 @@
     color: var(--spectrum-global-color-gray-600);
   }
   .edge-label :global(svg) {
-    padding: 8px;
+    padding: 4px;
   }
-  :global(.svelte-flow__edge:hover .svelte-flow__edge-path),
-  :global(.svelte-flow__edge-path.hovered) {
+  :global(.svelte-flow__edge-path.active) {
     stroke: var(--spectrum-global-color-blue-400);
   }
-  :global(.svelte-flow__edge-path.hovered.delete) {
+  :global(.svelte-flow__edge-path.active.delete) {
     stroke: var(--spectrum-global-color-red-400);
   }
 </style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
index 0f5f55b1bfb..6169013d129 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte
@@ -1,227 +1,8 @@
 <script>
-  import { Heading, Helpers } from "@budibase/bbui"
-  import { writable } from "svelte/store"
-  import {
-    SvelteFlow,
-    Background,
-    BackgroundVariant,
-    Position,
-  } from "@xyflow/svelte"
-  import "@xyflow/svelte/dist/style.css"
-  import RoleNode from "./RoleNode.svelte"
-  import RoleEdge from "./RoleEdge.svelte"
-  import { autoLayout } from "./layout"
-  import { setContext } from "svelte"
-  import Controls from "./Controls.svelte"
-  import { GridResolution, MaxAutoZoom } from "./constants"
-  import { roles } from "stores/builder"
-  import { Roles } from "constants/backend"
-  import { getSequentialName } from "helpers/duplicate"
-
-  const nodes = writable([])
-  const edges = writable([])
-  const dragging = writable(false)
-
-  // Ensure role changes are synced with nodes and edges
-  $: handleExternalRoleChanges($roles)
-
-  // Converts a role doc into a node structure
-  const roleToNode = role => ({
-    id: role._id,
-    sourcePosition: Position.Right,
-    targetPosition: Position.Left,
-    type: "role",
-    position: { x: 0, y: 0 },
-    data: {
-      ...role.uiMetadata,
-      custom: !role._id.match(/[A-Z]+/),
-    },
-  })
-
-  // Converts a node structure back into a role doc
-  const nodeToRole = node => {
-    const role = $roles.find(x => x._id === node.id)
-    const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
-    console.log(inherits)
-    return {
-      ...role,
-      // inherits,
-      uiMetadata: {
-        displayName: node.data.displayName,
-        color: node.data.color,
-        description: node.data.description,
-      },
-    }
-  }
-
-  // Builds a layout from an array of roles
-  const rolesToLayout = roles => {
-    let nodes = []
-    let edges = []
-    for (let role of roles.filter(role => role._id !== Roles.PUBLIC)) {
-      // Add node for this role
-      nodes.push(roleToNode(role))
-
-      // Add edges for this role
-      let inherits = []
-      if (role.inherits) {
-        inherits = Array.isArray(role.inherits)
-          ? role.inherits
-          : [role.inherits]
-      }
-      for (let sourceRole of inherits) {
-        if (!roles.some(x => x._id === sourceRole)) {
-          continue
-        }
-        edges.push({
-          id: `${sourceRole}-${role._id}`,
-          source: sourceRole,
-          target: role._id,
-        })
-      }
-    }
-    return {
-      nodes,
-      edges,
-    }
-  }
-
-  // Updates nodes and edges based on external changes to roles
-  const handleExternalRoleChanges = roles => {
-    const currentNodes = $nodes
-    const newLayout = autoLayout(rolesToLayout(roles))
-
-    // For roles we want to persist their current positions
-    nodes.set(
-      newLayout.nodes.map(node => {
-        const position = currentNodes.find(x => x.id === node.id)?.position
-        if (!position) {
-          return node
-        }
-        return { ...node, position }
-      })
-    )
-
-    // Edges can always be updated
-    edges.set(newLayout.edges)
-  }
-
-  // Creates a new role
-  const createRole = async () => {
-    const role = {
-      name: Helpers.uuid(),
-      uiMetadata: {
-        displayName: getSequentialName($roles, "New role ", {
-          getName: x => x.uiMetadata.displayName,
-        }),
-        color: "var(--spectrum-global-color-gray-700)",
-        description: "Custom role",
-      },
-      permissionId: "write",
-      inherits: Roles.BASIC,
-    }
-    await roles.save(role)
-  }
-
-  // Updates a role based on the latest flow data
-  const updateRole = async roleId => {
-    const node = $nodes.find(x => x.id === roleId)
-    if (node) {
-      const role = nodeToRole(node)
-      await roles.save(role)
-    }
-  }
-
-  // Deletes a role
-  const deleteRole = async roleId => {
-    const role = $roles.find(x => x._id === roleId)
-    if (role) {
-      roles.delete(role)
-    }
-  }
-
-  // Saves a new connection
-  const onConnect = async connection => {
-    await saveRole(connection.target)
-  }
-
-  setContext("flow", {
-    nodes,
-    edges,
-    dragging,
-    createRole,
-    updateRole,
-    deleteRole,
-  })
+  import { SvelteFlowProvider } from "@xyflow/svelte"
+  import RoleFlow from "./RoleFlow.svelte"
 </script>
 
-<div class="title">
-  <div class="heading">
-    <Heading size="S">Manage roles</Heading>
-  </div>
-  <div class="description">Roles inherit permissions from each other.</div>
-</div>
-<div class="flow">
-  <SvelteFlow
-    fitView
-    {nodes}
-    {edges}
-    snapGrid={[GridResolution, GridResolution]}
-    nodeTypes={{ role: RoleNode }}
-    edgeTypes={{ role: RoleEdge }}
-    proOptions={{ hideAttribution: true }}
-    fitViewOptions={{ maxZoom: MaxAutoZoom }}
-    defaultEdgeOptions={{ type: "role", animated: true, selectable: false }}
-    onconnectstart={() => dragging.set(true)}
-    onconnectend={() => dragging.set(false)}
-    onconnect={onConnect}
-  >
-    <Background variant={BackgroundVariant.Dots} />
-    <Controls />
-  </SvelteFlow>
-</div>
-
-<style>
-  .heading {
-    border-bottom: 1px solid var(--spectrum-global-color-gray-300);
-    margin-bottom: 20px;
-    padding-bottom: 12px;
-  }
-  .description {
-    color: var(--spectrum-global-color-gray-600);
-    margin-bottom: calc(20px - var(--spacing-l));
-  }
-  .flow {
-    flex: 1 1 auto;
-    border-radius: 8px;
-    overflow: hidden;
-    position: relative;
-    --background-color: var(--spectrum-global-color-gray-50);
-    --node-background: var(--spectrum-global-color-gray-100);
-    --node-background-hover: var(--spectrum-global-color-gray-300);
-    --border-color: var(--spectrum-global-color-gray-300);
-    --edge-color: var(--spectrum-global-color-gray-500);
-    --handle-color: var(--spectrum-global-color-gray-600);
-    --selected-color: var(--spectrum-global-color-blue-400);
-  }
-
-  /* Customise svelte-flow theme */
-  .flow :global(.svelte-flow) {
-    /* Panel */
-    --xy-background-color: var(--background-color);
-
-    /* Controls */
-    --xy-controls-button-background-color: var(--node-background);
-    --xy-controls-button-background-color-hover: var(--node-background-hover);
-    --xy-controls-button-border-color: var(--border-color);
-
-    /* Handles */
-    --xy-handle-background-color: var(--handle-color);
-    --xy-handle-border-color: var(--handle-color);
-
-    /* Edges */
-    --xy-edge-stroke: var(--edge-color);
-    --xy-edge-stroke-selected: var(--edge-color);
-    --xy-edge-stroke-width: 2px;
-  }
-</style>
+<SvelteFlowProvider>
+  <RoleFlow />
+</SvelteFlowProvider>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
new file mode 100644
index 00000000000..ae3da3cfdb2
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -0,0 +1,271 @@
+<script>
+  import { Heading, Helpers } from "@budibase/bbui"
+  import { writable, derived } from "svelte/store"
+  import {
+    SvelteFlow,
+    Background,
+    BackgroundVariant,
+    Position,
+    useSvelteFlow,
+  } from "@xyflow/svelte"
+  import "@xyflow/svelte/dist/style.css"
+  import RoleNode from "./RoleNode.svelte"
+  import RoleEdge from "./RoleEdge.svelte"
+  import { autoLayout } from "./layout"
+  import { setContext } from "svelte"
+  import Controls from "./Controls.svelte"
+  import { GridResolution, MaxAutoZoom } from "./constants"
+  import { roles } from "stores/builder"
+  import { Roles } from "constants/backend"
+  import { getSequentialName } from "helpers/duplicate"
+  import { meta } from "@roxi/routify"
+
+  const flow = useSvelteFlow()
+  const nodes = writable([])
+  const edges = writable([])
+  const dragging = writable(false)
+  const selectedNode = derived(
+    nodes,
+    $nodes => $nodes.find(x => x.selected)?.id
+  )
+
+  // Ensure role changes are synced with nodes and edges
+  $: handleExternalRoleChanges($roles)
+
+  // Converts a role doc into a node structure
+  const roleToNode = role => ({
+    id: role._id,
+    sourcePosition: Position.Right,
+    targetPosition: Position.Left,
+    type: "role",
+    position: { x: 0, y: 0 },
+    data: {
+      ...role.uiMetadata,
+      custom: !role._id.match(/[A-Z]+/),
+    },
+  })
+
+  // Converts a node structure back into a role doc
+  const nodeToRole = node => {
+    const role = $roles.find(x => x._id === node.id)
+    const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
+    console.log(inherits)
+    return {
+      ...role,
+      // inherits,
+      uiMetadata: {
+        displayName: node.data.displayName,
+        color: node.data.color,
+        description: node.data.description,
+      },
+    }
+  }
+
+  // Builds a layout from an array of roles
+  const rolesToLayout = roles => {
+    let nodes = []
+    let edges = []
+    for (let role of roles.filter(role => role._id !== Roles.PUBLIC)) {
+      // Add node for this role
+      nodes.push(roleToNode(role))
+
+      // Add edges for this role
+      let inherits = []
+      if (role.inherits) {
+        inherits = Array.isArray(role.inherits)
+          ? role.inherits
+          : [role.inherits]
+      }
+      for (let sourceRole of inherits) {
+        if (!roles.some(x => x._id === sourceRole)) {
+          continue
+        }
+        edges.push({
+          id: `${sourceRole}-${role._id}`,
+          source: sourceRole,
+          target: role._id,
+        })
+      }
+    }
+    return {
+      nodes,
+      edges,
+    }
+  }
+
+  // Updates nodes and edges based on external changes to roles
+  const handleExternalRoleChanges = roles => {
+    const currentNodes = $nodes
+    const newLayout = autoLayout(rolesToLayout(roles))
+
+    // For nodes we want to persist some metadata
+    nodes.set(
+      newLayout.nodes.map(node => {
+        const currentNode = currentNodes.find(x => x.id === node.id)
+        if (!currentNode) {
+          return node
+        }
+        return {
+          ...node,
+          position: currentNode.position || node.position,
+          selected: currentNode.selected || node.selected,
+        }
+      })
+    )
+
+    // Edges can always be updated
+    edges.set(newLayout.edges)
+  }
+
+  // Creates a new role
+  const createRole = async () => {
+    const newRole = {
+      name: Helpers.uuid(),
+      uiMetadata: {
+        displayName: getSequentialName($roles, "New role ", {
+          getName: x => x.uiMetadata.displayName,
+        }),
+        color: "var(--spectrum-global-color-gray-700)",
+        description: "Custom role",
+      },
+      permissionId: "write",
+      inherits: Roles.BASIC,
+    }
+
+    // Immediate state update
+    const newNode = {
+      ...roleToNode({ ...newRole, _id: newRole.name }),
+      selected: true,
+    }
+    const layout = autoLayout({
+      nodes: [...$nodes.map(node => ({ ...node, selected: false })), newNode],
+      edges: $edges,
+    })
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+
+    // Actually create role
+    await roles.save(newRole)
+  }
+
+  // Updates a role with new metadata
+  const updateRole = async (roleId, metadata) => {
+    // Don't update builtins
+    const node = $nodes.find(x => x.id === roleId)
+    if (!node || !node.data.custom) {
+      return
+    }
+
+    // Immediate state update
+    if (metadata) {
+      flow.updateNodeData(roleId, metadata)
+    }
+
+    // Actually save changes
+    const role = nodeToRole(node)
+    await roles.save(role)
+  }
+
+  // Deletes a role
+  const deleteRole = async roleId => {
+    // Immediate state update
+    const layout = autoLayout({
+      nodes: $nodes.filter(x => x.id !== roleId),
+      edges: $edges.filter(x => x.source !== roleId && x.target !== roleId),
+    })
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+
+    // Actually delete role
+    const role = $roles.find(x => x._id === roleId)
+    if (role) {
+      roles.delete(role)
+    }
+  }
+
+  // Saves a new connection
+  const onConnect = async connection => {
+    await updateRole(connection.target)
+  }
+
+  setContext("flow", {
+    nodes,
+    edges,
+    dragging,
+    selectedNode,
+    createRole,
+    updateRole,
+    deleteRole,
+  })
+</script>
+
+<div class="title">
+  <div class="heading">
+    <Heading size="S">Manage roles</Heading>
+  </div>
+  <div class="description">Roles inherit permissions from each other.</div>
+</div>
+<div class="flow">
+  <SvelteFlow
+    fitView
+    {nodes}
+    {edges}
+    snapGrid={[GridResolution, GridResolution]}
+    nodeTypes={{ role: RoleNode }}
+    edgeTypes={{ role: RoleEdge }}
+    proOptions={{ hideAttribution: true }}
+    fitViewOptions={{ maxZoom: MaxAutoZoom }}
+    defaultEdgeOptions={{ type: "role", animated: true, selectable: false }}
+    onconnectstart={() => dragging.set(true)}
+    onconnectend={() => dragging.set(false)}
+    onconnect={onConnect}
+  >
+    <Background variant={BackgroundVariant.Dots} />
+    <Controls />
+  </SvelteFlow>
+</div>
+
+<style>
+  .heading {
+    border-bottom: 1px solid var(--spectrum-global-color-gray-300);
+    margin-bottom: 20px;
+    padding-bottom: 12px;
+  }
+  .description {
+    color: var(--spectrum-global-color-gray-600);
+    margin-bottom: calc(20px - var(--spacing-l));
+  }
+  .flow {
+    flex: 1 1 auto;
+    border-radius: 8px;
+    overflow: hidden;
+    position: relative;
+    --background-color: var(--spectrum-global-color-gray-50);
+    --node-background: var(--spectrum-global-color-gray-100);
+    --node-background-hover: var(--spectrum-global-color-gray-300);
+    --border-color: var(--spectrum-global-color-gray-300);
+    --edge-color: var(--spectrum-global-color-gray-500);
+    --handle-color: var(--spectrum-global-color-gray-600);
+    --selected-color: var(--spectrum-global-color-blue-400);
+  }
+
+  /* Customise svelte-flow theme */
+  .flow :global(.svelte-flow) {
+    /* Panel */
+    --xy-background-color: var(--background-color);
+
+    /* Controls */
+    --xy-controls-button-background-color: var(--node-background);
+    --xy-controls-button-background-color-hover: var(--node-background-hover);
+    --xy-controls-button-border-color: var(--border-color);
+
+    /* Handles */
+    --xy-handle-background-color: var(--handle-color);
+    --xy-handle-border-color: var(--handle-color);
+
+    /* Edges */
+    --xy-edge-stroke: var(--edge-color);
+    --xy-edge-stroke-selected: var(--edge-color);
+    --xy-edge-stroke-width: 2px;
+  }
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 03890ac9b7f..9a65e4dad58 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Handle, Position, useSvelteFlow } from "@xyflow/svelte"
+  import { Handle, Position, useSvelteFlow, NodeToolbar } from "@xyflow/svelte"
   import {
     Icon,
     Input,
@@ -16,8 +16,9 @@
 
   export let data
   export let id
+  export let selected
 
-  const { nodes, edges, dragging, updateRole, deleteRole } = getContext("flow")
+  const { dragging, updateRole, deleteRole } = getContext("flow")
   const flow = useSvelteFlow()
 
   let anchor
@@ -48,34 +49,31 @@
     return null
   }
 
-  const openPopover = () => {
+  const openPopover = e => {
+    e.stopPropagation()
     tempDisplayName = data.displayName
     tempDescription = data.description
     tempColor = data.color
     modal.show()
   }
 
-  const saveChanges = async () => {
-    const newData = {
+  const saveChanges = () => {
+    updateRole(id, {
       displayName: tempDisplayName,
       description: tempDescription,
       color: tempColor,
-    }
-    flow.updateNodeData(id, newData)
-    await updateRole(id)
+    })
   }
 
-  const doAutoLayout = () => {
-    const layout = autoLayout({ nodes: $nodes, edges: $edges })
-    nodes.set(layout.nodes)
-    edges.set(layout.edges)
-    flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })
+  const handleDelete = async e => {
+    e.stopPropagation()
+    await deleteRole(id)
   }
 </script>
 
 <div
   class="node"
-  class:selected={false}
+  class:selected
   style={`--color:${data.color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
   bind:this={anchor}
 >
@@ -88,12 +86,7 @@
       {#if data.custom}
         <div class="buttons">
           <Icon size="S" name="Edit" hoverable on:click={openPopover} />
-          <Icon
-            size="S"
-            name="Delete"
-            hoverable
-            on:click={() => deleteRole(id)}
-          />
+          <Icon size="S" name="Delete" hoverable on:click={handleDelete} />
         </div>
       {/if}
     </div>
@@ -153,9 +146,11 @@
     display: flex;
     flex-direction: row;
     box-sizing: border-box;
+    cursor: pointer;
   }
   .node.selected {
     background: var(--spectrum-global-color-blue-100);
+    cursor: grab;
   }
   .color {
     border-top-left-radius: 4px;
@@ -207,7 +202,7 @@
     color: var(--spectrum-global-color-gray-600);
     font-size: 12px;
   }
-  .node:hover .buttons {
+  .node.selected .buttons {
     display: flex;
   }
   .node :global(.svelte-flow__handle) {
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index 61f0c2b53db..cb86991a9a4 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -23,7 +23,12 @@ export function createRolesStore() {
       roles.sort((a, b) => {
         const priorityA = RoleUtils.getRolePriority(a._id)
         const priorityB = RoleUtils.getRolePriority(b._id)
-        return priorityA > priorityB ? -1 : 1
+        if (priorityA !== priorityB) {
+          return priorityA > priorityB ? -1 : 1
+        }
+        const nameA = a.uiMetadata?.displayName || a.name
+        const nameB = b.uiMetadata?.displayName || b.name
+        return nameA < nameB ? -1 : 1
       })
     )
   }

From e7916c55f7dca49931c53caf4e09360485ce7635 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 13:42:47 +0100
Subject: [PATCH 137/455] Some style updates

---
 .../backend/RoleEditor/RoleEdge.svelte        |  7 +++-
 .../backend/RoleEditor/RoleNode.svelte        | 32 ++++++++-----------
 .../components/backend/RoleEditor/layout.js   |  2 +-
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 67e19c48e9a..e8cfb8a5145 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -69,7 +69,12 @@
     on:mouseover={() => (iconHovered = true)}
     on:mouseout={() => (iconHovered = false)}
   >
-    <Icon name="Delete" {tooltip} tooltipPosition={TooltipPosition.Top} />
+    <Icon
+      name="Delete"
+      size="S"
+      {tooltip}
+      tooltipPosition={TooltipPosition.Top}
+    />
   </div>
 </EdgeLabelRenderer>
 
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 9a65e4dad58..1615a355fa0 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -79,16 +79,8 @@
 >
   <div class="color" />
   <div class="content">
-    <div class="title">
-      <div class="name">
-        {data.displayName}
-      </div>
-      {#if data.custom}
-        <div class="buttons">
-          <Icon size="S" name="Edit" hoverable on:click={openPopover} />
-          <Icon size="S" name="Delete" hoverable on:click={handleDelete} />
-        </div>
-      {/if}
+    <div class="name">
+      {data.displayName}
     </div>
     {#if data.description}
       <div class="description" title={data.description}>
@@ -96,6 +88,12 @@
       </div>
     {/if}
   </div>
+  {#if data.custom}
+    <div class="buttons">
+      <Icon size="S" name="Edit" hoverable on:click={openPopover} />
+      <Icon size="S" name="Delete" hoverable on:click={handleDelete} />
+    </div>
+  {/if}
   {#if id !== Roles.BASIC}
     <Handle
       type="target"
@@ -104,7 +102,7 @@
       isConnectable={$dragging}
     />
   {/if}
-  {#if id !== Roles.ADMIN}
+  {#if id !== Roles.ADMIN && selected}
     <Handle type="source" position={Position.Right} />
   {/if}
 </div>
@@ -140,11 +138,11 @@
     position: relative;
     background: var(--node-background);
     border-radius: 4px;
-    border: 1px solid transparent;
     width: var(--width);
     height: var(--height);
     display: flex;
     flex-direction: row;
+    gap: 12px;
     box-sizing: border-box;
     cursor: pointer;
   }
@@ -163,35 +161,33 @@
   .content {
     width: 0;
     flex: 1 1 auto;
-    padding: 0 12px;
     display: flex;
     flex-direction: column;
     justify-content: center;
     align-items: stretch;
     gap: 2px;
     border: 1px solid var(--border-color);
+    border-left-width: 0;
     border-top-right-radius: 4px;
     border-bottom-right-radius: 4px;
   }
   .node.selected .content {
     border-color: transparent;
   }
-  .title,
   .buttons {
     display: flex;
     flex-direction: row;
     align-items: center;
     gap: 6px;
   }
-  .title {
-    justify-content: space-between;
-  }
   .buttons {
     display: none;
+    padding-right: 12px;
   }
-  .title :global(.spectrum-Icon) {
+  .buttons :global(.spectrum-Icon) {
     color: var(--spectrum-global-color-gray-600);
   }
+
   .name,
   .description {
     white-space: nowrap;
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index d41a2b04e88..103b585ef54 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -10,7 +10,7 @@ export const dagreLayout = ({ nodes, edges }) => {
   dagreGraph.setDefaultEdgeLabel(() => ({}))
   dagreGraph.setGraph({
     rankdir: "LR",
-    ranksep: GridResolution * 6,
+    ranksep: GridResolution * 8,
     nodesep: GridResolution * 2,
   })
   nodes.forEach(node => {

From 518f29030f441ca5e282ac91e5c3b570d53b3892 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 13:47:57 +0100
Subject: [PATCH 138/455] Fix edge issue

---
 .../backend/RoleEditor/RoleFlow.svelte           |  4 ----
 .../backend/RoleEditor/RoleNode.svelte           | 16 ++++++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index ae3da3cfdb2..afb0e68f6c4 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -241,8 +241,6 @@
     overflow: hidden;
     position: relative;
     --background-color: var(--spectrum-global-color-gray-50);
-    --node-background: var(--spectrum-global-color-gray-100);
-    --node-background-hover: var(--spectrum-global-color-gray-300);
     --border-color: var(--spectrum-global-color-gray-300);
     --edge-color: var(--spectrum-global-color-gray-500);
     --handle-color: var(--spectrum-global-color-gray-600);
@@ -255,8 +253,6 @@
     --xy-background-color: var(--background-color);
 
     /* Controls */
-    --xy-controls-button-background-color: var(--node-background);
-    --xy-controls-button-background-color-hover: var(--node-background-hover);
     --xy-controls-button-border-color: var(--border-color);
 
     /* Handles */
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 1615a355fa0..ffc0db3758e 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Handle, Position, useSvelteFlow, NodeToolbar } from "@xyflow/svelte"
+  import { Handle, Position, useSvelteFlow } from "@xyflow/svelte"
   import {
     Icon,
     Input,
@@ -9,9 +9,8 @@
     FieldLabel,
   } from "@budibase/bbui"
   import { Roles } from "constants/backend"
-  import { NodeWidth, NodeHeight, MaxAutoZoom, ZoomDuration } from "./constants"
+  import { NodeWidth, NodeHeight } from "./constants"
   import { getContext } from "svelte"
-  import { autoLayout } from "./layout"
   import { roles } from "stores/builder"
 
   export let data
@@ -31,6 +30,7 @@
   $: descriptionError = validateDescription(tempDescription)
   $: invalid = nameError || descriptionError
   $: targetClasses = `target${$dragging ? "" : " hidden"}`
+  $: sourceClasses = `source${selected ? "" : " hidden"}`
 
   const validateName = (name, roles) => {
     if (!name?.length) {
@@ -102,8 +102,8 @@
       isConnectable={$dragging}
     />
   {/if}
-  {#if id !== Roles.ADMIN && selected}
-    <Handle type="source" position={Position.Right} />
+  {#if id !== Roles.ADMIN}
+    <Handle type="source" position={Position.Right} class={sourceClasses} />
   {/if}
 </div>
 
@@ -136,7 +136,7 @@
 <style>
   .node {
     position: relative;
-    background: var(--node-background);
+    background: var(--spectrum-global-color-gray-100);
     border-radius: 4px;
     width: var(--width);
     height: var(--height);
@@ -145,6 +145,10 @@
     gap: 12px;
     box-sizing: border-box;
     cursor: pointer;
+    transition: background 130ms ease-out;
+  }
+  .node:hover {
+    background: var(--spectrum-global-color-gray-200);
   }
   .node.selected {
     background: var(--spectrum-global-color-blue-100);

From 1eee556bee4b5e410b9b57d1fcb2dbbccd151403 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 13:51:29 +0100
Subject: [PATCH 139/455] Fix issues with multi selectioni

---
 .../src/components/backend/RoleEditor/RoleEdge.svelte     | 6 +++---
 .../src/components/backend/RoleEditor/RoleFlow.svelte     | 8 ++++----
 .../src/components/backend/RoleEditor/RoleNode.svelte     | 3 +--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index e8cfb8a5145..94e00fa97f4 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -6,7 +6,7 @@
     useSvelteFlow,
   } from "@xyflow/svelte"
   import { Icon, TooltipPosition } from "@budibase/bbui"
-  import { onMount, getContext } from "svelte"
+  import { getContext } from "svelte"
   import { roles } from "stores/builder"
 
   export let sourceX
@@ -20,11 +20,11 @@
   export let target
 
   const flow = useSvelteFlow()
-  const { updateRole, selectedNode } = getContext("flow")
+  const { updateRole, selectedNodes } = getContext("flow")
 
   let iconHovered = false
 
-  $: active = source === $selectedNode || target === $selectedNode
+  $: active = $selectedNodes.includes(source) || $selectedNodes.includes(target)
   $: edgeClasses = getEdgeClasses(active, iconHovered)
   $: [edgePath, labelX, labelY] = getBezierPath({
     sourceX,
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index afb0e68f6c4..bf30bb3c7a4 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -24,9 +24,8 @@
   const nodes = writable([])
   const edges = writable([])
   const dragging = writable(false)
-  const selectedNode = derived(
-    nodes,
-    $nodes => $nodes.find(x => x.selected)?.id
+  const selectedNodes = derived(nodes, $nodes =>
+    $nodes.filter(x => x.selected).map(x => x.id)
   )
 
   // Ensure role changes are synced with nodes and edges
@@ -192,7 +191,7 @@
     nodes,
     edges,
     dragging,
-    selectedNode,
+    selectedNodes,
     createRole,
     updateRole,
     deleteRole,
@@ -219,6 +218,7 @@
     onconnectstart={() => dragging.set(true)}
     onconnectend={() => dragging.set(false)}
     onconnect={onConnect}
+    deleteKey={null}
   >
     <Background variant={BackgroundVariant.Dots} />
     <Controls />
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index ffc0db3758e..92111615061 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Handle, Position, useSvelteFlow } from "@xyflow/svelte"
+  import { Handle, Position } from "@xyflow/svelte"
   import {
     Icon,
     Input,
@@ -18,7 +18,6 @@
   export let selected
 
   const { dragging, updateRole, deleteRole } = getContext("flow")
-  const flow = useSvelteFlow()
 
   let anchor
   let modal

From e8f71f6aced5fcb5a6a9123177f03c1cfaa694e4 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 16 Sep 2024 14:24:50 +0100
Subject: [PATCH 140/455] Lint

---
 packages/client/src/components/app/DataProvider.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index c05d4de74e0..7b5187ffb6e 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -1,6 +1,6 @@
 <script>
   import { getContext } from "svelte"
-  import { Pagination, ProgressCircle, Helpers } from "@budibase/bbui"
+  import { Pagination, ProgressCircle } from "@budibase/bbui"
   import { fetchData, QueryUtils } from "@budibase/frontend-core"
   import { LogicalOperator, EmptyFilterOption } from "@budibase/types"
 

From c9c6d410c777e09c8d01a116118e2b3066eed401 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 14:55:40 +0100
Subject: [PATCH 141/455] Wait for server changes before updating state

---
 .../backend/RoleEditor/RoleFlow.svelte        | 28 ++-----------------
 .../backend/RoleEditor/RoleNode.svelte        | 12 +++++++-
 2 files changed, 13 insertions(+), 27 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index bf30bb3c7a4..6df8e342e96 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -118,7 +118,7 @@
 
   // Creates a new role
   const createRole = async () => {
-    const newRole = {
+    await roles.save({
       name: Helpers.uuid(),
       uiMetadata: {
         displayName: getSequentialName($roles, "New role ", {
@@ -129,22 +129,7 @@
       },
       permissionId: "write",
       inherits: Roles.BASIC,
-    }
-
-    // Immediate state update
-    const newNode = {
-      ...roleToNode({ ...newRole, _id: newRole.name }),
-      selected: true,
-    }
-    const layout = autoLayout({
-      nodes: [...$nodes.map(node => ({ ...node, selected: false })), newNode],
-      edges: $edges,
     })
-    nodes.set(layout.nodes)
-    edges.set(layout.edges)
-
-    // Actually create role
-    await roles.save(newRole)
   }
 
   // Updates a role with new metadata
@@ -155,7 +140,7 @@
       return
     }
 
-    // Immediate state update
+    // Update metadata
     if (metadata) {
       flow.updateNodeData(roleId, metadata)
     }
@@ -167,15 +152,6 @@
 
   // Deletes a role
   const deleteRole = async roleId => {
-    // Immediate state update
-    const layout = autoLayout({
-      nodes: $nodes.filter(x => x.id !== roleId),
-      edges: $edges.filter(x => x.source !== roleId && x.target !== roleId),
-    })
-    nodes.set(layout.nodes)
-    edges.set(layout.edges)
-
-    // Actually delete role
     const role = $roles.find(x => x._id === roleId)
     if (role) {
       roles.delete(role)
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 92111615061..169f8af973b 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -12,6 +12,7 @@
   import { NodeWidth, NodeHeight } from "./constants"
   import { getContext } from "svelte"
   import { roles } from "stores/builder"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
 
   export let data
   export let id
@@ -24,6 +25,7 @@
   let tempDisplayName
   let tempDescription
   let tempColor
+  let deleteModal
 
   $: nameError = validateName(tempDisplayName, $roles)
   $: descriptionError = validateDescription(tempDescription)
@@ -90,7 +92,7 @@
   {#if data.custom}
     <div class="buttons">
       <Icon size="S" name="Edit" hoverable on:click={openPopover} />
-      <Icon size="S" name="Delete" hoverable on:click={handleDelete} />
+      <Icon size="S" name="Delete" hoverable on:click={deleteModal?.show} />
     </div>
   {/if}
   {#if id !== Roles.BASIC}
@@ -106,6 +108,14 @@
   {/if}
 </div>
 
+<ConfirmDialog
+  bind:this={deleteModal}
+  title={`Delete ${data.displayName}`}
+  body="Are you sure you want to delete this role? This can't be undone."
+  okText="Delete"
+  onOk={async () => await deleteRole(id)}
+/>
+
 <Modal bind:this={modal}>
   <ModalContent
     title={`Edit ${data.displayName}`}

From 3e2fdee1864d7b1ac336285eafab34659382ea0d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 14:59:33 +0100
Subject: [PATCH 142/455] Remove logs

---
 .../builder/src/components/backend/RoleEditor/RoleFlow.svelte   | 2 +-
 packages/builder/src/stores/builder/roles.js                    | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 6df8e342e96..c4679f4d446 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -48,7 +48,7 @@
   const nodeToRole = node => {
     const role = $roles.find(x => x._id === node.id)
     const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
-    console.log(inherits)
+    // TODO save inherits array
     return {
       ...role,
       // inherits,
diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index cb86991a9a4..994f87a8d3c 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -68,7 +68,6 @@ export function createRolesStore() {
       if (role?._id.startsWith("role_")) {
         role._id = role._id.replace("role_", "")
       }
-      console.log("replace", roleId, role)
 
       // Handles external updates of roles
       if (!roleId) {

From 7406812b18a475c0a1fe2b3b4b98e7a26ffb8307 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 15:01:08 +0100
Subject: [PATCH 143/455] Lint

---
 .../src/components/backend/RoleEditor/Controls.svelte        | 2 +-
 .../src/components/backend/RoleEditor/RoleFlow.svelte        | 1 -
 .../src/components/backend/RoleEditor/RoleNode.svelte        | 5 -----
 3 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 9cadbb1be13..5e241880249 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -1,7 +1,7 @@
 <script>
   import { Button, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow } from "@xyflow/svelte"
-  import { getContext, tick } from "svelte"
+  import { getContext } from "svelte"
   import { autoLayout } from "./layout"
   import { MaxAutoZoom, ZoomDuration } from "./constants"
 
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index c4679f4d446..d10ffed55ae 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -18,7 +18,6 @@
   import { roles } from "stores/builder"
   import { Roles } from "constants/backend"
   import { getSequentialName } from "helpers/duplicate"
-  import { meta } from "@roxi/routify"
 
   const flow = useSvelteFlow()
   const nodes = writable([])
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 169f8af973b..ba0dbe6404c 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -65,11 +65,6 @@
       color: tempColor,
     })
   }
-
-  const handleDelete = async e => {
-    e.stopPropagation()
-    await deleteRole(id)
-  }
 </script>
 
 <div

From 93751f39157dda89108bfd9c019ce45aedb6952c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Sep 2024 15:09:15 +0100
Subject: [PATCH 144/455] Lint

---
 .../builder/src/components/backend/RoleEditor/RoleFlow.svelte   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index d10ffed55ae..9284dafc34b 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -46,7 +46,7 @@
   // Converts a node structure back into a role doc
   const nodeToRole = node => {
     const role = $roles.find(x => x._id === node.id)
-    const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
+    // const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
     // TODO save inherits array
     return {
       ...role,

From 6b610bd982a87ce7c345f68939d02a50e46dc8f9 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 16 Sep 2024 15:28:49 +0100
Subject: [PATCH 145/455] Change query defaulting behaviour

---
 packages/frontend-core/src/fetch/DataFetch.js |  7 +------
 packages/shared-core/src/filters.ts           |  7 +------
 packages/shared-core/src/utils.ts             | 13 +++++++------
 3 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index 4f51cdad71e..d5285934bee 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -177,16 +177,11 @@ export default class DataFetch {
       }
     }
 
-    let defaultQuery = {
-      logicalOperator: FilterGroupLogicalOperator.ALL,
-      groups: [],
-    }
-
     // Build the query
     let query = this.options.query
 
     if (!query && this.features.supportsSearch) {
-      query = buildQuery(filter || defaultQuery)
+      query = buildQuery(filter)
     }
 
     // Update store
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 360d3ae5127..e4f0282775f 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -574,12 +574,7 @@ export const buildQueryLegacy = (
 export const buildQuery = (
   filter?: SearchFilterGroup | SearchFilter[]
 ): SearchFilters | undefined => {
-  const parsedFilter: SearchFilterGroup | undefined =
-    processSearchFilters(filter)
-
-  if (!parsedFilter) {
-    return
-  }
+  const parsedFilter: SearchFilterGroup = processSearchFilters(filter)
 
   const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
     {
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index f37d50d1720..9eb71c1f3ca 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -98,16 +98,17 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
  */
 export const processSearchFilters = (
   filters: SearchFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup | undefined => {
-  if (!filters) {
-    return
-  }
+): SearchFilterGroup => {
+  // Base search config.
   const defaultCfg: SearchFilterGroup = {
     logicalOperator: FilterGroupLogicalOperator.ALL,
-    onEmptyFilter: EmptyFilterOption.RETURN_NONE,
     groups: [],
   }
 
+  if (!filters) {
+    return defaultCfg
+  }
+
   const filterWhitelistKeys = [
     "field",
     "operator",
@@ -182,7 +183,7 @@ export const processSearchFilters = (
 
     return migratedSetting
   } else if (!filters?.groups) {
-    return
+    return defaultCfg
   }
   return filters
 }

From 7e37420c9765a79c2ec24019904a98a4bca5a627 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 16 Sep 2024 15:48:55 +0100
Subject: [PATCH 146/455] Lint :melting_face:

---
 packages/frontend-core/src/fetch/DataFetch.js | 1 -
 packages/shared-core/src/utils.ts             | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index d5285934bee..246cc4f804e 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -2,7 +2,6 @@ import { writable, derived, get } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
-import { FilterGroupLogicalOperator } from "@budibase/types"
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 9eb71c1f3ca..623b1a4245b 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -2,7 +2,6 @@ import {
   SearchFilter,
   SearchFilterGroup,
   FilterGroupLogicalOperator,
-  EmptyFilterOption,
   SearchFilters,
   BasicOperator,
   ArrayOperator,

From 7f3d88265e38d20ed1edf8622069365dd4fa8441 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 17 Sep 2024 11:18:32 +0100
Subject: [PATCH 147/455] Allow passing no query to the search API.

---
 packages/frontend-core/src/api/tables.js        |  4 ++--
 packages/frontend-core/src/fetch/UserFetch.js   |  2 +-
 packages/frontend-core/src/fetch/ViewV2Fetch.js |  2 +-
 packages/shared-core/src/filters.ts             |  7 ++++++-
 packages/shared-core/src/utils.ts               | 12 ++++++------
 5 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/packages/frontend-core/src/api/tables.js b/packages/frontend-core/src/api/tables.js
index 34d2371e1a2..dc9008e4ebc 100644
--- a/packages/frontend-core/src/api/tables.js
+++ b/packages/frontend-core/src/api/tables.js
@@ -40,7 +40,7 @@ export const buildTableEndpoints = API => ({
     sortType,
     paginate,
   }) => {
-    if (!tableId || !query) {
+    if (!tableId) {
       return {
         rows: [],
       }
@@ -48,7 +48,7 @@ export const buildTableEndpoints = API => ({
     return await API.post({
       url: `/api/${tableId}/search`,
       body: {
-        query,
+        ...(query ? { query } : {}),
         bookmark,
         limit,
         sort,
diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js
index 39a8d79214e..902aa7edad3 100644
--- a/packages/frontend-core/src/fetch/UserFetch.js
+++ b/packages/frontend-core/src/fetch/UserFetch.js
@@ -32,7 +32,7 @@ export default class UserFetch extends DataFetch {
     const { cursor, query } = get(this.store)
     let finalQuery
     // convert old format to new one - we now allow use of the lucene format
-    const { appId, paginated, ...rest } = query
+    const { appId, paginated, ...rest } = query || {}
 
     finalQuery = utils.isSupportedUserSearch(rest)
       ? query
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 303414b2dc6..46343492b4d 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -64,7 +64,7 @@ export default class ViewV2Fetch extends DataFetch {
     try {
       const res = await this.API.viewV2.fetch({
         viewId: datasource.id,
-        query,
+        ...(query ? { query } : {}),
         paginate,
         limit,
         bookmark: cursor,
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index e4f0282775f..360d3ae5127 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -574,7 +574,12 @@ export const buildQueryLegacy = (
 export const buildQuery = (
   filter?: SearchFilterGroup | SearchFilter[]
 ): SearchFilters | undefined => {
-  const parsedFilter: SearchFilterGroup = processSearchFilters(filter)
+  const parsedFilter: SearchFilterGroup | undefined =
+    processSearchFilters(filter)
+
+  if (!parsedFilter) {
+    return
+  }
 
   const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
     {
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 623b1a4245b..5b4d4399847 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -97,17 +97,17 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
  */
 export const processSearchFilters = (
   filters: SearchFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup => {
+): SearchFilterGroup | undefined => {
+  if (!filters) {
+    return
+  }
+
   // Base search config.
   const defaultCfg: SearchFilterGroup = {
     logicalOperator: FilterGroupLogicalOperator.ALL,
     groups: [],
   }
 
-  if (!filters) {
-    return defaultCfg
-  }
-
   const filterWhitelistKeys = [
     "field",
     "operator",
@@ -182,7 +182,7 @@ export const processSearchFilters = (
 
     return migratedSetting
   } else if (!filters?.groups) {
-    return defaultCfg
+    return
   }
   return filters
 }

From c2116e37215a9502e275655f627756d3adbde2dd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 17 Sep 2024 16:38:34 +0100
Subject: [PATCH 148/455] Remove deprecated view access feature from builder
 store

---
 packages/builder/src/stores/portal/licensing.js | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js
index d48377207c0..ff534ea6672 100644
--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@@ -136,10 +136,6 @@ export const createLicensingStore = () => {
         Constants.Features.APP_BUILDERS
       )
 
-      const isViewPermissionsEnabled = license.features.includes(
-        Constants.Features.VIEW_PERMISSIONS
-      )
-
       const isViewReadonlyColumnsEnabled = license.features.includes(
         Constants.Features.VIEW_READONLY_COLUMNS
       )
@@ -171,7 +167,6 @@ export const createLicensingStore = () => {
           enforceableSSO,
           syncAutomationsEnabled,
           triggerAutomationRunEnabled,
-          isViewPermissionsEnabled,
           perAppBuildersEnabled,
           isViewReadonlyColumnsEnabled,
         }

From bb0fad8c9ad21b7192d139ba6a4dce76b1257843 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 11:48:50 +0100
Subject: [PATCH 149/455] Use 'app users' for users table title in new view nav
 bar

---
 .../data/table/[tableId]/_components/ViewNavBar.svelte          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index d74fb90cb29..26ca8edf5bf 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -191,7 +191,7 @@
     on:contextmenu={openTableContextMenu}
   >
     <div class="nav-item__title">
-      {table.name}
+      {table._id === TableNames.USERS ? "App users" : table.name}
     </div>
     {#if tableSelectedBy}
       <UserAvatars size="XS" users={tableSelectedBy} />

From d4db493519bfe45e662ad5c3c4bdfd5e3b4be6bc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 11:50:15 +0100
Subject: [PATCH 150/455] Set view permissions to explicit roles from the
 parent table

---
 .../server/src/api/controllers/permission.ts  |  4 +--
 packages/server/src/sdk/app/views/index.ts    | 33 ++++++++++++++++---
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index 66a3254348a..b75af880671 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -20,7 +20,7 @@ import {
 import { removeFromArray } from "../../utilities"
 import sdk from "../../sdk"
 
-const enum PermissionUpdateType {
+export const enum PermissionUpdateType {
   REMOVE = "remove",
   ADD = "add",
 }
@@ -37,7 +37,7 @@ async function getAllDBRoles(db: Database) {
   return body.rows.map(row => row.doc!)
 }
 
-async function updatePermissionOnRole(
+export async function updatePermissionOnRole(
   {
     roleId,
     resourceId,
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index d7e05abf2f8..c580bfde505 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -1,5 +1,6 @@
 import {
   FieldType,
+  PermissionLevel,
   RelationSchemaField,
   RenameColumn,
   Table,
@@ -10,20 +11,22 @@ import {
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
-import { HTTPError } from "@budibase/backend-core"
+import { HTTPError, roles } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
-
 import * as utils from "../../../db/utils"
 import { isExternalTableID } from "../../../integrations/utils"
-
 import * as internal from "./internal"
 import * as external from "./external"
 import sdk from "../../../sdk"
+import {
+  updatePermissionOnRole,
+  PermissionUpdateType,
+} from "src/api/controllers/permission"
 
 function pickApi(tableId: any) {
   if (isExternalTableID(tableId)) {
@@ -123,8 +126,30 @@ export async function create(
   viewRequest: Omit<ViewV2, "id" | "version">
 ): Promise<ViewV2> {
   await guardViewSchema(tableId, viewRequest)
+  const view = await pickApi(tableId).create(tableId, viewRequest)
+
+  // Set permissions to be the same as the table
+  const tablePerms = await sdk.permissions.getResourcePerms(tableId)
+  const readRole = tablePerms[PermissionLevel.READ]?.role
+  const writeRole = tablePerms[PermissionLevel.WRITE]?.role
+  await updatePermissionOnRole(
+    {
+      roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC,
+      resourceId: view.id,
+      level: PermissionLevel.READ,
+    },
+    PermissionUpdateType.ADD
+  )
+  await updatePermissionOnRole(
+    {
+      roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC,
+      resourceId: view.id,
+      level: PermissionLevel.WRITE,
+    },
+    PermissionUpdateType.ADD
+  )
 
-  return pickApi(tableId).create(tableId, viewRequest)
+  return view
 }
 
 export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {

From 55c7751dbbc6f96f66cc1b8d35d7561f797b528c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 19 Sep 2024 15:12:03 +0100
Subject: [PATCH 151/455] Move permission updates into SDK

---
 .../server/src/api/controllers/permission.ts  | 107 ++----------------
 .../server/src/sdk/app/permissions/index.ts   | 102 ++++++++++++++++-
 packages/server/src/sdk/app/views/index.ts    |   5 +-
 3 files changed, 111 insertions(+), 103 deletions(-)

diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index b75af880671..55b942686fe 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -3,7 +3,6 @@ import {
   UserCtx,
   Database,
   Role,
-  PermissionLevel,
   GetResourcePermsResponse,
   ResourcePermissionInfo,
   GetDependantResourcesResponse,
@@ -12,107 +11,15 @@ import {
   RemovePermissionRequest,
   RemovePermissionResponse,
 } from "@budibase/types"
-import { getRoleParams } from "../../db/utils"
 import {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
 } from "../../utilities/security"
-import { removeFromArray } from "../../utilities"
 import sdk from "../../sdk"
-
-export const enum PermissionUpdateType {
-  REMOVE = "remove",
-  ADD = "add",
-}
+import { PermissionUpdateType } from "../../sdk/app/permissions"
 
 const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
 
-// utility function to stop this repetition - permissions always stored under roles
-async function getAllDBRoles(db: Database) {
-  const body = await db.allDocs<Role>(
-    getRoleParams(null, {
-      include_docs: true,
-    })
-  )
-  return body.rows.map(row => row.doc!)
-}
-
-export async function updatePermissionOnRole(
-  {
-    roleId,
-    resourceId,
-    level,
-  }: { roleId: string; resourceId: string; level: PermissionLevel },
-  updateType: PermissionUpdateType
-) {
-  const db = context.getAppDB()
-  const remove = updateType === PermissionUpdateType.REMOVE
-  const isABuiltin = roles.isBuiltin(roleId)
-  const dbRoleId = roles.getDBRoleID(roleId)
-  const dbRoles = await getAllDBRoles(db)
-  const docUpdates: Role[] = []
-
-  // the permission is for a built in, make sure it exists
-  if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) {
-    const builtin = roles.getBuiltinRoles()[roleId]
-    builtin._id = roles.getDBRoleID(builtin._id!)
-    dbRoles.push(builtin)
-  }
-
-  // now try to find any roles which need updated, e.g. removing the
-  // resource from another role and then adding to the new role
-  for (let role of dbRoles) {
-    let updated = false
-    const rolePermissions: Record<string, PermissionLevel[]> = role.permissions
-      ? role.permissions
-      : {}
-    // make sure its an array, also handle migrating
-    if (
-      !rolePermissions[resourceId] ||
-      !Array.isArray(rolePermissions[resourceId])
-    ) {
-      rolePermissions[resourceId] =
-        typeof rolePermissions[resourceId] === "string"
-          ? [rolePermissions[resourceId] as unknown as PermissionLevel]
-          : []
-    }
-    // handle the removal/updating the role which has this permission first
-    // the updating (role._id !== dbRoleId) is required because a resource/level can
-    // only be permitted in a single role (this reduces hierarchy confusion and simplifies
-    // the general UI for this, rather than needing to show everywhere it is used)
-    if (
-      (role._id !== dbRoleId || remove) &&
-      rolePermissions[resourceId].indexOf(level) !== -1
-    ) {
-      removeFromArray(rolePermissions[resourceId], level)
-      updated = true
-    }
-    // handle the adding, we're on the correct role, at it to this
-    if (!remove && role._id === dbRoleId) {
-      const set = new Set(rolePermissions[resourceId])
-      rolePermissions[resourceId] = [...set.add(level)]
-      updated = true
-    }
-    // handle the update, add it to bulk docs to perform at end
-    if (updated) {
-      role.permissions = rolePermissions
-      docUpdates.push(role)
-    }
-  }
-
-  const response = await db.bulkDocs(docUpdates)
-  return response.map(resp => {
-    const version = docUpdates.find(role => role._id === resp.id)?.version
-    const _id = roles.getExternalRoleID(resp.id, version)
-    return {
-      _id,
-      rev: resp.rev,
-      error: resp.error,
-      reason: resp.reason,
-    }
-  })
-}
-
 export function fetchBuiltin(ctx: UserCtx) {
   ctx.body = Object.values(permissions.getBuiltinPermissions())
 }
@@ -124,7 +31,7 @@ export function fetchLevels(ctx: UserCtx) {
 
 export async function fetch(ctx: UserCtx) {
   const db = context.getAppDB()
-  const dbRoles: Role[] = await getAllDBRoles(db)
+  const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db)
   let permissions: any = {}
   // create an object with structure role ID -> resource ID -> level
   for (let role of dbRoles) {
@@ -186,12 +93,18 @@ export async function getDependantResources(
 
 export async function addPermission(ctx: UserCtx<void, AddPermissionResponse>) {
   const params: AddPermissionRequest = ctx.params
-  ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD)
+  ctx.body = await sdk.permissions.updatePermissionOnRole(
+    params,
+    PermissionUpdateType.ADD
+  )
 }
 
 export async function removePermission(
   ctx: UserCtx<void, RemovePermissionResponse>
 ) {
   const params: RemovePermissionRequest = ctx.params
-  ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE)
+  ctx.body = await sdk.permissions.updatePermissionOnRole(
+    params,
+    PermissionUpdateType.REMOVE
+  )
 }
diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts
index a6e81652ee4..5f8882399bd 100644
--- a/packages/server/src/sdk/app/permissions/index.ts
+++ b/packages/server/src/sdk/app/permissions/index.ts
@@ -1,22 +1,34 @@
-import { db, roles } from "@budibase/backend-core"
+import { db, roles, context } from "@budibase/backend-core"
 import {
   PermissionLevel,
   PermissionSource,
   VirtualDocumentType,
+  Role,
+  Database,
 } from "@budibase/types"
-import { extractViewInfoFromID, isViewID } from "../../../db/utils"
+import {
+  extractViewInfoFromID,
+  isViewID,
+  getRoleParams,
+} from "../../../db/utils"
 import {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
 } from "../../../utilities/security"
 import sdk from "../../../sdk"
 import { isV2 } from "../views"
+import { removeFromArray } from "../../../utilities"
 
 type ResourcePermissions = Record<
   string,
   { role: string; type: PermissionSource }
 >
 
+export const enum PermissionUpdateType {
+  REMOVE = "remove",
+  ADD = "add",
+}
+
 export async function getInheritablePermissions(
   resourceId: string
 ): Promise<ResourcePermissions | undefined> {
@@ -100,3 +112,89 @@ export async function getDependantResources(
 
   return
 }
+
+export async function updatePermissionOnRole(
+  {
+    roleId,
+    resourceId,
+    level,
+  }: { roleId: string; resourceId: string; level: PermissionLevel },
+  updateType: PermissionUpdateType
+) {
+  const db = context.getAppDB()
+  const remove = updateType === PermissionUpdateType.REMOVE
+  const isABuiltin = roles.isBuiltin(roleId)
+  const dbRoleId = roles.getDBRoleID(roleId)
+  const dbRoles = await getAllDBRoles(db)
+  const docUpdates: Role[] = []
+
+  // the permission is for a built in, make sure it exists
+  if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) {
+    const builtin = roles.getBuiltinRoles()[roleId]
+    builtin._id = roles.getDBRoleID(builtin._id!)
+    dbRoles.push(builtin)
+  }
+
+  // now try to find any roles which need updated, e.g. removing the
+  // resource from another role and then adding to the new role
+  for (let role of dbRoles) {
+    let updated = false
+    const rolePermissions: Record<string, PermissionLevel[]> = role.permissions
+      ? role.permissions
+      : {}
+    // make sure its an array, also handle migrating
+    if (
+      !rolePermissions[resourceId] ||
+      !Array.isArray(rolePermissions[resourceId])
+    ) {
+      rolePermissions[resourceId] =
+        typeof rolePermissions[resourceId] === "string"
+          ? [rolePermissions[resourceId] as unknown as PermissionLevel]
+          : []
+    }
+    // handle the removal/updating the role which has this permission first
+    // the updating (role._id !== dbRoleId) is required because a resource/level can
+    // only be permitted in a single role (this reduces hierarchy confusion and simplifies
+    // the general UI for this, rather than needing to show everywhere it is used)
+    if (
+      (role._id !== dbRoleId || remove) &&
+      rolePermissions[resourceId].indexOf(level) !== -1
+    ) {
+      removeFromArray(rolePermissions[resourceId], level)
+      updated = true
+    }
+    // handle the adding, we're on the correct role, at it to this
+    if (!remove && role._id === dbRoleId) {
+      const set = new Set(rolePermissions[resourceId])
+      rolePermissions[resourceId] = [...set.add(level)]
+      updated = true
+    }
+    // handle the update, add it to bulk docs to perform at end
+    if (updated) {
+      role.permissions = rolePermissions
+      docUpdates.push(role)
+    }
+  }
+
+  const response = await db.bulkDocs(docUpdates)
+  return response.map(resp => {
+    const version = docUpdates.find(role => role._id === resp.id)?.version
+    const _id = roles.getExternalRoleID(resp.id, version)
+    return {
+      _id,
+      rev: resp.rev,
+      error: resp.error,
+      reason: resp.reason,
+    }
+  })
+}
+
+// utility function to stop this repetition - permissions always stored under roles
+export async function getAllDBRoles(db: Database) {
+  const body = await db.allDocs<Role>(
+    getRoleParams(null, {
+      include_docs: true,
+    })
+  )
+  return body.rows.map(row => row.doc!)
+}
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index c580bfde505..47af484ebca 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -23,10 +23,7 @@ import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./internal"
 import * as external from "./external"
 import sdk from "../../../sdk"
-import {
-  updatePermissionOnRole,
-  PermissionUpdateType,
-} from "src/api/controllers/permission"
+import { updatePermissionOnRole, PermissionUpdateType } from "../permissions"
 
 function pickApi(tableId: any) {
   if (isExternalTableID(tableId)) {

From 418bbff2f57e573ce9f36796f5fb77014aaa4870 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 19 Sep 2024 15:15:19 +0100
Subject: [PATCH 152/455] Lint

---
 packages/server/src/api/controllers/permission.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index 55b942686fe..c7afb6a3517 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -1,7 +1,6 @@
 import { permissions, roles, context } from "@budibase/backend-core"
 import {
   UserCtx,
-  Database,
   Role,
   GetResourcePermsResponse,
   ResourcePermissionInfo,

From 72c8ff9fa77ebb67da11ceb36effc91678a2e7dc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Sep 2024 13:59:50 +0100
Subject: [PATCH 153/455] Update view handling of filters

---
 .../grid/stores/datasources/viewV2.js         | 63 ++++++++++++-------
 .../src/components/grid/stores/filter.js      | 36 ++++++-----
 packages/frontend-core/src/fetch/DataFetch.js |  5 +-
 .../frontend-core/src/fetch/ViewV2Fetch.js    | 17 +----
 4 files changed, 68 insertions(+), 53 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 425519d97e9..2ee2039e4c5 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -97,9 +97,12 @@ export const initialise = context => {
       order: get(initialSortOrder) || "ascending",
     })
 
-    // Keep sort and filter state in line with the view definition
+    // Keep sort and filter state in line with the view definition when in builder
     unsubscribers.push(
       definition.subscribe($definition => {
+        if (!get(config).canSaveSchema) {
+          return
+        }
         if ($definition?.id !== $datasource.id) {
           return
         }
@@ -122,7 +125,6 @@ export const initialise = context => {
       sort.subscribe(async $sort => {
         // If we can mutate schema then update the view definition
         if (get(config).canSaveSchema) {
-          // Ensure we're updating the correct view
           const $view = get(definition)
           if ($view?.id !== $datasource.id) {
             return
@@ -144,7 +146,7 @@ export const initialise = context => {
         // Also update the fetch to ensure the new sort is respected.
         // Ensure we're updating the correct fetch.
         const $fetch = get(fetch)
-        if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) {
+        if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }
         $fetch.update({
@@ -157,32 +159,49 @@ export const initialise = context => {
     // When filters change, ensure view definition is kept up to date
     unsubscribers?.push(
       filter.subscribe(async $filter => {
-        // If we can mutate schema then update the view definition
-        if (get(config).canSaveSchema) {
-          // Ensure we're updating the correct view
-          const $view = get(definition)
-          if ($view?.id !== $datasource.id) {
-            return
-          }
-          if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
-            await datasource.actions.saveDefinition({
-              ...$view,
-              query: $filter,
-            })
-          }
+        if (!get(config).canSaveSchema) {
+          return
+        }
+        const $view = get(definition)
+        if ($view?.id !== $datasource.id) {
+          return
+        }
+        if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
+          await datasource.actions.saveDefinition({
+            ...$view,
+            query: $filter,
+          })
+
+          // Refresh data since view definition changed
+          await rows.actions.refreshData()
+        }
+      })
+    )
+
+    // Keep fetch up to date with inline filters when in the data section
+    unsubscribers.push(
+      inlineFilters.subscribe($inlineFilters => {
+        if (!get(config).canSaveSchema) {
+          return
         }
+        const $fetch = get(fetch)
+        if ($fetch?.options?.datasource?.id !== $datasource.id) {
+          return
+        }
+        $fetch.update({
+          filter: $inlineFilters,
+        })
       })
     )
 
-    // Keep fetch up to date with filters.
-    // If we're able to save filters against the view then we only need to apply
-    // inline filters to the fetch, as saved filters are applied server side.
-    // If we can't save filters, then all filters must be applied to the fetch.
+    // Keep fetch up to date with all filters when not in the data section
     unsubscribers.push(
       allFilters.subscribe($allFilters => {
-        // Ensure we're updating the correct fetch
+        if (get(config).canSaveSchema) {
+          return
+        }
         const $fetch = get(fetch)
-        if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) {
+        if ($fetch?.options?.datasource?.id !== $datasource.id) {
           return
         }
         $fetch.update({
diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js
index f5feee9c358..0de74379d98 100644
--- a/packages/frontend-core/src/components/grid/stores/filter.js
+++ b/packages/frontend-core/src/components/grid/stores/filter.js
@@ -1,12 +1,13 @@
 import { writable, get, derived } from "svelte/store"
 import { FieldType, FilterGroupLogicalOperator } from "@budibase/types"
+import { memo } from "../../../utils/memo"
 
 export const createStores = context => {
   const { props } = context
 
   // Initialise to default props
-  const filter = writable(get(props).initialFilter)
-  const inlineFilters = writable([])
+  const filter = memo(get(props).initialFilter)
+  const inlineFilters = memo([])
 
   return {
     filter,
@@ -19,19 +20,26 @@ export const deriveStores = context => {
   const allFilters = derived(
     [filter, inlineFilters],
     ([$filter, $inlineFilters]) => {
-      const inlineFilterGroup = $inlineFilters?.length
-        ? {
-            logicalOperator: FilterGroupLogicalOperator.ALL,
-            filters: [...($inlineFilters || [])],
-          }
-        : null
-
-      return inlineFilterGroup
-        ? {
+      // Just use filter prop if no inline filters
+      if (!$inlineFilters?.length) {
+        return $filter
+      }
+      let allFilters = {
+        logicalOperator: FilterGroupLogicalOperator.ALL,
+        groups: [
+          {
             logicalOperator: FilterGroupLogicalOperator.ALL,
-            groups: [...($filter?.groups || []), inlineFilterGroup],
-          }
-        : $filter
+            filters: $inlineFilters,
+          },
+        ],
+      }
+      // Just use inline if no filter
+      if (!$filter?.groups?.length) {
+        return allFilters
+      }
+      // Join them together if both
+      allFilters.groups = [...allFilters.groups, ...$filter.groups]
+      return allFilters
     }
   )
 
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index 246cc4f804e..a056cdff5d2 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -178,7 +178,6 @@ export default class DataFetch {
 
     // Build the query
     let query = this.options.query
-
     if (!query && this.features.supportsSearch) {
       query = buildQuery(filter)
     }
@@ -365,7 +364,9 @@ export default class DataFetch {
     let refresh = false
     const entries = Object.entries(newOptions || {})
     for (let [key, value] of entries) {
-      if (JSON.stringify(value) !== JSON.stringify(this.options[key])) {
+      const oldVal = this.options[key] == null ? null : this.options[key]
+      const newVal = value == null ? null : value
+      if (JSON.stringify(newVal) !== JSON.stringify(oldVal)) {
         refresh = true
         break
       }
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 46343492b4d..1b4c8e091e3 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -36,15 +36,8 @@ export default class ViewV2Fetch extends DataFetch {
   }
 
   async getData() {
-    const {
-      datasource,
-      limit,
-      sortColumn,
-      sortOrder,
-      sortType,
-      paginate,
-      filter,
-    } = this.options
+    const { datasource, limit, sortColumn, sortOrder, sortType, paginate } =
+      this.options
     const { cursor, query, definition } = get(this.store)
 
     // If sort/filter params are not defined, update options to store the
@@ -55,12 +48,6 @@ export default class ViewV2Fetch extends DataFetch {
       this.options.sortOrder = definition.sort.order
     }
 
-    const parsed = utils.processSearchFilters(filter)
-
-    if (!parsed?.groups?.length && definition.query?.groups?.length) {
-      this.options.filter = definition.query
-    }
-
     try {
       const res = await this.API.viewV2.fetch({
         viewId: datasource.id,

From f34261a9b71bae7a7360fc39a4e0cb8a69df9c74 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Sep 2024 14:05:03 +0100
Subject: [PATCH 154/455] Lint

---
 packages/frontend-core/src/components/grid/stores/filter.js | 2 +-
 packages/frontend-core/src/fetch/ViewV2Fetch.js             | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js
index 0de74379d98..6e6c37da87c 100644
--- a/packages/frontend-core/src/components/grid/stores/filter.js
+++ b/packages/frontend-core/src/components/grid/stores/filter.js
@@ -1,4 +1,4 @@
-import { writable, get, derived } from "svelte/store"
+import { get, derived } from "svelte/store"
 import { FieldType, FilterGroupLogicalOperator } from "@budibase/types"
 import { memo } from "../../../utils/memo"
 
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 1b4c8e091e3..40135746df3 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -1,6 +1,5 @@
 import DataFetch from "./DataFetch.js"
 import { get } from "svelte/store"
-import { utils } from "@budibase/shared-core"
 
 export default class ViewV2Fetch extends DataFetch {
   determineFeatureFlags() {

From 64f70a1ae25480aacd5452f264da9181829e4b52 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Sep 2024 15:18:09 +0100
Subject: [PATCH 155/455] Update text of filter button

---
 .../backend/DataTable/buttons/TableFilterButton.svelte          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index a4678012149..e616e274678 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -53,7 +53,7 @@
   selected={filterCount > 0}
   accentColor="#004EA6"
 >
-  {filterCount ? `Filter (${filterCount})` : "Filter"}
+  {filterCount ? `Filter: ${filterCount}` : "Filter"}
 </ActionButton>
 
 <Drawer

From 1dc37d8955d4b16d950aeb74682d8fed2ab17156 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Sep 2024 15:24:25 +0100
Subject: [PATCH 156/455] Make datasource icon in nav bar link to datasource
 page

---
 .../table/[tableId]/_components/ViewNavBar.svelte | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 26ca8edf5bf..69b792c6122 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -4,6 +4,7 @@
     datasources,
     userSelectedResourceMap,
     contextMenuStore,
+    appStore,
   } from "stores/builder"
   import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
   import { Icon, ActionButton, ActionMenu, MenuItem } from "@budibase/bbui"
@@ -179,11 +180,15 @@
 </script>
 
 <div class="nav">
-  <IntegrationIcon
-    integrationType={datasource?.source}
-    schema={datasource?.schema}
-    size="24"
-  />
+  <a
+    href={`/builder/app/${$appStore.appId}/data/datasource/${datasource?._id}`}
+  >
+    <IntegrationIcon
+      integrationType={datasource?.source}
+      schema={datasource?.schema}
+      size="24"
+    />
+  </a>
   <a
     href={$tableUrl(tableId)}
     class="nav-item"

From e663a800201e14fe0d2afd2bb5e3767116e49faf Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Sep 2024 15:26:27 +0100
Subject: [PATCH 157/455] Reduce gap in list items

---
 packages/bbui/src/List/ListItem.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 2b1f6b030fc..5b6152781a9 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -74,7 +74,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
-    gap: var(--spacing-l);
+    gap: var(--spacing-s);
   }
   .left {
     width: 0;

From 2d1662c0c79bdc841717cb9b0e9b06bf53b5bc98 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Sep 2024 15:50:08 +0100
Subject: [PATCH 158/455] Allow views to refresh individual rows after
 executing row actions

---
 .../components/grid/stores/datasources/viewV2.js   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 2ee2039e4c5..b46c6247292 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -29,8 +29,18 @@ export const createActions = context => {
     })
   }
 
-  const getRow = () => {
-    throw "Views don't support fetching individual rows"
+  const getRow = async id => {
+    const res = await API.viewV2.fetch({
+      viewId: get(datasource).id,
+      limit: 1,
+      query: {
+        equal: {
+          _id: id,
+        },
+      },
+      paginate: false,
+    })
+    return res?.rows?.[0]
   }
 
   const isDatasourceValid = datasource => {

From 51e09ddf7b2c2d0274cd2b4a503a375993505f1d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 23 Sep 2024 10:08:21 +0100
Subject: [PATCH 159/455] Update row action tests to revoke explicit view
 permissions when testing triggering against views

---
 packages/server/src/api/routes/tests/rowAction.spec.ts | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index ef7d2afbba6..efd28eb92f9 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -826,11 +826,20 @@ describe("/rowsActions", () => {
               )
             ).id
 
+            // Allow row action on view
             await config.api.rowAction.setViewPermission(
               tableId,
               viewId,
               rowAction.id
             )
+
+            // Delete explicit view permissions so they inherit table permissions
+            await config.api.permission.revoke({
+              level: PermissionLevel.READ,
+              resourceId: viewId,
+              roleId: "inherited",
+            })
+
             return { permissionResource: tableId, triggerResouce: viewId }
           },
         ],

From de34217bb21664dc47c46611c09fcb53051b687c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 23 Sep 2024 11:57:04 +0100
Subject: [PATCH 160/455] Increase spacing to M in list items

---
 packages/bbui/src/List/ListItem.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 384d20d1876..699df2d4560 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -84,7 +84,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
-    gap: var(--spacing-s);
+    gap: var(--spacing-m);
   }
   .left {
     width: 0;

From b2b6cc2ecfe1db7e5f12e21a1756b679d559885d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 23 Sep 2024 15:45:57 +0100
Subject: [PATCH 161/455] Reorder grid buttons and remove export button for
 users table

---
 .../buttons/grid/GridAutomationsButton.svelte      |  2 +-
 .../buttons/grid/GridGenerateButton.svelte         |  2 +-
 .../buttons/grid/GridScreensButton.svelte          |  2 +-
 .../data/table/[tableId]/[viewId]/index.svelte     |  4 +---
 .../data/table/[tableId]/index.svelte              | 14 ++++----------
 5 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
index 9670bb2f358..b2448eeaf31 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte
@@ -34,7 +34,7 @@
 
   const generateAutomation = () => {
     popover?.hide()
-    dispatch("request-generate")
+    dispatch("generate")
   }
 </script>
 
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
index 5cc3aca19e4..5d1ec835d54 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte
@@ -91,7 +91,7 @@
 
 <DetailPopover title="Generate" bind:this={popover}>
   <svelte:fragment slot="anchor" let:open>
-    <ActionButton selected={open}>
+    <ActionButton quiet selected={open}>
       <div class="center">
         <img height={16} alt="magic wand" src={MagicWand} />
         Generate
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
index 701a286112a..db446b3c9e5 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte
@@ -22,7 +22,7 @@
 
   const generateScreen = () => {
     popover?.hide()
-    dispatch("request-generate")
+    dispatch("generate")
   }
 </script>
 
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 82ffbd9ba5f..fefceec55fc 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -56,9 +56,7 @@
     <GridSizeButton />
     <GridColumnsSettingButton />
     <GridRowActionsButton />
-    <GridScreensButton on:request-generate={() => generateButton?.show()} />
-  </svelte:fragment>
-  <svelte:fragment slot="controls-right">
+    <GridScreensButton on:generate={() => generateButton?.show()} />
     <GridGenerateButton bind:this={generateButton} />
   </svelte:fragment>
   <GridCreateEditRowModal />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 93ec3d07648..e9c6ad11679 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -141,17 +141,11 @@
         <GridRelationshipButton />
       {/if}
       {#if !isUsersTable}
-        <GridRowActionsButton />
-        <GridScreensButton on:request-generate={() => generateButton?.show()} />
-        <GridAutomationsButton
-          on:request-generate={() => generateButton?.show()}
-        />
         <GridImportButton />
-      {/if}
-      <GridExportButton />
-    </svelte:fragment>
-    <svelte:fragment slot="controls-right">
-      {#if !isUsersTable}
+        <GridExportButton />
+        <GridRowActionsButton />
+        <GridScreensButton on:generate={() => generateButton?.show()} />
+        <GridAutomationsButton on:generate={() => generateButton?.show()} />
         <GridGenerateButton bind:this={generateButton} />
       {/if}
     </svelte:fragment>

From fd21503f3e665c826e4174b957945e3e229df1aa Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 24 Sep 2024 12:43:41 +0100
Subject: [PATCH 162/455] Enable default values on options columns with
 validation and fix default values being available on primary display columns

---
 .../DataTable/modals/CreateEditColumn.svelte  | 57 ++++++++++++-------
 packages/shared-core/src/table.ts             |  2 +-
 2 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index a956d09ee67..b04ccdb6a91 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -136,9 +136,8 @@
   }
   $: initialiseField(field, savingColumn)
   $: checkConstraints(editableColumn)
-  $: required = hasDefault
-    ? false
-    : !!editableColumn?.constraints?.presence || primaryDisplay
+  $: required =
+    primaryDisplay || (!hasDefault && !!editableColumn?.constraints?.presence)
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@@ -168,7 +167,9 @@
   $: canBeDisplay =
     canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn
   $: canHaveDefault =
-    isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type)
+    !required &&
+    isEnabled("DEFAULT_VALUES") &&
+    canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== LINK_TYPE &&
     !uneditable &&
@@ -187,11 +188,11 @@
     (originalName &&
       SWITCHABLE_TYPES[field.type] &&
       !editableColumn?.autocolumn)
-
   $: allowedTypes = getAllowedTypes(datasource).map(t => ({
     fieldId: makeFieldId(t.type, t.subtype),
     ...t,
   }))
+  $: bindings = getBindings({ table })
 
   const fieldDefinitions = Object.values(FIELDS).reduce(
     // Storing the fields by complex field id
@@ -281,6 +282,20 @@
       delete saveColumn.fieldName
     }
 
+    // Ensure the field is not required if we have a default value
+    if (saveColumn.default) {
+      saveColumn.constraints.presence = false
+    }
+
+    // Delete default value for options fields if the option is no longer available
+    if (
+      saveColumn.type === FieldType.OPTIONS &&
+      saveColumn.default &&
+      !saveColumn.constraints.inclusion?.includes(saveColumn.default)
+    ) {
+      delete saveColumn.default
+    }
+
     try {
       await tables.saveField({
         originalName,
@@ -727,7 +742,7 @@
               formula: e.detail,
             }
           }}
-          bindings={getBindings({ table })}
+          {bindings}
           allowJS
           context={rowGoldenSample}
         />
@@ -766,27 +781,27 @@
   {/if}
 
   {#if canHaveDefault}
-    <div>
+    {#if editableColumn.type === FieldType.OPTIONS}
+      <Select
+        options={editableColumn.constraints?.inclusion || []}
+        label="Default value"
+        value={editableColumn.default}
+        on:change={e => (editableColumn.default = e.detail)}
+        placeholder="None"
+      />
+    {:else}
       <ModalBindableInput
         panel={ServerBindingPanel}
-        title="Default"
-        label="Default"
+        title="Default value"
+        label="Default value"
+        placeholder="None"
         value={editableColumn.default}
-        on:change={e => {
-          editableColumn = {
-            ...editableColumn,
-            default: e.detail,
-          }
-
-          if (e.detail) {
-            setRequired(false)
-          }
-        }}
-        bindings={getBindings({ table })}
+        on:change={e => (editableColumn.default = e.detail)}
+        {bindings}
         allowJS
         context={rowGoldenSample}
       />
-    </div>
+    {/if}
   {/if}
 </Layout>
 
diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts
index 8a8069ce4de..fe7a0ebebf3 100644
--- a/packages/shared-core/src/table.ts
+++ b/packages/shared-core/src/table.ts
@@ -53,8 +53,8 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.DATETIME]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.STRING]: true,
+  [FieldType.OPTIONS]: true,
 
-  [FieldType.OPTIONS]: false,
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
   [FieldType.BARCODEQR]: false,

From 06c8e5f815bce7ce7bfa7a733748a750d31a4ab6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 24 Sep 2024 12:47:51 +0100
Subject: [PATCH 163/455] Fix primary display schemas when saving

---
 .../backend/DataTable/modals/CreateEditColumn.svelte        | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index b04ccdb6a91..1460f92da57 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -296,6 +296,12 @@
       delete saveColumn.default
     }
 
+    // Ensure primary display columns are always required and don't have default values
+    if (primaryDisplay) {
+      saveColumn.constraints.presence = true
+      delete saveColumn.default
+    }
+
     try {
       await tables.saveField({
         originalName,

From f493025d94245588132e2b2d2cb042443e088a58 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 24 Sep 2024 14:47:53 +0100
Subject: [PATCH 164/455] Ensure schema is sanitised when updating primary
 display column

---
 .../DataTable/modals/CreateEditColumn.svelte    |  4 ++--
 .../src/components/grid/stores/datasource.js    | 17 +++++++++++++----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 1460f92da57..e651259615b 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -284,7 +284,7 @@
 
     // Ensure the field is not required if we have a default value
     if (saveColumn.default) {
-      saveColumn.constraints.presence = false
+      saveColumn.constraints.presence = { allowEmpty: true }
     }
 
     // Delete default value for options fields if the option is no longer available
@@ -298,7 +298,7 @@
 
     // Ensure primary display columns are always required and don't have default values
     if (primaryDisplay) {
-      saveColumn.constraints.presence = true
+      saveColumn.constraints.presence = { allowEmpty: false }
       delete saveColumn.default
     }
 
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 68053f38ae4..cf098237210 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -1,6 +1,7 @@
 import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { memo } from "../../../utils"
+import { cloneDeep } from "lodash"
 
 export const createStores = () => {
   const definition = memo(null)
@@ -161,10 +162,18 @@ export const createActions = context => {
 
   // Updates the datasources primary display column
   const changePrimaryDisplay = async column => {
-    return await saveDefinition({
-      ...get(definition),
-      primaryDisplay: column,
-    })
+    let newDefinition = cloneDeep(get(definition))
+
+    // Update primary display
+    newDefinition.primaryDisplay = column
+
+    // Sanitise schema to ensure field is required and has no default value
+    if (!newDefinition.schema[column].constraints) {
+      newDefinition.schema[column].constraints = {}
+    }
+    newDefinition.schema[column].constraints.presence = { allowEmpty: false }
+    delete newDefinition.schema[column].default
+    return await saveDefinition(newDefinition)
   }
 
   // Adds a schema mutation for a single field

From 40372841d840b7535dc72aa4c883787d3bcd3a0b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 24 Sep 2024 15:04:08 +0100
Subject: [PATCH 165/455] Show but disable default values when they are enabled
 but cannot be used

---
 .../DataTable/modals/CreateEditColumn.svelte  | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index e651259615b..c4c3d661a36 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -166,10 +166,8 @@
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
     canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn
-  $: canHaveDefault =
-    !required &&
-    isEnabled("DEFAULT_VALUES") &&
-    canHaveDefaultColumn(editableColumn.type)
+  $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
+  $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== LINK_TYPE &&
     !uneditable &&
@@ -282,9 +280,9 @@
       delete saveColumn.fieldName
     }
 
-    // Ensure the field is not required if we have a default value
-    if (saveColumn.default) {
-      saveColumn.constraints.presence = { allowEmpty: true }
+    // Ensure we don't have a default value if we can't have one
+    if (!canHaveDefault || !defaultValuesEnabled) {
+      delete saveColumn.default
     }
 
     // Delete default value for options fields if the option is no longer available
@@ -302,6 +300,11 @@
       delete saveColumn.default
     }
 
+    // Ensure the field is not required if we have a default value
+    if (saveColumn.default) {
+      saveColumn.constraints.presence = { allowEmpty: true }
+    }
+
     try {
       await tables.saveField({
         originalName,
@@ -786,9 +789,10 @@
     </div>
   {/if}
 
-  {#if canHaveDefault}
+  {#if defaultValuesEnabled}
     {#if editableColumn.type === FieldType.OPTIONS}
       <Select
+        disabled={!canHaveDefault}
         options={editableColumn.constraints?.inclusion || []}
         label="Default value"
         value={editableColumn.default}
@@ -797,6 +801,7 @@
       />
     {:else}
       <ModalBindableInput
+        disabled={!canHaveDefault}
         panel={ServerBindingPanel}
         title="Default value"
         label="Default value"

From 6f054c390aac696fb75409341ccdacca388ab458 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 25 Sep 2024 11:58:23 +0100
Subject: [PATCH 166/455] Update role nodes to always show icons

---
 .../backend/RoleEditor/RoleNode.svelte        | 73 ++++++++++---------
 yarn.lock                                     | 64 +++++++---------
 2 files changed, 68 insertions(+), 69 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index ba0dbe6404c..ac33aa94b08 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -31,7 +31,6 @@
   $: descriptionError = validateDescription(tempDescription)
   $: invalid = nameError || descriptionError
   $: targetClasses = `target${$dragging ? "" : " hidden"}`
-  $: sourceClasses = `source${selected ? "" : " hidden"}`
 
   const validateName = (name, roles) => {
     if (!name?.length) {
@@ -75,21 +74,23 @@
 >
   <div class="color" />
   <div class="content">
-    <div class="name">
-      {data.displayName}
+    <div class="text">
+      <div class="name">
+        {data.displayName}
+      </div>
+      {#if data.description}
+        <div class="description" title={data.description}>
+          {data.description}
+        </div>
+      {/if}
     </div>
-    {#if data.description}
-      <div class="description" title={data.description}>
-        {data.description}
+    {#if data.custom}
+      <div class="buttons">
+        <Icon size="S" name="Edit" hoverable on:click={openPopover} />
+        <Icon size="S" name="Delete" hoverable on:click={deleteModal?.show} />
       </div>
     {/if}
   </div>
-  {#if data.custom}
-    <div class="buttons">
-      <Icon size="S" name="Edit" hoverable on:click={openPopover} />
-      <Icon size="S" name="Delete" hoverable on:click={deleteModal?.show} />
-    </div>
-  {/if}
   {#if id !== Roles.BASIC}
     <Handle
       type="target"
@@ -99,7 +100,7 @@
     />
   {/if}
   {#if id !== Roles.ADMIN}
-    <Handle type="source" position={Position.Right} class={sourceClasses} />
+    <Handle type="source" position={Position.Right} />
   {/if}
 </div>
 
@@ -138,6 +139,7 @@
 </Modal>
 
 <style>
+  /* Node styles */
   .node {
     position: relative;
     background: var(--spectrum-global-color-gray-100);
@@ -146,7 +148,6 @@
     height: var(--height);
     display: flex;
     flex-direction: row;
-    gap: 12px;
     box-sizing: border-box;
     cursor: pointer;
     transition: background 130ms ease-out;
@@ -166,36 +167,32 @@
     flex: 0 0 10px;
     background: var(--color);
   }
+
+  /* Main container */
   .content {
-    width: 0;
     flex: 1 1 auto;
     display: flex;
-    flex-direction: column;
-    justify-content: center;
-    align-items: stretch;
-    gap: 2px;
+    flex-direction: row;
     border: 1px solid var(--border-color);
     border-left-width: 0;
     border-top-right-radius: 4px;
     border-bottom-right-radius: 4px;
+    padding: 12px;
+    gap: 6px;
   }
   .node.selected .content {
-    border-color: transparent;
+    border-color: var(--spectrum-global-color-blue-100);
   }
-  .buttons {
+
+  /* Text */
+  .text {
+    width: 0;
+    flex: 1 1 auto;
     display: flex;
-    flex-direction: row;
-    align-items: center;
-    gap: 6px;
-  }
-  .buttons {
-    display: none;
-    padding-right: 12px;
-  }
-  .buttons :global(.spectrum-Icon) {
-    color: var(--spectrum-global-color-gray-600);
+    flex-direction: column;
+    justify-content: center;
+    align-items: stretch;
   }
-
   .name,
   .description {
     white-space: nowrap;
@@ -206,9 +203,19 @@
     color: var(--spectrum-global-color-gray-600);
     font-size: 12px;
   }
-  .node.selected .buttons {
+
+  /* Icons */
+  .buttons {
     display: flex;
+    flex-direction: row;
+    align-items: center;
+    gap: 6px;
   }
+  .buttons :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  /* Handlers */
   .node :global(.svelte-flow__handle) {
     width: 6px;
     height: 6px;
diff --git a/yarn.lock b/yarn.lock
index 5fba87838f8..3a82dc7066d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17894,21 +17894,11 @@ periscopic@^3.1.0:
     estree-walker "^3.0.0"
     is-reference "^3.0.0"
 
-pg-cloudflare@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
-  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
-
 pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
   integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
 
-pg-connection-string@^2.6.4:
-  version "2.6.4"
-  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d"
-  integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==
-
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
@@ -17919,21 +17909,11 @@ pg-pool@^3.6.0:
   resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e"
   integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==
 
-pg-pool@^3.6.2:
-  version "3.6.2"
-  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2"
-  integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==
-
 pg-protocol@*, pg-protocol@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
   integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
 
-pg-protocol@^1.6.1:
-  version "1.6.1"
-  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3"
-  integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==
-
 pg-types@^2.1.0, pg-types@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
@@ -17958,19 +17938,6 @@ pg@8.10.0:
     pg-types "^2.1.0"
     pgpass "1.x"
 
-pg@^8.12.0:
-  version "8.12.0"
-  resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79"
-  integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==
-  dependencies:
-    pg-connection-string "^2.6.4"
-    pg-pool "^3.6.2"
-    pg-protocol "^1.6.1"
-    pg-types "^2.1.0"
-    pgpass "1.x"
-  optionalDependencies:
-    pg-cloudflare "^1.1.1"
-
 pgpass@1.x:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
@@ -20865,7 +20832,16 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
+"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20956,7 +20932,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20970,6 +20946,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22925,7 +22908,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22943,6 +22926,15 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"

From 3fc3edfe59869ecfe24b4feb6e3ad320466d4c07 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 25 Sep 2024 12:04:18 +0100
Subject: [PATCH 167/455] Allow hovering over edges to make them active as well
 as supporting selection

---
 .../backend/RoleEditor/RoleEdge.svelte        | 31 +++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 94e00fa97f4..84eccc03de7 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -6,7 +6,7 @@
     useSvelteFlow,
   } from "@xyflow/svelte"
   import { Icon, TooltipPosition } from "@budibase/bbui"
-  import { getContext } from "svelte"
+  import { getContext, onMount } from "svelte"
   import { roles } from "stores/builder"
 
   export let sourceX
@@ -23,8 +23,13 @@
   const { updateRole, selectedNodes } = getContext("flow")
 
   let iconHovered = false
+  let edgeHovered = false
 
-  $: active = $selectedNodes.includes(source) || $selectedNodes.includes(target)
+  $: hovered = iconHovered || edgeHovered
+  $: active =
+    hovered ||
+    $selectedNodes.includes(source) ||
+    $selectedNodes.includes(target)
   $: edgeClasses = getEdgeClasses(active, iconHovered)
   $: [edgePath, labelX, labelY] = getBezierPath({
     sourceX,
@@ -54,6 +59,28 @@
     })
     await updateRole(target)
   }
+
+  const onEdgeMouseOver = e => {
+    edgeHovered = true
+  }
+
+  const onEdgeMouseOut = e => {
+    edgeHovered = false
+  }
+
+  onMount(() => {
+    const edge = document.querySelector(`.svelte-flow__edge[data-id="${id}"]`)
+    if (edge) {
+      edge.addEventListener("mouseover", onEdgeMouseOver)
+      edge.addEventListener("mouseout", onEdgeMouseOut)
+    }
+    return () => {
+      if (edge) {
+        edge.removeEventListener("mouseover", onEdgeMouseOver)
+        edge.removeEventListener("mouseout", onEdgeMouseOut)
+      }
+    }
+  })
 </script>
 
 <BaseEdge path={edgePath} class={edgeClasses} />

From bf91266ac056a793687538ec80ebcf97ae55acb1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 25 Sep 2024 12:08:35 +0100
Subject: [PATCH 168/455] Select nodes after creation

---
 .../backend/RoleEditor/RoleFlow.svelte          | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 9284dafc34b..8b2bd5c14dc 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -12,7 +12,7 @@
   import RoleNode from "./RoleNode.svelte"
   import RoleEdge from "./RoleEdge.svelte"
   import { autoLayout } from "./layout"
-  import { setContext } from "svelte"
+  import { setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
   import { GridResolution, MaxAutoZoom } from "./constants"
   import { roles } from "stores/builder"
@@ -115,10 +115,21 @@
     edges.set(newLayout.edges)
   }
 
+  // Manually selects a node
+  const selectNode = roleId => {
+    nodes.update($nodes => {
+      return $nodes.map(node => ({
+        ...node,
+        selected: node.id === roleId,
+      }))
+    })
+  }
+
   // Creates a new role
   const createRole = async () => {
+    const roleId = Helpers.uuid()
     await roles.save({
-      name: Helpers.uuid(),
+      name: roleId,
       uiMetadata: {
         displayName: getSequentialName($roles, "New role ", {
           getName: x => x.uiMetadata.displayName,
@@ -129,6 +140,8 @@
       permissionId: "write",
       inherits: Roles.BASIC,
     })
+    await tick()
+    selectNode(roleId)
   }
 
   // Updates a role with new metadata

From f3c219ace49ea40faf256ef513cad4f55c60690f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 25 Sep 2024 12:52:46 +0100
Subject: [PATCH 169/455] Increase spacing between RBAC editor controls

---
 .../builder/src/components/backend/RoleEditor/Controls.svelte   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 5e241880249..ff41f89daab 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -53,7 +53,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
-    gap: 8px;
+    gap: 12px;
   }
   .top-left {
     top: 20px;

From f08e4497580c231e22d061656119d6db908f6fd5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 26 Sep 2024 09:02:32 +0100
Subject: [PATCH 170/455] Lint

---
 .../builder/src/components/backend/RoleEditor/RoleEdge.svelte | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 84eccc03de7..8419b6a7763 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -60,11 +60,11 @@
     await updateRole(target)
   }
 
-  const onEdgeMouseOver = e => {
+  const onEdgeMouseOver = () => {
     edgeHovered = true
   }
 
-  const onEdgeMouseOut = e => {
+  const onEdgeMouseOut = () => {
     edgeHovered = false
   }
 

From baa088049a98d9814134fc07b9e992838e9f18d5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 27 Sep 2024 08:33:09 +0100
Subject: [PATCH 171/455] Account for both shapes of the required constraint
 and ensure required trumps default values

---
 .../backend/DataTable/modals/CreateEditColumn.svelte          | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index c4c3d661a36..05e0eba0420 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -137,7 +137,9 @@
   $: initialiseField(field, savingColumn)
   $: checkConstraints(editableColumn)
   $: required =
-    primaryDisplay || (!hasDefault && !!editableColumn?.constraints?.presence)
+    primaryDisplay ||
+    editableColumn?.constraints?.presence === true ||
+    editableColumn?.constraints?.presence?.allowEmpty === false
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)

From b53bc5dfafde52441c691e0c4e7ce50bc4e4f149 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 27 Sep 2024 09:08:38 +0100
Subject: [PATCH 172/455] Update tests

---
 packages/server/src/api/routes/tests/rowAction.spec.ts | 2 +-
 packages/server/src/api/routes/tests/viewV2.spec.ts    | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index efd28eb92f9..4fe248984a7 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -835,9 +835,9 @@ describe("/rowsActions", () => {
 
             // Delete explicit view permissions so they inherit table permissions
             await config.api.permission.revoke({
+              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission
               level: PermissionLevel.READ,
               resourceId: viewId,
-              roleId: "inherited",
             })
 
             return { permissionResource: tableId, triggerResouce: viewId }
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index c4a39ae8a91..e727f952e50 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -2248,6 +2248,11 @@ describe.each([
           level: PermissionLevel.READ,
           resourceId: table._id!,
         })
+        await config.api.permission.revoke({
+          roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission
+          level: PermissionLevel.READ,
+          resourceId: view.id,
+        })
         await config.publish()
 
         const response = await config.api.viewV2.publicSearch(view.id)

From 80f1de27de0c1a93cc9b07a5ce988cbf96441231 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 27 Sep 2024 09:19:42 +0100
Subject: [PATCH 173/455] Please just work tests

---
 .../server/src/api/routes/tests/permissions.spec.ts  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 0f059998ae1..d2df14d7af1 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -125,6 +125,12 @@ describe("/permission", () => {
     })
 
     it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
+      // Make view inherit table permissions
+      await config.api.permission.revoke({
+        roleId: STD_ROLE_ID,
+        resourceId: view.id,
+        level: PermissionLevel.READ,
+      })
       // replicate changes before checking permissions
       await config.publish()
 
@@ -138,6 +144,12 @@ describe("/permission", () => {
         resourceId: table._id,
         level: PermissionLevel.READ,
       })
+      // Make view inherit table permissions
+      await config.api.permission.revoke({
+        roleId: STD_ROLE_ID,
+        resourceId: view.id,
+        level: PermissionLevel.READ,
+      })
       // replicate changes before checking permissions
       await config.publish()
 

From 80cdcf99da47eb258222d5a3633af224333918d1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 27 Sep 2024 10:35:23 +0100
Subject: [PATCH 174/455] Add additional comments to tests

---
 packages/server/src/api/routes/tests/permissions.spec.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index d2df14d7af1..625143e7fbc 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -125,7 +125,7 @@ describe("/permission", () => {
     })
 
     it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
-      // Make view inherit table permissions
+      // Make view inherit table permissions. Needed for backwards compatibility with existing views.
       await config.api.permission.revoke({
         roleId: STD_ROLE_ID,
         resourceId: view.id,
@@ -144,7 +144,7 @@ describe("/permission", () => {
         resourceId: table._id,
         level: PermissionLevel.READ,
       })
-      // Make view inherit table permissions
+      // Make view inherit table permissions. Needed for backwards compatibility with existing views.
       await config.api.permission.revoke({
         roleId: STD_ROLE_ID,
         resourceId: view.id,

From 60c55b06eddd7188a51864e83a05cb47d52a661b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 27 Sep 2024 10:39:50 +0100
Subject: [PATCH 175/455] Update new view popover to allow for creating
 calculation views

---
 packages/bbui/src/List/ListItem.svelte        | 75 ++++++++++++++++---
 .../components/common/DetailPopover.svelte    |  5 +-
 .../_components/CreateViewButton.svelte       | 32 +++++++-
 packages/types/src/documents/app/view.ts      |  1 +
 4 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte
index 5b6152781a9..ee6e3259216 100644
--- a/packages/bbui/src/List/ListItem.svelte
+++ b/packages/bbui/src/List/ListItem.svelte
@@ -8,17 +8,22 @@
   export let url = null
   export let hoverable = false
   export let showArrow = false
+  export let selected = false
 </script>
 
 <a
   href={url}
   class="list-item"
   class:hoverable={hoverable || url != null}
+  class:selected
+  class:large={!!subtitle}
   on:click
 >
-  <div class="left">
+  <div class="list-item__left">
     {#if icon}
-      <Icon name={icon} color={iconColor} />
+      <div class="list-item__icon">
+        <Icon name={icon} color={iconColor} size={subtitle ? "XL" : "M"} />
+      </div>
     {/if}
     <div class="list-item__text">
       {#if title}
@@ -33,7 +38,7 @@
       {/if}
     </div>
   </div>
-  <div class="right">
+  <div class="list-item__right">
     <slot name="right" />
     {#if showArrow}
       <Icon name="ChevronRight" />
@@ -49,9 +54,12 @@
     flex-direction: row;
     justify-content: space-between;
     border: 1px solid var(--spectrum-global-color-gray-300);
-    transition: background 130ms ease-out;
+    transition: background 130ms ease-out, border-color 130ms ease-out;
     gap: var(--spacing-m);
     color: var(--spectrum-global-color-gray-800);
+    cursor: pointer;
+    position: relative;
+    box-sizing: border-box;
   }
   .list-item:not(:first-child) {
     border-top: none;
@@ -64,27 +72,71 @@
     border-bottom-left-radius: 4px;
     border-bottom-right-radius: 4px;
   }
-  .hoverable:hover {
-    cursor: pointer;
+  .list-item.hoverable:not(.selected):hover {
     background: var(--spectrum-global-color-gray-200);
+    border-color: var(--spectrum-global-color-gray-400);
   }
 
-  .left,
-  .right {
+  /* Selection is only meant for standalone list items (non stacked) so we just set a fixed border radius */
+  .list-item.selected {
+    background-color: var(--spectrum-global-color-blue-100);
+    border-color: var(--spectrum-global-color-blue-100);
+  }
+  .list-item.selected:after {
+    content: "";
+    position: absolute;
+    height: 100%;
+    width: 100%;
+    border: 1px solid var(--spectrum-global-color-blue-400);
+    pointer-events: none;
+    top: 0;
+    left: 0;
+    border-radius: 4px;
+    box-sizing: border-box;
+    z-index: 1;
+    opacity: 0.5;
+  }
+
+  /* Large icons */
+  .list-item.large .list-item__icon {
+    background-color: var(--spectrum-global-color-gray-200);
+    padding: 4px;
+    border-radius: 4px;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    transition: background-color 130ms ease-out, border-color 130ms ease-out,
+      color 130ms ease-out;
+  }
+  .list-item.large.hoverable:not(.selected):hover .list-item__icon {
+    background-color: var(--spectrum-global-color-gray-300);
+  }
+  .list-item.large.selected .list-item__icon {
+    background-color: var(--spectrum-global-color-blue-400);
+    color: white;
+    border-color: var(--spectrum-global-color-blue-100);
+  }
+
+  /* Internal layout */
+  .list-item__left,
+  .list-item__right {
     display: flex;
     flex-direction: row;
     align-items: center;
     gap: var(--spacing-s);
   }
-  .left {
+  .list-item.large .list-item__left,
+  .list-item.large .list-item__right {
+    gap: var(--spacing-m);
+  }
+  .list-item__left {
     width: 0;
     flex: 1 1 auto;
   }
-  .right {
+  .list-item__right {
     flex: 0 0 auto;
     color: var(--spectrum-global-color-gray-600);
   }
 
+  /* Text */
   .list-item__text {
     flex: 1 1 auto;
     width: 0;
@@ -96,6 +148,7 @@
     text-overflow: ellipsis;
   }
   .list-item__subtitle {
-    color: var(--spectrum-global-color-gray-600);
+    color: var(--spectrum-global-color-gray-700);
+    font-size: 12px;
   }
 </style>
diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index bc891bfe421..c437804b332 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -4,6 +4,7 @@
   export let title
   export let align = "left"
   export let showPopover
+  export let width
 
   let popover
   let anchor
@@ -22,8 +23,8 @@
 <Popover
   bind:this={popover}
   bind:open
-  minWidth={400}
-  maxWidth={400}
+  minWidth={width || 400}
+  maxWidth={width || 400}
   {anchor}
   {align}
   {showPopover}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index 8af5007022b..f3261a39b47 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -1,12 +1,13 @@
 <script>
   import DetailPopover from "components/common/DetailPopover.svelte"
-  import { Input, notifications, Button, Icon } from "@budibase/bbui"
+  import { Input, notifications, Button, Icon, ListItem } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { viewsV2 } from "stores/builder"
 
   export let table
   export let firstView = false
 
+  let calculation = false
   let name
   let popover
 
@@ -39,6 +40,7 @@
         tableId: table._id,
         schema: enrichSchema(table.schema),
         primaryDisplay: table.primaryDisplay,
+        calculation,
       })
       notifications.success(`View ${name} created`)
       $goto(`./${newView.id}`)
@@ -52,6 +54,7 @@
   title="Create view"
   bind:this={popover}
   on:open={() => (name = null)}
+  width={540}
 >
   <svelte:fragment slot="anchor" let:open>
     {#if firstView}
@@ -66,6 +69,28 @@
       </div>
     {/if}
   </svelte:fragment>
+  <div class="options">
+    <div>
+      <ListItem
+        title="Table"
+        subtitle="Create a subset of your data"
+        hoverable
+        on:click={() => (calculation = false)}
+        selected={!calculation}
+        icon="Rail"
+      />
+    </div>
+    <div>
+      <ListItem
+        title="Calculation"
+        subtitle="Calculate groups of rows"
+        hoverable
+        on:click={() => (calculation = true)}
+        selected={calculation}
+        icon="123"
+      />
+    </div>
+  </div>
   <Input
     label="Name"
     thin
@@ -81,6 +106,11 @@
 </DetailPopover>
 
 <style>
+  .options {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: var(--spacing-m);
+  }
   .icon {
     height: 32px;
     padding: 0 8px;
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 16817f177ac..04ee6b01a1f 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -72,6 +72,7 @@ export interface ViewV2 {
     type?: SortType
   }
   schema?: Record<string, ViewFieldMetadata>
+  calculation?: boolean
 }
 
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema

From 5853f3b148bb8322cb979d3f69f070484f8da1aa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 27 Sep 2024 14:37:10 +0200
Subject: [PATCH 176/455] Fix signature theme

---
 .../data/table/[tableId]/[viewId]/index.svelte              | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 8b1ad6cda17..395889ba16f 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "stores/builder"
-  import { admin } from "stores/portal"
+  import { admin, themeStore } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@@ -24,6 +24,9 @@
   $: buttons = makeRowActionButtons($rowActions[id])
   $: rowActions.refreshRowActions(id)
 
+  $: currentTheme = $themeStore?.theme
+  $: darkMode = !currentTheme.includes("light")
+
   const makeRowActionButtons = actions => {
     return (actions || []).map(action => ({
       text: action.name,
@@ -40,6 +43,7 @@
 
 <Grid
   {API}
+  {darkMode}
   {datasource}
   allowAddRows
   allowDeleteRows

From 8fab576b5ab75b4283827b420239a38980a97c0b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 30 Sep 2024 08:23:57 +0100
Subject: [PATCH 177/455] Remove duplicate dark mode attribute

---
 .../app/[application]/data/table/[tableId]/[viewId]/index.svelte | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index b1c9c84e423..b5b65b3f1d1 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -48,7 +48,6 @@
   {darkMode}
   {datasource}
   {buttons}
-  {darkMode}
   allowAddRows
   allowDeleteRows
   showAvatars={false}

From 2099ee8bafb2b256411c92a28be3096fc0c4b487 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 30 Sep 2024 08:24:42 +0100
Subject: [PATCH 178/455] Fix duplicates

---
 .../[application]/data/table/[tableId]/[viewId]/index.svelte   | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index be682fb744f..042c6f59fed 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -26,9 +26,6 @@
   $: currentTheme = $themeStore?.theme
   $: darkMode = !currentTheme.includes("light")
 
-  $: currentTheme = $themeStore?.theme
-  $: darkMode = !currentTheme.includes("light")
-
   const makeRowActionButtons = actions => {
     return (actions || []).map(action => ({
       text: action.name,

From 354b5041c74e95620729c8523a8e916ad9ee4985 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 1 Oct 2024 09:49:53 +0100
Subject: [PATCH 179/455] Update handling of basic and admin edges

---
 .../backend/RoleEditor/RoleEdge.svelte        | 11 +---
 .../backend/RoleEditor/RoleFlow.svelte        | 52 +++++++++++++------
 .../backend/RoleEditor/RoleNode.svelte        |  7 ++-
 .../components/backend/RoleEditor/layout.js   | 50 +++++++++---------
 4 files changed, 67 insertions(+), 53 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 8419b6a7763..14b1e03efa5 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -20,7 +20,7 @@
   export let target
 
   const flow = useSvelteFlow()
-  const { updateRole, selectedNodes } = getContext("flow")
+  const { deleteEdge, selectedNodes } = getContext("flow")
 
   let iconHovered = false
   let edgeHovered = false
@@ -53,13 +53,6 @@
     return classes
   }
 
-  const deleteEdge = async () => {
-    flow.deleteElements({
-      edges: [{ id }],
-    })
-    await updateRole(target)
-  }
-
   const onEdgeMouseOver = () => {
     edgeHovered = true
   }
@@ -92,7 +85,7 @@
     style:transform="translate(-50%, -50%) translate({labelX}px,{labelY}px)"
     class="edge-label nodrag nopan"
     class:active
-    on:click={deleteEdge}
+    on:click={() => deleteEdge(id)}
     on:mouseover={() => (iconHovered = true)}
     on:mouseout={() => (iconHovered = false)}
   >
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 8b2bd5c14dc..04ff540a823 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -31,26 +31,33 @@
   $: handleExternalRoleChanges($roles)
 
   // Converts a role doc into a node structure
-  const roleToNode = role => ({
-    id: role._id,
-    sourcePosition: Position.Right,
-    targetPosition: Position.Left,
-    type: "role",
-    position: { x: 0, y: 0 },
-    data: {
-      ...role.uiMetadata,
-      custom: !role._id.match(/[A-Z]+/),
-    },
-  })
+  const roleToNode = role => {
+    const custom = !role._id.match(/[A-Z]+/)
+
+    return {
+      id: role._id,
+      sourcePosition: Position.Right,
+      targetPosition: Position.Left,
+      type: "role",
+      position: { x: 0, y: 0 },
+      data: {
+        ...role.uiMetadata,
+        custom,
+      },
+      deletable: custom,
+      draggable: custom,
+      connectable: custom,
+    }
+  }
 
   // Converts a node structure back into a role doc
   const nodeToRole = node => {
     const role = $roles.find(x => x._id === node.id)
-    // const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
+    const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
     // TODO save inherits array
     return {
       ...role,
-      // inherits,
+      inherits,
       uiMetadata: {
         displayName: node.data.displayName,
         color: node.data.color,
@@ -63,7 +70,11 @@
   const rolesToLayout = roles => {
     let nodes = []
     let edges = []
-    for (let role of roles.filter(role => role._id !== Roles.PUBLIC)) {
+
+    // Remove some builtins
+    const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
+    roles = roles.filter(role => !ignoredRoles.includes(role._id))
+    for (let role of roles) {
       // Add node for this role
       nodes.push(roleToNode(role))
 
@@ -138,7 +149,6 @@
         description: "Custom role",
       },
       permissionId: "write",
-      inherits: Roles.BASIC,
     })
     await tick()
     selectNode(roleId)
@@ -148,7 +158,7 @@
   const updateRole = async (roleId, metadata) => {
     // Don't update builtins
     const node = $nodes.find(x => x.id === roleId)
-    if (!node || !node.data.custom) {
+    if (!node) {
       return
     }
 
@@ -170,6 +180,15 @@
     }
   }
 
+  const deleteEdge = async edgeId => {
+    const edge = $edges.find(x => x.id === edgeId)
+    if (!edge) {
+      return
+    }
+    edges.set($edges.filter(x => x.id !== edgeId))
+    await updateRole(edge.target)
+  }
+
   // Saves a new connection
   const onConnect = async connection => {
     await updateRole(connection.target)
@@ -183,6 +202,7 @@
     createRole,
     updateRole,
     deleteRole,
+    deleteEdge,
   })
 </script>
 
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index ac33aa94b08..bae7c797553 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -17,6 +17,7 @@
   export let data
   export let id
   export let selected
+  export let isConnectable
 
   const { dragging, updateRole, deleteRole } = getContext("flow")
 
@@ -30,7 +31,7 @@
   $: nameError = validateName(tempDisplayName, $roles)
   $: descriptionError = validateDescription(tempDescription)
   $: invalid = nameError || descriptionError
-  $: targetClasses = `target${$dragging ? "" : " hidden"}`
+  $: targetClasses = `target${!$dragging ? " hidden" : ""}`
 
   const validateName = (name, roles) => {
     if (!name?.length) {
@@ -91,15 +92,13 @@
       </div>
     {/if}
   </div>
-  {#if id !== Roles.BASIC}
+  {#if isConnectable}
     <Handle
       type="target"
       position={Position.Left}
       class={targetClasses}
       isConnectable={$dragging}
     />
-  {/if}
-  {#if id !== Roles.ADMIN}
     <Handle type="source" position={Position.Right} />
   {/if}
 </div>
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
index 103b585ef54..90522680458 100644
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ b/packages/builder/src/components/backend/RoleEditor/layout.js
@@ -10,7 +10,7 @@ export const dagreLayout = ({ nodes, edges }) => {
   dagreGraph.setDefaultEdgeLabel(() => ({}))
   dagreGraph.setGraph({
     rankdir: "LR",
-    ranksep: GridResolution * 8,
+    ranksep: GridResolution * 4,
     nodesep: GridResolution * 2,
   })
   nodes.forEach(node => {
@@ -19,6 +19,26 @@ export const dagreLayout = ({ nodes, edges }) => {
   edges.forEach(edge => {
     dagreGraph.setEdge(edge.source, edge.target)
   })
+
+  // Add ephemeral edges for basic and admin so that we can position them properly
+
+  for (let node of nodes) {
+    if (
+      !edges.some(x => x.target === node.id) &&
+      node.id !== Roles.BASIC &&
+      node.id !== Roles.ADMIN
+    ) {
+      dagreGraph.setEdge(Roles.BASIC, node.id)
+    }
+    if (
+      !edges.some(x => x.source === node.id) &&
+      node.id !== Roles.BASIC &&
+      node.id !== Roles.ADMIN
+    ) {
+      dagreGraph.setEdge(node.id, Roles.ADMIN)
+    }
+  }
+
   dagre.layout(dagreGraph)
   nodes.forEach(node => {
     const pos = dagreGraph.node(node.id)
@@ -34,32 +54,14 @@ export const dagreLayout = ({ nodes, edges }) => {
 
 // Adds additional edges as needed to the flow structure to ensure compatibility with BB role logic
 const sanitiseLayout = ({ nodes, edges }) => {
-  let additions = []
-
-  for (let node of nodes) {
-    // If a node does not inherit anything, let it inherit basic
-    if (!edges.some(x => x.target === node.id) && node.id !== Roles.BASIC) {
-      additions.push({
-        id: Helpers.uuid(),
-        source: Roles.BASIC,
-        target: node.id,
-        animated: true,
-      })
-    }
-
-    // If a node is not inherited by anything, let it be inherited by admin
-    if (!edges.some(x => x.source === node.id) && node.id !== Roles.ADMIN) {
-      additions.push({
-        id: Helpers.uuid(),
-        source: node.id,
-        target: Roles.ADMIN,
-      })
-    }
-  }
+  // Remove any inheritance of basic and admin since this is implied
+  edges = edges.filter(
+    edge => edge.source !== Roles.BASIC && edge.target !== Roles.ADMIN
+  )
 
   return {
     nodes,
-    edges: [...edges, ...additions],
+    edges,
   }
 }
 

From db41b4c385c98ced392c9feec20fe5a0adf8e100 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 1 Oct 2024 10:25:32 +0100
Subject: [PATCH 180/455] Add initial differences for calculation views

---
 .../grid/GridViewCalculationButton.svelte     | 19 +++++++++++++++++++
 .../table/[tableId]/[viewId]/index.svelte     | 17 ++++++++++++-----
 .../_components/CreateViewButton.svelte       |  4 ++--
 .../src/api/controllers/view/viewsV2.ts       |  2 ++
 4 files changed, 35 insertions(+), 7 deletions(-)
 create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
new file mode 100644
index 00000000000..6bed99cac93
--- /dev/null
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -0,0 +1,19 @@
+<script>
+  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import { getContext, createEventDispatcher } from "svelte"
+
+  const { datasource } = getContext("grid")
+  const dispatch = createEventDispatcher()
+
+  let modal
+</script>
+
+<ActionButton icon="WebPage" quiet on:click={modal?.show}>
+  Configure calculations
+</ActionButton>
+
+<Modal bind:this={modal}>
+  <ModalContent title="Calculations" confirmText="Save" size="L">
+    Show calculations which are based on
+  </ModalContent>
+</Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 8b1ad6cda17..4e313ddc62b 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -12,14 +12,16 @@
   import GridGenerateButton from "components/backend/DataTable/buttons/grid/GridGenerateButton.svelte"
   import GridScreensButton from "components/backend/DataTable/buttons/grid/GridScreensButton.svelte"
   import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
+  import GridViewCalculationButton from "components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte"
 
   let generateButton
 
-  $: id = $viewsV2.selected?.id
+  $: view = $viewsV2.selected
+  $: id = view?.id
   $: datasource = {
     type: "viewV2",
     id,
-    tableId: $viewsV2.selected?.tableId,
+    tableId: view?.tableId,
   }
   $: buttons = makeRowActionButtons($rowActions[id])
   $: rowActions.refreshRowActions(id)
@@ -50,13 +52,18 @@
   buttonsCollapsed
 >
   <svelte:fragment slot="controls">
+    {#if view?.calculation}
+      <GridViewCalculationButton />
+    {/if}
     <GridFilterButton />
     <GridSortButton />
     <GridSizeButton />
-    <GridColumnsSettingButton />
     <GridManageAccessButton />
-    <GridRowActionsButton />
-    <GridScreensButton on:request-generate={() => generateButton?.show()} />
+    {#if !view?.calculation}
+      <GridColumnsSettingButton />
+      <GridRowActionsButton />
+      <GridScreensButton on:request-generate={() => generateButton?.show()} />
+    {/if}
   </svelte:fragment>
   <svelte:fragment slot="controls-right">
     <GridGenerateButton bind:this={generateButton} />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index f3261a39b47..8f6eba3f0b6 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -38,8 +38,8 @@
       const newView = await viewsV2.create({
         name: trimmedName,
         tableId: table._id,
-        schema: enrichSchema(table.schema),
-        primaryDisplay: table.primaryDisplay,
+        schema: calculation ? {} : enrichSchema(table.schema),
+        primaryDisplay: calculation ? undefined : table.primaryDisplay,
         calculation,
       })
       notifications.success(`View ${name} created`)
diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index 7f6f6385419..70434ad4c52 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -106,6 +106,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
+    calculation: view.calculation,
   }
   const result = await sdk.views.create(tableId, parsedView)
   ctx.status = 201
@@ -141,6 +142,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
+    calculation: view.calculation,
   }
 
   const result = await sdk.views.update(tableId, parsedView)

From 930dd812cc0040242d2e21bc3f5ab2e00a25f867 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 1 Oct 2024 10:59:48 +0100
Subject: [PATCH 181/455] Add uiMetadata to viewv2

---
 .../grid/GridViewCalculationButton.svelte     | 24 +++++++++++++++++--
 .../table/[tableId]/[viewId]/index.svelte     |  8 +++----
 .../_components/CreateViewButton.svelte       |  6 +++--
 .../frontend-core/src/fetch/ViewV2Fetch.js    | 12 ++++++++++
 .../src/api/controllers/view/viewsV2.ts       |  4 ++--
 packages/types/src/documents/app/view.ts      |  2 +-
 6 files changed, 44 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 6bed99cac93..33931cdd3e2 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -1,11 +1,26 @@
 <script>
   import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import { CalculationType } from "@budibase/types"
+  import { API } from "api"
   import { getContext, createEventDispatcher } from "svelte"
 
-  const { datasource } = getContext("grid")
+  const { definition } = getContext("grid")
   const dispatch = createEventDispatcher()
 
   let modal
+
+  const save = async () => {
+    await API.viewV2.update({
+      ...$definition,
+      schema: {
+        "Average game length": {
+          visible: true,
+          calculationType: CalculationType.AVG,
+          field: "Game Length",
+        },
+      },
+    })
+  }
 </script>
 
 <ActionButton icon="WebPage" quiet on:click={modal?.show}>
@@ -13,7 +28,12 @@
 </ActionButton>
 
 <Modal bind:this={modal}>
-  <ModalContent title="Calculations" confirmText="Save" size="L">
+  <ModalContent
+    title="Calculations"
+    confirmText="Save"
+    size="L"
+    onConfirm={save}
+  >
     Show calculations which are based on
   </ModalContent>
 </Modal>
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 715268d9016..112722363a1 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -17,6 +17,7 @@
   let generateButton
 
   $: view = $viewsV2.selected
+  $: calculation = view?.uiMetadata?.calculation
   $: id = view?.id
   $: datasource = {
     type: "viewV2",
@@ -28,9 +29,6 @@
   $: currentTheme = $themeStore?.theme
   $: darkMode = !currentTheme.includes("light")
 
-  $: currentTheme = $themeStore?.theme
-  $: darkMode = !currentTheme.includes("light")
-
   const makeRowActionButtons = actions => {
     return (actions || []).map(action => ({
       text: action.name,
@@ -58,14 +56,14 @@
   buttonsCollapsed
 >
   <svelte:fragment slot="controls">
-    {#if view?.calculation}
+    {#if calculation}
       <GridViewCalculationButton />
     {/if}
     <GridFilterButton />
     <GridSortButton />
     <GridSizeButton />
     <GridManageAccessButton />
-    {#if !view?.calculation}
+    {#if !calculation}
       <GridColumnsSettingButton />
       <GridRowActionsButton />
       <GridScreensButton on:request-generate={() => generateButton?.show()} />
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index 8f6eba3f0b6..c420bb508e0 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -38,9 +38,11 @@
       const newView = await viewsV2.create({
         name: trimmedName,
         tableId: table._id,
-        schema: calculation ? {} : enrichSchema(table.schema),
+        schema: enrichSchema(table.schema),
         primaryDisplay: calculation ? undefined : table.primaryDisplay,
-        calculation,
+        uiMetadata: {
+          calculation,
+        },
       })
       notifications.success(`View ${name} created`)
       $goto(`./${newView.id}`)
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 40135746df3..191cb87b54c 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -39,6 +39,18 @@ export default class ViewV2Fetch extends DataFetch {
       this.options
     const { cursor, query, definition } = get(this.store)
 
+    // If this is a calculation view and there are no schema fields then do nothing
+    console.log(definition)
+    if (
+      definition.calculation &&
+      !Object.keys(definition.schema || {}).length
+    ) {
+      return {
+        rows: [],
+        hasNextPage: false,
+      }
+    }
+
     // If sort/filter params are not defined, update options to store the
     // params built in to this view. This ensures that we can accurately
     // compare old and new params and skip a redundant API call.
diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index 70434ad4c52..1d19e2d19c8 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -106,7 +106,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
-    calculation: view.calculation,
+    uiMetadata: view.uiMetadata,
   }
   const result = await sdk.views.create(tableId, parsedView)
   ctx.status = 201
@@ -142,7 +142,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
-    calculation: view.calculation,
+    uiMetadata: view.uiMetadata,
   }
 
   const result = await sdk.views.update(tableId, parsedView)
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 04ee6b01a1f..9c558a8fb5d 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -72,7 +72,7 @@ export interface ViewV2 {
     type?: SortType
   }
   schema?: Record<string, ViewFieldMetadata>
-  calculation?: boolean
+  uiMetadata?: Record<string, any>
 }
 
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema

From 2d21afbe101e646261652b5a6efbd27d504e4a41 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 1 Oct 2024 12:52:11 +0100
Subject: [PATCH 182/455] rough v1

---
 packages/backend-core/src/sql/sql.ts          |   3 +-
 packages/backend-core/src/sql/sqlTable.ts     |   5 +-
 .../DataTable/modals/CreateEditColumn.svelte  |  10 ++
 .../common/AIFieldConfiguration.svelte        | 123 ++++++++++++++++++
 .../builder/src/constants/backend/index.js    |   6 +
 .../src/components/FilterBuilder.svelte       |   2 +-
 .../components/grid/cells/HeaderCell.svelte   |   3 +-
 .../src/components/grid/lib/renderers.js      |   2 +
 packages/frontend-core/src/constants.js       |   1 +
 .../api/controllers/row/ExternalRequest.ts    |   3 +-
 .../src/api/controllers/row/aiColumn.ts       |  43 ++++++
 .../src/api/controllers/row/staticFormula.ts  |  12 +-
 .../src/api/controllers/row/utils/sqlUtils.ts |   1 +
 .../src/api/controllers/table/bulkFormula.ts  |  27 +++-
 .../server/src/api/controllers/table/utils.ts |   2 +-
 packages/server/src/db/linkedRows/index.ts    |   2 +-
 .../server/src/integrations/googlesheets.ts   |   4 +-
 .../src/sdk/app/tables/internal/index.ts      |   3 +-
 .../server/src/sdk/app/tables/internal/sqs.ts |   1 +
 packages/server/src/sdk/app/views/index.ts    |   2 +-
 .../src/utilities/rowProcessor/index.ts       |   4 +
 .../src/utilities/rowProcessor/utils.ts       |  51 ++++++++
 packages/shared-core/src/filters.ts           |   2 +
 packages/shared-core/src/table.ts             |   3 +
 packages/types/src/documents/app/row.ts       |   4 +
 .../src/documents/app/table/constants.ts      |   1 +
 .../types/src/documents/app/table/schema.ts   |  13 ++
 27 files changed, 314 insertions(+), 19 deletions(-)
 create mode 100644 packages/builder/src/components/common/AIFieldConfiguration.svelte
 create mode 100644 packages/server/src/api/controllers/row/aiColumn.ts

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 2b20938981e..8c97b334180 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1141,7 +1141,8 @@ class InternalBuilder {
           schema.constraints?.presence === true ||
           schema.type === FieldType.FORMULA ||
           schema.type === FieldType.AUTO ||
-          schema.type === FieldType.LINK
+          schema.type === FieldType.LINK ||
+          schema.type === FieldType.AI
         ) {
           continue
         }
diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts
index f5b02cc4e4b..84f4e290aa5 100644
--- a/packages/backend-core/src/sql/sqlTable.ts
+++ b/packages/backend-core/src/sql/sqlTable.ts
@@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
 
 function isIgnoredType(type: FieldType) {
-  const ignored = [FieldType.LINK, FieldType.FORMULA]
+  const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
   return ignored.indexOf(type) !== -1
 }
 
@@ -144,6 +144,9 @@ function generateSchema(
       case FieldType.FORMULA:
         // This is allowed, but nothing to do on the external datasource
         break
+      case FieldType.AI:
+        // This is allowed, but nothing to do on the external datasource
+        break
       case FieldType.ATTACHMENTS:
       case FieldType.ATTACHMENT_SINGLE:
       case FieldType.SIGNATURE_SINGLE:
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 0130c397156..da68b4182b0 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -33,6 +33,7 @@
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
@@ -54,6 +55,7 @@
   const NUMBER_TYPE = FieldType.NUMBER
   const JSON_TYPE = FieldType.JSON
   const DATE_TYPE = FieldType.DATETIME
+  const AI_TYPE = FieldType.AI
 
   const dispatch = createEventDispatcher()
   const { dispatch: gridDispatch, rows } = getContext("grid")
@@ -421,6 +423,7 @@
         FIELDS.ATTACHMENT_SINGLE,
         FIELDS.ATTACHMENTS,
         FIELDS.FORMULA,
+        FIELDS.AI,
         FIELDS.JSON,
         FIELDS.BARCODEQR,
         FIELDS.SIGNATURE_SINGLE,
@@ -732,6 +735,13 @@
         />
       </div>
     </div>
+  {:else if editableColumn.type === AI_TYPE}
+    <AIFieldConfiguration
+      aiField={editableColumn}
+      context={rowGoldenSample}
+      bindings={getBindings({ table })}
+      schema={table.schema}
+    />
   {:else if editableColumn.type === JSON_TYPE}
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte
new file mode 100644
index 00000000000..66af1266bed
--- /dev/null
+++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte
@@ -0,0 +1,123 @@
+<script>
+  import { Input, Multiselect, Select, TextArea } from "@budibase/bbui"
+  import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
+  import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
+
+  const AIOperations = {
+    SUMMARISE_TEXT: {
+      label: "Summarise Text",
+      value: "SUMMARISE_TEXT"
+    },
+    CLEAN_DATA: {
+      label: "Clean Data",
+      value: "CLEAN_DATA"
+    },
+    TRANSLATE: {
+      label: "Translate",
+      value: "TRANSLATE"
+    },
+    CATEGORISE_TEXT: {
+      label: "Categorise Text",
+      value: "CATEGORISE_TEXT"
+    },
+    SENTIMENT_ANALYSIS: {
+      label: "Sentiment Analysis",
+      value: "SENTIMENT_ANALYSIS"
+    },
+    PROMPT: {
+      label: "Prompt",
+      value: "PROMPT"
+    },
+    SEARCH_WEB: {
+      label: "Search Web",
+      value: "SEARCH_WEB"
+    }
+  }
+
+  const OperationFieldTypes = {
+    MULTI_COLUMN: "columns",
+    COLUMN: "column",
+    BINDABLE_TEXT: "prompt",
+    // LANGUAGE: "language",
+  }
+
+  const OperationFields = {
+    SUMMARISE_TEXT: {
+      columns: OperationFieldTypes.MULTI_COLUMN,
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    },
+    CLEAN_DATA: {
+      columns: OperationFieldTypes.MULTI_COLUMN,
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    },
+    TRANSLATE: {
+      columns: OperationFieldTypes.MULTI_COLUMN,
+      language: OperationFieldTypes.BINDABLE_TEXT,
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    },
+    CATEGORISE_TEXT: {
+      columns: OperationFieldTypes.MULTI_COLUMN,
+      categories: OperationFieldTypes.BINDABLE_TEXT,
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    },
+    SENTIMENT_ANALYSIS: {
+      column: OperationFieldTypes.COLUMN,
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    },
+    PROMPT: {
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    },
+    SEARCH_WEB: {
+      columns: OperationFieldTypes.MULTI_COLUMN,
+      prompt: OperationFieldTypes.BINDABLE_TEXT,
+    }
+  }
+
+  const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({
+    label: AIOperations[key].label,
+    value: AIOperations[key].value
+  }))
+
+  export let bindings
+  export let context
+  export let schema
+  export let aiField = {}
+
+  $: OperationField = OperationFields[aiField.operation] || null
+  $: console.log(aiField)
+  $: console.log(schema)
+  $: schemaWithoutRelations = Object.keys(schema).filter(key => schema[key].type !== "link")
+</script>
+
+<Select
+  options={AIFieldConfigOptions}
+  bind:value={aiField.operation}
+/>
+{#if aiField.operation}
+  {#each Object.keys(OperationField) as key}
+    {#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT}
+      <ModalBindableInput
+        label={key}
+        panel={ServerBindingPanel}
+        title="Prompt"
+        on:change={e => aiField[key] = e.detail}
+        value={aiField[key]}
+        {bindings}
+        allowJS
+        {context}
+      />
+    {:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN}
+      <Multiselect
+        bind:value={aiField[key]}
+        label={key}
+        options={schemaWithoutRelations}
+      />
+    {:else if OperationField[key] === OperationFieldTypes.COLUMN}
+      <Select
+        bind:value={aiField[key]}
+        label={key}
+        options={schemaWithoutRelations}
+      />
+    {/if}
+  {/each}
+{/if}
\ No newline at end of file
diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js
index 6fbc36afe25..6ddf4c21383 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@@ -159,6 +159,12 @@ export const FIELDS = {
     icon: TypeIconMap[FieldType.FORMULA],
     constraints: {},
   },
+  AI: {
+    name: "AI",
+    type: FieldType.AI,
+    icon: TypeIconMap[FieldType.AI],
+    constraints: {},
+  },
   JSON: {
     name: "JSON",
     type: FieldType.JSON,
diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte
index 3a0c789b9e1..d255d022bf9 100644
--- a/packages/frontend-core/src/components/FilterBuilder.svelte
+++ b/packages/frontend-core/src/components/FilterBuilder.svelte
@@ -262,7 +262,7 @@
               {/if}
               {#if allowBindings && filter.field && filter.valueType === "Binding"}
                 <slot name="binding" {filter} />
-              {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
+              {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)}
                 <Input disabled={filter.noValue} bind:value={filter.value} />
               {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)}
                 <Multiselect
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index f516ba7d297..bdfbc93cd69 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -95,7 +95,8 @@
     const { type, formulaType } = col.schema
     return (
       searchableTypes.includes(type) ||
-      (type === FieldType.FORMULA && formulaType === FormulaType.STATIC)
+      (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) ||
+      type === FieldType.AI
     )
   }
 
diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index 6af21e65f07..3d0c0c77970 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -30,6 +30,8 @@ const TypeComponentMap = {
   [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
+  // TODO: fix
+  [FieldType.AI]: FormulaCell,
   [FieldType.JSON]: JSONCell,
   [FieldType.BB_REFERENCE]: BBReferenceCell,
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index 105928c66ba..dfdcaf5c014 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -160,6 +160,7 @@ export const TypeIconMap = {
   [FieldType.ATTACHMENT_SINGLE]: "DocumentFragment",
   [FieldType.LINK]: "DataCorrelated",
   [FieldType.FORMULA]: "Calculator",
+  [FieldType.AI]: "MagicWand",
   [FieldType.JSON]: "Brackets",
   [FieldType.BIGINT]: "TagBold",
   [FieldType.AUTO]: "MagicWand",
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 9c9bd0b2841..b153b442a8a 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -154,7 +154,8 @@ function isEditableColumn(column: FieldSchema) {
     column.autoReason !== AutoReason.FOREIGN_KEY &&
     column.subtype !== AutoFieldSubType.AUTO_ID
   const isFormula = column.type === FieldType.FORMULA
-  return !(isExternalAutoColumn || isFormula)
+  const isAIColumn = column.type === FieldType.AI
+  return !(isExternalAutoColumn || isFormula || isAIColumn)
 }
 
 export class ExternalRequest<T extends Operation> {
diff --git a/packages/server/src/api/controllers/row/aiColumn.ts b/packages/server/src/api/controllers/row/aiColumn.ts
new file mode 100644
index 00000000000..fc543c34870
--- /dev/null
+++ b/packages/server/src/api/controllers/row/aiColumn.ts
@@ -0,0 +1,43 @@
+import { getRowParams } from "../../../db/utils"
+import {
+  outputProcessing,
+  processAIColumns,
+} from "../../../utilities/rowProcessor"
+import { context } from "@budibase/backend-core"
+import { Table, Row } from "@budibase/types"
+import isEqual from "lodash/isEqual"
+import { cloneDeep } from "lodash/fp"
+
+export async function updateAllAIColumnsInTable(table: Table) {
+  const db = context.getAppDB()
+  // start by getting the raw rows (which will be written back to DB after update)
+  let rows = (
+    await db.allDocs<Row>(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc!)
+  // now enrich the rows, note the clone so that we have the base state of the
+  // rows so that we don't write any of the enriched information back
+  let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
+    squash: false,
+  })
+  const updatedRows = []
+  for (let row of rows) {
+    // find the enriched row, if found process the formulas
+    const enrichedRow = enrichedRows.find(
+      (enriched: Row) => enriched._id === row._id
+    )
+    if (enrichedRow) {
+      let processed = await processAIColumns(table, cloneDeep(row), {
+        contextRows: [enrichedRow],
+      })
+      // values have changed, need to add to bulk docs to update
+      if (!isEqual(processed, row)) {
+        updatedRows.push(processed)
+      }
+    }
+  }
+  await db.bulkDocs(updatedRows)
+}
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index 777379db144..54cb7d0728d 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -1,6 +1,6 @@
 import { getRowParams } from "../../../db/utils"
 import {
-  outputProcessing,
+  outputProcessing, processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
@@ -101,7 +101,7 @@ export async function updateAllFormulasInTable(table: Table) {
       (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
-      const processed = await processFormulas(table, cloneDeep(row), {
+      let processed = await processFormulas(table, cloneDeep(row), {
         dynamic: false,
         contextRows: [enrichedRow],
       })
@@ -142,6 +142,10 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
+  row = await processAIColumns(table, row, {
+    contextRows: [enrichedRow],
+  })
+
   // don't worry about rev, tables handle rev/lastID updates
   // if another row has been written since processing this will
   // handle the auto ID clash
@@ -154,6 +158,10 @@ export async function finaliseRow(
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
+  enrichedRow = await processAIColumns(table, row, {
+    contextRows: [enrichedRow],
+  })
+
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 249bb43bbce..22b34e1720c 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -119,6 +119,7 @@ export function buildSqlFieldList(
         ([columnName, column]) =>
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
+          column.type !== FieldType.AI &&
           !existing.find((field: string) => field === columnName)
       )
       .map(column => `${table.name}.${column[0]}`)
diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts
index 060b67e8ce6..6e47f30233f 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.ts
+++ b/packages/server/src/api/controllers/table/bulkFormula.ts
@@ -5,15 +5,10 @@ import isEqual from "lodash/isEqual"
 import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
-import {
-  FormulaType,
-  FieldSchema,
-  FieldType,
-  FormulaFieldMetadata,
-  Table,
-} from "@budibase/types"
+import { FieldSchema, FieldType, FormulaFieldMetadata, FormulaType, Table, } from "@budibase/types"
 import sdk from "../../../sdk"
 import { isRelationshipColumn } from "../../../db/utils"
+import { updateAllAIColumnsInTable } from "../row/aiColumn"
 
 function isStaticFormula(
   column: FieldSchema
@@ -198,3 +193,21 @@ export async function runStaticFormulaChecks(
     await checkIfFormulaUpdated(table, { oldTable })
   }
 }
+
+export async function runAIColumnChecks(
+  table: Table,
+  { oldTable }: { oldTable?: Table }
+) {
+  // look to see if any formula values have changed
+  const shouldUpdate = Object.values(table.schema).find(
+    column =>
+      column.type === FieldType.AI &&
+      (!oldTable ||
+        !oldTable.schema[column.name] ||
+        !isEqual(oldTable.schema[column.name], column))
+  )
+  // if a static formula column has updated, then need to run the update
+  if (shouldUpdate != null) {
+    await updateAllAIColumnsInTable(table)
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index 269f079ae8a..b916cb9c041 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -33,7 +33,7 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import env from "../../../environment"
-import { runStaticFormulaChecks } from "./bulkFormula"
+import { runAIColumnChecks, runStaticFormulaChecks } from "./bulkFormula"
 
 export async function clearColumns(table: Table, columnNames: string[]) {
   const db = context.getAppDB()
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 2f7f7fd44cc..f0e6c695638 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -296,7 +296,7 @@ export async function squashLinks<T = Row[] | Row>(
                 return false
               }
               if (
-                [FieldType.LINK, FieldType.FORMULA].includes(tableColumn.type)
+                [FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(tableColumn.type)
               ) {
                 return false
               }
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 831528f84de..5f617916839 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -56,6 +56,7 @@ interface AuthTokenResponse {
 const isTypeAllowed: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
   [FieldType.FORMULA]: true,
+  [FieldType.AI]: true,
   [FieldType.NUMBER]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.DATETIME]: true,
@@ -490,7 +491,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
         }
         if (
           !sheet.headerValues.includes(key) &&
-          column.type !== FieldType.FORMULA
+          column.type !== FieldType.FORMULA &&
+          column.type !== FieldType.AI
         ) {
           updatedHeaderValues.push(key)
         }
diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts
index c0beed0db80..c372b65bd04 100644
--- a/packages/server/src/sdk/app/tables/internal/index.ts
+++ b/packages/server/src/sdk/app/tables/internal/index.ts
@@ -15,7 +15,7 @@ import {
 import { EventType, updateLinks } from "../../../../db/linkedRows"
 import { cloneDeep } from "lodash/fp"
 import isEqual from "lodash/isEqual"
-import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
+import { runAIColumnChecks, runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
 import { context } from "@budibase/backend-core"
 import { findDuplicateInternalColumns } from "@budibase/shared-core"
 import { getTable } from "../getters"
@@ -133,6 +133,7 @@ export async function save(
   }
   // has to run after, make sure it has _id
   await runStaticFormulaChecks(table, { oldTable, deletion: false })
+  await runAIColumnChecks(table, { oldTable, deletion: false })
   return { table }
 }
 
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index bd716445375..7533e2b22ac 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -19,6 +19,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
   [FieldType.DATETIME]: SQLiteType.TEXT,
   [FieldType.FORMULA]: SQLiteType.TEXT,
+  [FieldType.AI]: SQLiteType.TEXT,
   [FieldType.LONGFORM]: SQLiteType.TEXT,
   [FieldType.NUMBER]: SQLiteType.REAL,
   [FieldType.STRING]: SQLiteType.TEXT,
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index 269158e61ee..f2f6f764d13 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -172,7 +172,7 @@ export async function enrichSchema(
 
     for (const relTableFieldName of Object.keys(relTable.schema)) {
       const relTableField = relTable.schema[relTableFieldName]
-      if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
+      if ([FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(relTableField.type)) {
         continue
       }
 
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 2f227dd6466..c4b8130966d 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -189,6 +189,10 @@ export async function inputProcessing(
     if (field.type === FieldType.FORMULA) {
       delete clonedRow[key]
     }
+    // remove any AI values, they are to be generated
+    if (field.type === FieldType.AI) {
+      delete clonedRow[key]
+    }
     // otherwise coerce what is there to correct types
     else {
       clonedRow[key] = coerce(value, field.type)
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 0fa6f628071..30942edc93f 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -11,6 +11,7 @@ import {
 } from "@budibase/types"
 import tracer from "dd-trace"
 import { context } from "@budibase/backend-core"
+import * as pro from "@budibase/pro"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -91,6 +92,56 @@ export async function processFormulas<T extends Row | Row[]>(
   })
 }
 
+/**
+ * Looks through the rows provided and finds AI columns - which it then processes.
+ */
+export async function processAIColumns<T extends Row | Row[]>(
+  table: Table,
+  inputRows: T,
+  { contextRows }: FormulaOpts
+): Promise<T> {
+  return tracer.trace("processAIColumns", {}, async span => {
+    const numRows = Array.isArray(inputRows) ? inputRows.length : 1
+    span?.addTags({ table_id: table._id })
+    const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+    if (rows) {
+      // Ensure we have snippet context
+      await context.ensureSnippetContext()
+
+      for (let [column, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldType.AI) {
+          continue
+        }
+
+        // const llm = pro.ai.LargeLanguageModel()
+        // if (
+        //   schema.formula == null ||
+        //   (dynamic && isStatic) ||
+        //   (!dynamic && !isStatic)
+        // ) {
+        //   continue
+        // }
+        // iterate through rows and process formula
+        for (let i = 0; i < rows.length; i++) {
+          let row = rows[i]
+          // let context = contextRows ? contextRows[i] : row
+          // let formula = schema.prompt
+          rows[i] = {
+            ...row,
+            [column]: tracer.trace("processAIColumn", {}, span => {
+              span?.addTags({ table_id: table._id, column })
+              // return processStringSync(formula, context)
+              // TODO: Add the AI stuff in to this
+              return "YEET AI"
+            }),
+          }
+        }
+      }
+    }
+    return Array.isArray(inputRows) ? rows : rows[0]
+  })
+}
+
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 45e9a7c6d05..6400b5852bf 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -87,6 +87,8 @@ export const getValidOperatorsForType = (
     ops = numOps
   } else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
     ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  } else if (type === FieldType.AI) {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
   } else if (
     type === FieldType.BB_REFERENCE_SINGLE ||
     schema.isDeprecatedSingleUserColumn(fieldType)
diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts
index 8a8069ce4de..4067e251d12 100644
--- a/packages/shared-core/src/table.ts
+++ b/packages/shared-core/src/table.ts
@@ -8,6 +8,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.NUMBER]: true,
   [FieldType.DATETIME]: true,
   [FieldType.FORMULA]: true,
+  [FieldType.AI]: true,
   [FieldType.AUTO]: true,
   [FieldType.INTERNAL]: true,
   [FieldType.BARCODEQR]: true,
@@ -38,6 +39,7 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
   [FieldType.JSON]: true,
 
   [FieldType.FORMULA]: false,
+  [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
@@ -61,6 +63,7 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.BIGINT]: false,
   [FieldType.BOOLEAN]: false,
   [FieldType.FORMULA]: false,
+  [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index 42440b29882..cb4b1d560d0 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -76,6 +76,10 @@ export enum FieldType {
    * that is part of the initial formula definition, the formula will be live evaluated in the browser.
    */
   AUTO = "auto",
+  /**
+   * A complex type, called an AI column within Budibase. This type has a... TODO: fill out
+   */
+  AI = "ai",
   /**
    * a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
    * type, which will be represented as a JSON object in the row. This type depends on a schema being
diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts
index fffaddc5df7..6ba23eba258 100644
--- a/packages/types/src/documents/app/table/constants.ts
+++ b/packages/types/src/documents/app/table/constants.ts
@@ -30,6 +30,7 @@ export enum JsonFieldSubType {
 export enum FormulaType {
   STATIC = "static",
   DYNAMIC = "dynamic",
+  AI = "ai"
 }
 
 export enum BBReferenceFieldSubType {
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 6078f73d1df..b0c78f9dc5d 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -116,6 +116,17 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   formulaType?: FormulaType
 }
 
+export interface AIFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  formula: string
+  // TODO: needs better types
+  operation: string
+  columns?: string[]
+  column?: string
+  prompt?: string
+  language?: string
+}
+
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE
@@ -190,6 +201,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.LINK
     | FieldType.AUTO
     | FieldType.FORMULA
+    | FieldType.AI
     | FieldType.NUMBER
     | FieldType.LONGFORM
     | FieldType.BB_REFERENCE
@@ -207,6 +219,7 @@ export type FieldSchema =
   | RelationshipFieldMetadata
   | AutoColumnFieldMetadata
   | FormulaFieldMetadata
+  | AIFieldMetadata
   | NumberFieldMetadata
   | LongFormFieldMetadata
   | StringFieldMetadata

From 0b9b9ab440cee0cd0e5413d28ea648cc1c7540cd Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 1 Oct 2024 14:56:16 +0100
Subject: [PATCH 183/455] AI functionaly working E2E

---
 .../common/AIFieldConfiguration.svelte        |  4 +-
 .../src/utilities/rowProcessor/utils.ts       | 52 +++++++++++--------
 2 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte
index 66af1266bed..73e0562dbe5 100644
--- a/packages/builder/src/components/common/AIFieldConfiguration.svelte
+++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte
@@ -47,11 +47,11 @@
       prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     CLEAN_DATA: {
-      columns: OperationFieldTypes.MULTI_COLUMN,
+      column: OperationFieldTypes.COLUMN,
       prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     TRANSLATE: {
-      columns: OperationFieldTypes.MULTI_COLUMN,
+      column: OperationFieldTypes.COLUMN,
       language: OperationFieldTypes.BINDABLE_TEXT,
       prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 30942edc93f..6055a2511cb 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -102,8 +102,9 @@ export async function processAIColumns<T extends Row | Row[]>(
 ): Promise<T> {
   return tracer.trace("processAIColumns", {}, async span => {
     const numRows = Array.isArray(inputRows) ? inputRows.length : 1
-    span?.addTags({ table_id: table._id })
+    span?.addTags({ table_id: table._id, numRows })
     const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+    const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini")
     if (rows) {
       // Ensure we have snippet context
       await context.ensureSnippetContext()
@@ -113,29 +114,34 @@ export async function processAIColumns<T extends Row | Row[]>(
           continue
         }
 
-        // const llm = pro.ai.LargeLanguageModel()
-        // if (
-        //   schema.formula == null ||
-        //   (dynamic && isStatic) ||
-        //   (!dynamic && !isStatic)
-        // ) {
-        //   continue
-        // }
-        // iterate through rows and process formula
-        for (let i = 0; i < rows.length; i++) {
-          let row = rows[i]
-          // let context = contextRows ? contextRows[i] : row
-          // let formula = schema.prompt
-          rows[i] = {
-            ...row,
-            [column]: tracer.trace("processAIColumn", {}, span => {
-              span?.addTags({ table_id: table._id, column })
-              // return processStringSync(formula, context)
-              // TODO: Add the AI stuff in to this
-              return "YEET AI"
-            }),
+        const rowUpdates = rows.map((row, i) => {
+          const contextRow = contextRows ? contextRows[i] : row
+          // TODO: Map the prompts with string-templates
+          // grab the operation based on the schema
+          // then check the types in the fields, and decide whether to pass them through string templates
+          // TODO: cleaner way to map to the schema, move things into BB types and check against the AI schema
+          for (const key in schema) {
+            if (["prompt", "categories"].includes(key)) {
+              schema[key] = processStringSync(schema[key], contextRow)
+            }
           }
-        }
+
+          const prompt = llm.buildPromptFromAIOperation({ schema, row })
+
+          return tracer.trace("processAIColumn", {}, async span => {
+            span?.addTags({ table_id: table._id, column })
+            const llmResponse = await llm.run(prompt)
+            return {
+              ...row,
+              [column]: llmResponse
+            }
+          })
+        })
+
+        const processedRows = await Promise.all(rowUpdates)
+
+        // Promise.all is deterministic so can rely on the indexing here
+        processedRows.forEach((processedRow, index) => rows[index] = processedRow)
       }
     }
     return Array.isArray(inputRows) ? rows : rows[0]

From d2b59fedeb6afc5c31e7deccb1c3901d8712ca85 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 1 Oct 2024 15:27:33 +0100
Subject: [PATCH 184/455] Refactor node management to automatically position
 basic and admin at each edge

---
 .../backend/RoleEditor/Controls.svelte        |   2 +-
 .../backend/RoleEditor/RoleFlow.svelte        | 144 +++++------------
 .../backend/RoleEditor/constants.js           |   2 +
 .../components/backend/RoleEditor/layout.js   |  71 ---------
 .../components/backend/RoleEditor/utils.js    | 147 ++++++++++++++++++
 5 files changed, 191 insertions(+), 175 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/RoleEditor/layout.js
 create mode 100644 packages/builder/src/components/backend/RoleEditor/utils.js

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index ff41f89daab..919093bee56 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -2,7 +2,7 @@
   import { Button, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow } from "@xyflow/svelte"
   import { getContext } from "svelte"
-  import { autoLayout } from "./layout"
+  import { autoLayout } from "./utils"
   import { MaxAutoZoom, ZoomDuration } from "./constants"
 
   const { nodes, edges, createRole } = getContext("flow")
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 04ff540a823..ef3c7a4f546 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -5,109 +5,51 @@
     SvelteFlow,
     Background,
     BackgroundVariant,
-    Position,
     useSvelteFlow,
+    getNodesBounds,
   } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
   import RoleEdge from "./RoleEdge.svelte"
-  import { autoLayout } from "./layout"
+  import {
+    autoLayout,
+    getAdminPosition,
+    getBasicPosition,
+    rolesToLayout,
+  } from "./utils"
   import { setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
   import { GridResolution, MaxAutoZoom } from "./constants"
   import { roles } from "stores/builder"
   import { Roles } from "constants/backend"
   import { getSequentialName } from "helpers/duplicate"
+  import { derivedMemo } from "@budibase/frontend-core"
 
   const flow = useSvelteFlow()
-  const nodes = writable([])
   const edges = writable([])
+  const nodes = writable([])
   const dragging = writable(false)
-  const selectedNodes = derived(nodes, $nodes =>
-    $nodes.filter(x => x.selected).map(x => x.id)
-  )
-
-  // Ensure role changes are synced with nodes and edges
-  $: handleExternalRoleChanges($roles)
-
-  // Converts a role doc into a node structure
-  const roleToNode = role => {
-    const custom = !role._id.match(/[A-Z]+/)
-
-    return {
-      id: role._id,
-      sourcePosition: Position.Right,
-      targetPosition: Position.Left,
-      type: "role",
-      position: { x: 0, y: 0 },
-      data: {
-        ...role.uiMetadata,
-        custom,
-      },
-      deletable: custom,
-      draggable: custom,
-      connectable: custom,
-    }
-  }
 
-  // Converts a node structure back into a role doc
-  const nodeToRole = node => {
-    const role = $roles.find(x => x._id === node.id)
-    const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
-    // TODO save inherits array
-    return {
-      ...role,
-      inherits,
-      uiMetadata: {
-        displayName: node.data.displayName,
-        color: node.data.color,
-        description: node.data.description,
-      },
-    }
-  }
-
-  // Builds a layout from an array of roles
-  const rolesToLayout = roles => {
-    let nodes = []
-    let edges = []
+  // Derive the list of selected nodes
+  const selectedNodes = derived(nodes, $nodes => {
+    return $nodes.filter(node => node.selected).map(node => node.id)
+  })
 
-    // Remove some builtins
-    const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
-    roles = roles.filter(role => !ignoredRoles.includes(role._id))
-    for (let role of roles) {
-      // Add node for this role
-      nodes.push(roleToNode(role))
+  // Derive the bounds of all custom role nodes
+  const bounds = derivedMemo(nodes, $nodes => {
+    return getNodesBounds($nodes.filter(node => node.data.custom))
+  })
 
-      // Add edges for this role
-      let inherits = []
-      if (role.inherits) {
-        inherits = Array.isArray(role.inherits)
-          ? role.inherits
-          : [role.inherits]
-      }
-      for (let sourceRole of inherits) {
-        if (!roles.some(x => x._id === sourceRole)) {
-          continue
-        }
-        edges.push({
-          id: `${sourceRole}-${role._id}`,
-          source: sourceRole,
-          target: role._id,
-        })
-      }
-    }
-    return {
-      nodes,
-      edges,
-    }
-  }
+  $: handleExternalRoleChanges($roles)
+  $: updateBuiltins($bounds)
 
   // Updates nodes and edges based on external changes to roles
   const handleExternalRoleChanges = roles => {
     const currentNodes = $nodes
     const newLayout = autoLayout(rolesToLayout(roles))
+    edges.set(newLayout.edges)
 
-    // For nodes we want to persist some metadata
+    // For nodes we want to persist some metadata if possible
     nodes.set(
       newLayout.nodes.map(node => {
         const currentNode = currentNodes.find(x => x.id === node.id)
@@ -121,22 +63,18 @@
         }
       })
     )
-
-    // Edges can always be updated
-    edges.set(newLayout.edges)
   }
 
-  // Manually selects a node
-  const selectNode = roleId => {
-    nodes.update($nodes => {
-      return $nodes.map(node => ({
-        ...node,
-        selected: node.id === roleId,
-      }))
+  // Positions the basic and admin role at either edge of the flow
+  const updateBuiltins = bounds => {
+    flow.updateNode(Roles.BASIC, {
+      position: getBasicPosition(bounds),
+    })
+    flow.updateNode(Roles.ADMIN, {
+      position: getAdminPosition(bounds),
     })
   }
 
-  // Creates a new role
   const createRole = async () => {
     const roleId = Helpers.uuid()
     await roles.save({
@@ -151,30 +89,30 @@
       permissionId: "write",
     })
     await tick()
-    selectNode(roleId)
+
+    // Select the new node
+    nodes.update($nodes => {
+      return $nodes.map(node => ({
+        ...node,
+        selected: node.id === roleId,
+      }))
+    })
   }
 
-  // Updates a role with new metadata
   const updateRole = async (roleId, metadata) => {
-    // Don't update builtins
-    const node = $nodes.find(x => x.id === roleId)
+    const node = $nodes.find(node => node.id === roleId)
     if (!node) {
       return
     }
-
-    // Update metadata
+    // Update metadata immediately, before saving
     if (metadata) {
       flow.updateNodeData(roleId, metadata)
     }
-
-    // Actually save changes
-    const role = nodeToRole(node)
-    await roles.save(role)
+    await roles.save(nodeToRole(node))
   }
 
-  // Deletes a role
   const deleteRole = async roleId => {
-    const role = $roles.find(x => x._id === roleId)
+    const role = $roles.find(role => role._id === roleId)
     if (role) {
       roles.delete(role)
     }
@@ -189,7 +127,7 @@
     await updateRole(edge.target)
   }
 
-  // Saves a new connection
+  // Updates roles which have had a new connection
   const onConnect = async connection => {
     await updateRole(connection.target)
   }
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index d22a228a877..ba43e349b51 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -3,3 +3,5 @@ export const MaxAutoZoom = 1.2
 export const GridResolution = 20
 export const NodeHeight = GridResolution * 3
 export const NodeWidth = GridResolution * 12
+export const NodeHSpacing = GridResolution * 4
+export const NodeVSpacing = GridResolution * 2
diff --git a/packages/builder/src/components/backend/RoleEditor/layout.js b/packages/builder/src/components/backend/RoleEditor/layout.js
deleted file mode 100644
index 90522680458..00000000000
--- a/packages/builder/src/components/backend/RoleEditor/layout.js
+++ /dev/null
@@ -1,71 +0,0 @@
-import dagre from "@dagrejs/dagre"
-import { NodeWidth, NodeHeight, GridResolution } from "./constants"
-import { Position } from "@xyflow/svelte"
-import { Roles } from "constants/backend"
-import { Helpers } from "@budibase/bbui"
-
-// Updates positions of nodes and edges into a nice graph structure
-export const dagreLayout = ({ nodes, edges }) => {
-  const dagreGraph = new dagre.graphlib.Graph()
-  dagreGraph.setDefaultEdgeLabel(() => ({}))
-  dagreGraph.setGraph({
-    rankdir: "LR",
-    ranksep: GridResolution * 4,
-    nodesep: GridResolution * 2,
-  })
-  nodes.forEach(node => {
-    dagreGraph.setNode(node.id, { width: NodeWidth, height: NodeHeight })
-  })
-  edges.forEach(edge => {
-    dagreGraph.setEdge(edge.source, edge.target)
-  })
-
-  // Add ephemeral edges for basic and admin so that we can position them properly
-
-  for (let node of nodes) {
-    if (
-      !edges.some(x => x.target === node.id) &&
-      node.id !== Roles.BASIC &&
-      node.id !== Roles.ADMIN
-    ) {
-      dagreGraph.setEdge(Roles.BASIC, node.id)
-    }
-    if (
-      !edges.some(x => x.source === node.id) &&
-      node.id !== Roles.BASIC &&
-      node.id !== Roles.ADMIN
-    ) {
-      dagreGraph.setEdge(node.id, Roles.ADMIN)
-    }
-  }
-
-  dagre.layout(dagreGraph)
-  nodes.forEach(node => {
-    const pos = dagreGraph.node(node.id)
-    node.targetPosition = Position.Left
-    node.sourcePosition = Position.Right
-    node.position = {
-      x: Math.round((pos.x - NodeWidth / 2) / GridResolution) * GridResolution,
-      y: Math.round((pos.y - NodeHeight / 2) / GridResolution) * GridResolution,
-    }
-  })
-  return { nodes, edges }
-}
-
-// Adds additional edges as needed to the flow structure to ensure compatibility with BB role logic
-const sanitiseLayout = ({ nodes, edges }) => {
-  // Remove any inheritance of basic and admin since this is implied
-  edges = edges.filter(
-    edge => edge.source !== Roles.BASIC && edge.target !== Roles.ADMIN
-  )
-
-  return {
-    nodes,
-    edges,
-  }
-}
-
-// Automatically lays out the graph, sanitising and enriching the structure
-export const autoLayout = ({ nodes, edges }) => {
-  return dagreLayout(sanitiseLayout({ nodes, edges }))
-}
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
new file mode 100644
index 00000000000..edd4cbe3356
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -0,0 +1,147 @@
+import dagre from "@dagrejs/dagre"
+import {
+  NodeWidth,
+  NodeHeight,
+  GridResolution,
+  NodeHSpacing,
+  NodeVSpacing,
+} from "./constants"
+import { getNodesBounds, Position } from "@xyflow/svelte"
+import { Roles } from "constants/backend"
+
+// Gets the position of the basic role
+export const getBasicPosition = bounds => ({
+  x: bounds.x - NodeHSpacing - NodeWidth,
+  y: bounds.y + bounds.height / 2 - NodeHeight / 2,
+})
+
+// Gets the position of the admin role
+export const getAdminPosition = bounds => ({
+  x: bounds.x + bounds.width + NodeHSpacing,
+  y: bounds.y + bounds.height / 2 - NodeHeight / 2,
+})
+
+// Updates positions of nodes and edges into a nice graph structure
+export const dagreLayout = ({ nodes, edges }) => {
+  const dagreGraph = new dagre.graphlib.Graph()
+  dagreGraph.setDefaultEdgeLabel(() => ({}))
+  dagreGraph.setGraph({
+    rankdir: "LR",
+    ranksep: NodeHSpacing,
+    nodesep: NodeVSpacing,
+  })
+  nodes.forEach(node => {
+    dagreGraph.setNode(node.id, { width: NodeWidth, height: NodeHeight })
+  })
+  edges.forEach(edge => {
+    dagreGraph.setEdge(edge.source, edge.target)
+  })
+  dagre.layout(dagreGraph)
+  nodes.forEach(node => {
+    const pos = dagreGraph.node(node.id)
+    node.targetPosition = Position.Left
+    node.sourcePosition = Position.Right
+    node.position = {
+      x: Math.round((pos.x - NodeWidth / 2) / GridResolution) * GridResolution,
+      y: Math.round((pos.y - NodeHeight / 2) / GridResolution) * GridResolution,
+    }
+  })
+
+  // Reposition basic and admin to bound the custom nodes
+  const bounds = getNodesBounds(nodes.filter(node => node.data.custom))
+  nodes.find(x => x.id === Roles.BASIC).position = getBasicPosition(bounds)
+  nodes.find(x => x.id === Roles.ADMIN).position = getAdminPosition(bounds)
+
+  return { nodes, edges }
+}
+
+// Adds additional edges as needed to the flow structure to ensure compatibility with BB role logic
+const sanitiseLayout = ({ nodes, edges }) => {
+  const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
+  const edglessRoles = [...ignoredRoles, Roles.BASIC, Roles.ADMIN]
+  return {
+    nodes: nodes.filter(node => !ignoredRoles.includes(node.id)),
+    edges: edges.filter(edge => {
+      return (
+        !edglessRoles.includes(edge.source) &&
+        !edglessRoles.includes(edge.target)
+      )
+    }),
+  }
+}
+
+// Automatically lays out the graph, sanitising and enriching the structure
+export const autoLayout = ({ nodes, edges }) => {
+  return dagreLayout(sanitiseLayout({ nodes, edges }))
+}
+
+// Converts a role doc into a node structure
+export const roleToNode = role => {
+  const custom = !role._id.match(/[A-Z]+/)
+  return {
+    id: role._id,
+    sourcePosition: Position.Right,
+    targetPosition: Position.Left,
+    type: "role",
+    position: { x: 0, y: 0 },
+    data: {
+      ...role.uiMetadata,
+      custom,
+    },
+    measured: {
+      width: NodeWidth,
+      height: NodeHeight,
+    },
+    deletable: custom,
+    draggable: custom,
+    connectable: custom,
+  }
+}
+
+// Converts a node structure back into a role doc
+export const nodeToRole = node => {
+  const role = $roles.find(x => x._id === node.id)
+  const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
+  // TODO save inherits array
+  return {
+    ...role,
+    inherits,
+    uiMetadata: {
+      displayName: node.data.displayName,
+      color: node.data.color,
+      description: node.data.description,
+    },
+  }
+}
+
+// Builds a layout from an array of roles
+export const rolesToLayout = roles => {
+  let nodes = []
+  let edges = []
+
+  // Remove some builtins
+  for (let role of roles) {
+    // Add node for this role
+    nodes.push(roleToNode(role))
+
+    // Add edges for this role
+    let inherits = []
+    if (role.inherits) {
+      inherits = Array.isArray(role.inherits) ? role.inherits : [role.inherits]
+    }
+    for (let sourceRole of inherits) {
+      if (!roles.some(x => x._id === sourceRole)) {
+        continue
+      }
+      edges.push({
+        id: `${sourceRole}-${role._id}`,
+        source: sourceRole,
+        target: role._id,
+      })
+    }
+  }
+  return {
+    nodes,
+    edges,
+  }
+}

From 3f520f05a0ced8cf214758537fbe633a28477097 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 1 Oct 2024 16:16:31 +0100
Subject: [PATCH 185/455] Add bounding brackets to role editor

---
 .../backend/RoleEditor/RoleFlow.svelte        |  1 +
 .../backend/RoleEditor/RoleNode.svelte        | 42 ++++++++++++++++++-
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index ef3c7a4f546..7a9280d679d 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -137,6 +137,7 @@
     edges,
     dragging,
     selectedNodes,
+    bounds,
     createRole,
     updateRole,
     deleteRole,
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index bae7c797553..57ef54ee5ce 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -9,7 +9,7 @@
     FieldLabel,
   } from "@budibase/bbui"
   import { Roles } from "constants/backend"
-  import { NodeWidth, NodeHeight } from "./constants"
+  import { NodeWidth, NodeHeight, NodeVSpacing } from "./constants"
   import { getContext } from "svelte"
   import { roles } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -19,7 +19,7 @@
   export let selected
   export let isConnectable
 
-  const { dragging, updateRole, deleteRole } = getContext("flow")
+  const { dragging, updateRole, deleteRole, bounds } = getContext("flow")
 
   let anchor
   let modal
@@ -103,6 +103,14 @@
   {/if}
 </div>
 
+{#if id === Roles.BASIC || id === Roles.ADMIN}
+  <div
+    class="bounds"
+    class:flip={id === Roles.ADMIN}
+    style="--height:{$bounds.height}px; --spacing:{NodeVSpacing}px;"
+  />
+{/if}
+
 <ConfirmDialog
   bind:this={deleteModal}
   title={`Delete ${data.displayName}`}
@@ -227,4 +235,34 @@
     opacity: 0;
     pointer-events: none;
   }
+
+  /* Bounds brackets */
+  .bounds {
+    height: calc(var(--height) + 2 * var(--spacing));
+    position: absolute;
+    width: calc(var(--spacing) / 1.5);
+    border: 2px dashed var(--edge-color);
+    border-right: none;
+    top: 50%;
+    left: calc(100% + var(--spacing));
+    transform: translateY(-50%);
+    border-top-left-radius: 8px;
+    border-bottom-left-radius: 8px;
+  }
+  .bounds.flip {
+    left: calc(-1 * var(--spacing));
+    transform: translateY(-50%) translateX(-100%) scale(-1, 1);
+  }
+  .bounds::after {
+    content: "";
+    display: block;
+    position: absolute;
+    top: 50%;
+    left: calc(-1 * var(--spacing));
+    height: 0;
+    border-top: 2px dashed var(--edge-color);
+    width: var(--spacing);
+    box-sizing: border-box;
+    transform: translateY(-50%);
+  }
 </style>

From 2aaef12a1ae174a23eff4b5e92d272089d7664b0 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 1 Oct 2024 16:36:37 +0100
Subject: [PATCH 186/455] better typing - discriminated unions

---
 .../DataTable/modals/CreateEditColumn.svelte  |   2 +-
 .../common/AIFieldConfiguration.svelte        |  15 +--
 packages/pro                                  |   2 +-
 .../src/api/controllers/table/bulkFormula.ts  |   4 +-
 packages/types/src/documents/app/ai.ts        | 105 ++++++++++++++++++
 packages/types/src/documents/app/index.ts     |   1 +
 .../types/src/documents/app/table/schema.ts   |   5 +-
 7 files changed, 118 insertions(+), 16 deletions(-)
 create mode 100644 packages/types/src/documents/app/ai.ts

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index da68b4182b0..30472c49394 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -417,13 +417,13 @@
         FIELDS.BOOLEAN,
         FIELDS.DATETIME,
         FIELDS.LINK,
+        FIELDS.AI,
         FIELDS.LONGFORM,
         FIELDS.USER,
         FIELDS.USERS,
         FIELDS.ATTACHMENT_SINGLE,
         FIELDS.ATTACHMENTS,
         FIELDS.FORMULA,
-        FIELDS.AI,
         FIELDS.JSON,
         FIELDS.BARCODEQR,
         FIELDS.SIGNATURE_SINGLE,
diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte
index 73e0562dbe5..84a9deca956 100644
--- a/packages/builder/src/components/common/AIFieldConfiguration.svelte
+++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Input, Multiselect, Select, TextArea } from "@budibase/bbui"
+  import { Input, Helpers, Multiselect, Select, TextArea } from "@budibase/bbui"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
 
@@ -44,32 +44,26 @@
   const OperationFields = {
     SUMMARISE_TEXT: {
       columns: OperationFieldTypes.MULTI_COLUMN,
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     CLEAN_DATA: {
       column: OperationFieldTypes.COLUMN,
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     TRANSLATE: {
       column: OperationFieldTypes.COLUMN,
       language: OperationFieldTypes.BINDABLE_TEXT,
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     CATEGORISE_TEXT: {
       columns: OperationFieldTypes.MULTI_COLUMN,
       categories: OperationFieldTypes.BINDABLE_TEXT,
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     SENTIMENT_ANALYSIS: {
       column: OperationFieldTypes.COLUMN,
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     PROMPT: {
       prompt: OperationFieldTypes.BINDABLE_TEXT,
     },
     SEARCH_WEB: {
       columns: OperationFieldTypes.MULTI_COLUMN,
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
     }
   }
 
@@ -90,6 +84,7 @@
 </script>
 
 <Select
+  label={"Operation"}
   options={AIFieldConfigOptions}
   bind:value={aiField.operation}
 />
@@ -97,7 +92,7 @@
   {#each Object.keys(OperationField) as key}
     {#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT}
       <ModalBindableInput
-        label={key}
+        label={Helpers.capitalise(key)}
         panel={ServerBindingPanel}
         title="Prompt"
         on:change={e => aiField[key] = e.detail}
@@ -109,13 +104,13 @@
     {:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN}
       <Multiselect
         bind:value={aiField[key]}
-        label={key}
+        label={Helpers.capitalise(key)}
         options={schemaWithoutRelations}
       />
     {:else if OperationField[key] === OperationFieldTypes.COLUMN}
       <Select
         bind:value={aiField[key]}
-        label={key}
+        label={Helpers.capitalise(key)}
         options={schemaWithoutRelations}
       />
     {/if}
diff --git a/packages/pro b/packages/pro
index dcc9e50b806..d194be2339a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit dcc9e50b8064a2097d408771462ad80f48de7ff6
+Subproject commit d194be2339a1ee2e5b2ed1fe59d103249f2d8d2c
diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts
index 6e47f30233f..b2c9f6a27b1 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.ts
+++ b/packages/server/src/api/controllers/table/bulkFormula.ts
@@ -198,7 +198,7 @@ export async function runAIColumnChecks(
   table: Table,
   { oldTable }: { oldTable?: Table }
 ) {
-  // look to see if any formula values have changed
+  // look to see if any AI column values have changed
   const shouldUpdate = Object.values(table.schema).find(
     column =>
       column.type === FieldType.AI &&
@@ -206,7 +206,7 @@ export async function runAIColumnChecks(
         !oldTable.schema[column.name] ||
         !isEqual(oldTable.schema[column.name], column))
   )
-  // if a static formula column has updated, then need to run the update
+  // if an AI column has updated, then need to run the update
   if (shouldUpdate != null) {
     await updateAllAIColumnsInTable(table)
   }
diff --git a/packages/types/src/documents/app/ai.ts b/packages/types/src/documents/app/ai.ts
new file mode 100644
index 00000000000..6d3f164fa40
--- /dev/null
+++ b/packages/types/src/documents/app/ai.ts
@@ -0,0 +1,105 @@
+
+export enum AIOperationEnum {
+  SUMMARISE_TEXT = "SUMMARISE_TEXT",
+  CLEAN_DATA = "CLEAN_DATA",
+  TRANSLATE = "TRANSLATE",
+  CATEGORISE_TEXT = "CATEGORISE_TEXT",
+  SENTIMENT_ANALYSIS = "SENTIMENT_ANALYSIS",
+  PROMPT = "PROMPT",
+  SEARCH_WEB = "SEARCH_WEB",
+}
+
+
+enum OperationFieldTypeEnum {
+  MULTI_COLUMN = "columns",
+  COLUMN = "column",
+  BINDABLE_TEXT = "prompt",
+}
+
+
+type OperationFieldsType = {
+  [AIOperationEnum.SUMMARISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+  }
+  [AIOperationEnum.CLEAN_DATA]: {
+    column: OperationFieldTypeEnum.COLUMN
+  }
+  [AIOperationEnum.TRANSLATE]: {
+    column: OperationFieldTypeEnum.COLUMN
+    language: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.CATEGORISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+    categories: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: {
+    column: OperationFieldTypeEnum.COLUMN
+  }
+  [AIOperationEnum.PROMPT]: {
+    prompt: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.SEARCH_WEB]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+  }
+}
+
+// Define the AI operations with just their labels (no need for separate 'value' property)
+const AIOperations: { [key in AIOperationEnum]: string } = {
+  [AIOperationEnum.SUMMARISE_TEXT]: "Summarise Text",
+  [AIOperationEnum.CLEAN_DATA]: "Clean Data",
+  [AIOperationEnum.TRANSLATE]: "Translate",
+  [AIOperationEnum.CATEGORISE_TEXT]: "Categorise Text",
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: "Sentiment Analysis",
+  [AIOperationEnum.PROMPT]: "Prompt",
+  [AIOperationEnum.SEARCH_WEB]: "Search Web",
+}
+
+type BaseSchema = {
+  operation: AIOperationEnum
+}
+
+type SummariseTextSchema = BaseSchema & {
+  operation: AIOperationEnum.SUMMARISE_TEXT
+  columns: string[]
+}
+
+type CleanDataSchema = BaseSchema & {
+  operation: AIOperationEnum.CLEAN_DATA
+  column: string
+}
+
+type TranslateSchema = BaseSchema & {
+  operation: AIOperationEnum.TRANSLATE
+  column: string
+  language: string
+}
+
+type CategoriseTextSchema = BaseSchema & {
+  operation: AIOperationEnum.CATEGORISE_TEXT
+  columns: string[]
+  categories: string
+}
+
+type SentimentAnalysisSchema = BaseSchema & {
+  operation: AIOperationEnum.SENTIMENT_ANALYSIS
+  column: string
+}
+
+type PromptSchema = BaseSchema & {
+  operation: AIOperationEnum.PROMPT
+  prompt: string
+}
+
+type SearchWebSchema = BaseSchema & {
+  operation: AIOperationEnum.SEARCH_WEB
+  columns: string[]
+}
+
+export type AIColumnSchema =
+  | SummariseTextSchema
+  | CleanDataSchema
+  | TranslateSchema
+  | CategoriseTextSchema
+  | SentimentAnalysisSchema
+  | PromptSchema
+  | SearchWebSchema
\ No newline at end of file
diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts
index 2b13676ba13..0290e9ed2ef 100644
--- a/packages/types/src/documents/app/index.ts
+++ b/packages/types/src/documents/app/index.ts
@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./app"
 export * from "./automation"
 export * from "./datasource"
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index b0c78f9dc5d..de1395cf31a 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -9,6 +9,7 @@ import {
   JsonFieldSubType,
   RelationshipType,
 } from "./constants"
+import { AIOperationEnum } from "../ai"
 
 export interface UIFieldMetadata {
   order?: number
@@ -119,10 +120,10 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
 export interface AIFieldMetadata extends BaseFieldSchema {
   type: FieldType.AI
   formula: string
-  // TODO: needs better types
-  operation: string
+  operation: AIOperationEnum
   columns?: string[]
   column?: string
+  categories?: string
   prompt?: string
   language?: string
 }

From b3afbd57be0310d7dcfcbdeeeba0b957bd143c25 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 1 Oct 2024 16:55:20 +0100
Subject: [PATCH 187/455] longform text

---
 packages/frontend-core/src/components/grid/lib/renderers.js | 3 +--
 packages/pro                                                | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index 3d0c0c77970..8939103fe0a 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -30,8 +30,7 @@ const TypeComponentMap = {
   [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
-  // TODO: fix
-  [FieldType.AI]: FormulaCell,
+  [FieldType.AI]: LongFormCell,
   [FieldType.JSON]: JSONCell,
   [FieldType.BB_REFERENCE]: BBReferenceCell,
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
diff --git a/packages/pro b/packages/pro
index d194be2339a..0c8f8349326 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit d194be2339a1ee2e5b2ed1fe59d103249f2d8d2c
+Subproject commit 0c8f834932670ce1d3b3f639e6e8d0d0086e508d

From 61a0db0984ee4bd59094c214ebd55c43e763ce45 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 2 Oct 2024 09:51:13 +0100
Subject: [PATCH 188/455] Add explicit typing for view search filter config

---
 packages/server/src/sdk/app/rows/search.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 8de5818805c..dae24c6bc0b 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -3,6 +3,8 @@ import {
   LogicalOperator,
   Row,
   RowSearchParams,
+  SearchFilter,
+  SearchFilterGroup,
   SearchFilterKey,
   SearchFilters,
   SearchResponse,
@@ -91,11 +93,12 @@ export async function search(
 
       if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
         // Lucene does not accept conditional filters, so we need to keep the old logic
-        const query: SearchFilters = viewQuery
+        const query: SearchFilters = viewQuery || {}
+        const viewFilters = view.query as SearchFilter[]
 
         // Extract existing fields
         const existingFields =
-          view.query
+          viewFilters
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
@@ -112,7 +115,7 @@ export async function search(
       } else {
         options.query = {
           $and: {
-            conditions: [viewQuery, options.query],
+            conditions: [viewQuery as SearchFilterGroup, options.query],
           },
         }
       }

From f6a783a9f13a7c0cfc601ba23664e010f9b29a70 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 10:25:22 +0100
Subject: [PATCH 189/455] Update submodules

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index c7a9fecd651..7172689d695 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -767,7 +767,7 @@
               formula: e.detail,
             }
           }}
-          {bindings}
+          bindings={getBindings({ table })}
           allowJS
           context={rowGoldenSample}
         />

From a49e779d111f051f42a718debdea2e1a58c8d43f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 10:50:22 +0100
Subject: [PATCH 190/455] Add default value for multi-selects and improve
 default value validation

---
 packages/account-portal                       |  2 +-
 .../DataTable/modals/CreateEditColumn.svelte  | 39 +++++++----
 packages/pro                                  |  2 +-
 .../src/utilities/rowProcessor/index.ts       |  6 +-
 packages/shared-core/src/table.ts             |  2 +-
 yarn.lock                                     | 64 ++++++++++++++++++-
 6 files changed, 93 insertions(+), 22 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index 905773d7085..3e24f6293ff 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 905773d70854a43c6ef2461c7a49671bff56fedc
+Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 7172689d695..19cc4ec1c09 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -207,6 +207,11 @@
     },
     ...getUserBindings(),
   ]
+  $: sanitiseDefaultValue(
+    editableColumn.type,
+    editableColumn.constraints?.inclusion || [],
+    editableColumn.default
+  )
 
   const fieldDefinitions = Object.values(FIELDS).reduce(
     // Storing the fields by complex field id
@@ -301,15 +306,6 @@
       delete saveColumn.default
     }
 
-    // Delete default value for options fields if the option is no longer available
-    if (
-      saveColumn.type === FieldType.OPTIONS &&
-      saveColumn.default &&
-      !saveColumn.constraints.inclusion?.includes(saveColumn.default)
-    ) {
-      delete saveColumn.default
-    }
-
     // Ensure primary display columns are always required and don't have default values
     if (primaryDisplay) {
       saveColumn.constraints.presence = { allowEmpty: false }
@@ -318,7 +314,7 @@
 
     // Ensure the field is not required if we have a default value
     if (saveColumn.default) {
-      saveColumn.constraints.presence = { allowEmpty: true }
+      saveColumn.constraints.presence = false
     }
 
     try {
@@ -567,6 +563,20 @@
     return newError
   }
 
+  const sanitiseDefaultValue = (type, options, defaultValue) => {
+    if (!defaultValue?.length) {
+      return
+    }
+    // Delete default value for options fields if the option is no longer available
+    if (type === FieldType.OPTIONS && !options.includes(defaultValue)) {
+      delete editableColumn.default
+    }
+    // Filter array default values to only valid options
+    if (type === FieldType.ARRAY) {
+      editableColumn.default = defaultValue.filter(x => options.includes(x))
+    }
+  }
+
   onMount(() => {
     mounted = true
   })
@@ -774,9 +784,9 @@
       </div>
     </div>
   {:else if editableColumn.type === JSON_TYPE}
-    <Button primary text on:click={openJsonSchemaEditor}
-      >Open schema editor</Button
-    >
+    <Button primary text on:click={openJsonSchemaEditor}>
+      Open schema editor
+    </Button>
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select
@@ -821,7 +831,8 @@
         options={editableColumn.constraints?.inclusion || []}
         label="Default value"
         value={editableColumn.default}
-        on:change={e => (editableColumn.default = e.detail)}
+        on:change={e =>
+          (editableColumn.default = e.detail?.length ? e.detail : undefined)}
         placeholder="None"
       />
     {:else}
diff --git a/packages/pro b/packages/pro
index 922431260e9..aca9828117b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 922431260e90d558a1ca55398475412e75088057
+Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 7332f8b244b..02733759f91 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -134,8 +134,10 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed = await processString(schema.default, ctx)
-
+      let processed = schema.default
+      if (typeof processed === "string") {
+        processed = await processString(schema.default, ctx)
+      }
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {
diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts
index fe7a0ebebf3..57f68546043 100644
--- a/packages/shared-core/src/table.ts
+++ b/packages/shared-core/src/table.ts
@@ -54,6 +54,7 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.LONGFORM]: true,
   [FieldType.STRING]: true,
   [FieldType.OPTIONS]: true,
+  [FieldType.ARRAY]: true,
 
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
@@ -64,7 +65,6 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
-  [FieldType.ARRAY]: false,
   [FieldType.LINK]: false,
   [FieldType.BB_REFERENCE]: false,
   [FieldType.BB_REFERENCE_SINGLE]: false,
diff --git a/yarn.lock b/yarn.lock
index 0cddbf981fd..383f9c6e7b5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17758,11 +17758,21 @@ periscopic@^3.1.0:
     estree-walker "^3.0.0"
     is-reference "^3.0.0"
 
+pg-cloudflare@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
+  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
+
 pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
   integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
 
+pg-connection-string@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.7.0.tgz#f1d3489e427c62ece022dba98d5262efcb168b37"
+  integrity sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==
+
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
@@ -17773,11 +17783,21 @@ pg-pool@^3.6.0:
   resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e"
   integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==
 
+pg-pool@^3.7.0:
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.7.0.tgz#d4d3c7ad640f8c6a2245adc369bafde4ebb8cbec"
+  integrity sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g==
+
 pg-protocol@*, pg-protocol@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
   integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
 
+pg-protocol@^1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.7.0.tgz#ec037c87c20515372692edac8b63cf4405448a93"
+  integrity sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==
+
 pg-types@^2.1.0, pg-types@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
@@ -17802,6 +17822,19 @@ pg@8.10.0:
     pg-types "^2.1.0"
     pgpass "1.x"
 
+pg@^8.12.0:
+  version "8.13.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.13.0.tgz#e3d245342eb0158112553fcc1890a60720ae2a3d"
+  integrity sha512-34wkUTh3SxTClfoHB3pQ7bIMvw9dpFU1audQQeZG837fmHfHpr14n/AELVDoOYVDW2h5RDWU78tFjkD+erSBsw==
+  dependencies:
+    pg-connection-string "^2.7.0"
+    pg-pool "^3.7.0"
+    pg-protocol "^1.7.0"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+  optionalDependencies:
+    pg-cloudflare "^1.1.1"
+
 pgpass@1.x:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
@@ -20696,7 +20729,16 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
+"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20787,7 +20829,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20801,6 +20843,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22756,7 +22805,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22774,6 +22823,15 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"

From 27c65ccff29284733bbd0e696ebea41f1ee95b1f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 10:54:17 +0100
Subject: [PATCH 191/455] Cleanup

---
 packages/server/src/utilities/rowProcessor/index.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 02733759f91..e63750bff9a 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -134,10 +134,10 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      let processed = schema.default
-      if (typeof processed === "string") {
-        processed = await processString(schema.default, ctx)
-      }
+      const processed =
+        typeof schema.default === "string"
+          ? await processString(schema.default, ctx)
+          : schema.default
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {

From a4c67f1812a780e60cfc9451dd505aff1acc0cff Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 11:11:20 +0100
Subject: [PATCH 192/455] Show notification when running row actions in the
 data section

---
 .../[application]/data/table/[tableId]/[viewId]/index.svelte | 5 ++---
 .../app/[application]/data/table/[tableId]/index.svelte      | 3 ++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index b5b65b3f1d1..eb5e7eb95d4 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -3,6 +3,7 @@
   import { admin, themeStore } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
+  import { notifications } from "@budibase/bbui"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
   import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
@@ -26,14 +27,12 @@
   $: currentTheme = $themeStore?.theme
   $: darkMode = !currentTheme.includes("light")
 
-  $: currentTheme = $themeStore?.theme
-  $: darkMode = !currentTheme.includes("light")
-
   const makeRowActionButtons = actions => {
     return (actions || []).map(action => ({
       text: action.name,
       onClick: async row => {
         await rowActions.trigger(id, action.id, row._id)
+        notifications.success("Row action triggered successfully")
       },
     }))
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index b9b58cbfce8..24851c723d7 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Banner } from "@budibase/bbui"
+  import { Banner, notifications } from "@budibase/bbui"
   import {
     datasources,
     tables,
@@ -67,6 +67,7 @@
       text: action.name,
       onClick: async row => {
         await rowActions.trigger(id, action.id, row._id)
+        notifications.success("Row action triggered successfully")
       },
     }))
   }

From ce07aa98d24062aef6331a568384c3d19fbb64dd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 11:11:47 +0100
Subject: [PATCH 193/455] Make collapsed buttons in grids CTAs

---
 .../frontend-core/src/components/grid/layout/ButtonColumn.svelte | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
index 0e2cf7951e4..d82ed2b4e23 100644
--- a/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte
@@ -98,7 +98,6 @@
                   align="right"
                   offset={5}
                   size="S"
-                  quiet
                   animate={false}
                   on:mouseenter={() => ($hoveredRowId = row._id)}
                 />

From 45e708652e567ba31a3ea3e993e9a9fb923d6752 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 15:39:48 +0100
Subject: [PATCH 194/455]  Revert backend changes

---
 packages/server/src/utilities/rowProcessor/index.ts | 6 ++----
 packages/shared-core/src/table.ts                   | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index e63750bff9a..7332f8b244b 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -134,10 +134,8 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed =
-        typeof schema.default === "string"
-          ? await processString(schema.default, ctx)
-          : schema.default
+      const processed = await processString(schema.default, ctx)
+
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {
diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts
index 57f68546043..8a8069ce4de 100644
--- a/packages/shared-core/src/table.ts
+++ b/packages/shared-core/src/table.ts
@@ -53,9 +53,8 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.DATETIME]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.STRING]: true,
-  [FieldType.OPTIONS]: true,
-  [FieldType.ARRAY]: true,
 
+  [FieldType.OPTIONS]: false,
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
   [FieldType.BARCODEQR]: false,
@@ -65,6 +64,7 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
+  [FieldType.ARRAY]: false,
   [FieldType.LINK]: false,
   [FieldType.BB_REFERENCE]: false,
   [FieldType.BB_REFERENCE_SINGLE]: false,

From 8cf7d1a2b5e692db70c6409986e2409c950941c5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 15:41:20 +0100
Subject: [PATCH 195/455] Revert yarn.lock

---
 yarn.lock | 66 ++++---------------------------------------------------
 1 file changed, 4 insertions(+), 62 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 383f9c6e7b5..8b7fd5ccf72 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17758,21 +17758,11 @@ periscopic@^3.1.0:
     estree-walker "^3.0.0"
     is-reference "^3.0.0"
 
-pg-cloudflare@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
-  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
-
 pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
   integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
 
-pg-connection-string@^2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.7.0.tgz#f1d3489e427c62ece022dba98d5262efcb168b37"
-  integrity sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==
-
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
@@ -17783,21 +17773,11 @@ pg-pool@^3.6.0:
   resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e"
   integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==
 
-pg-pool@^3.7.0:
-  version "3.7.0"
-  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.7.0.tgz#d4d3c7ad640f8c6a2245adc369bafde4ebb8cbec"
-  integrity sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g==
-
 pg-protocol@*, pg-protocol@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
   integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
 
-pg-protocol@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.7.0.tgz#ec037c87c20515372692edac8b63cf4405448a93"
-  integrity sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==
-
 pg-types@^2.1.0, pg-types@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
@@ -17822,19 +17802,6 @@ pg@8.10.0:
     pg-types "^2.1.0"
     pgpass "1.x"
 
-pg@^8.12.0:
-  version "8.13.0"
-  resolved "https://registry.yarnpkg.com/pg/-/pg-8.13.0.tgz#e3d245342eb0158112553fcc1890a60720ae2a3d"
-  integrity sha512-34wkUTh3SxTClfoHB3pQ7bIMvw9dpFU1audQQeZG837fmHfHpr14n/AELVDoOYVDW2h5RDWU78tFjkD+erSBsw==
-  dependencies:
-    pg-connection-string "^2.7.0"
-    pg-pool "^3.7.0"
-    pg-protocol "^1.7.0"
-    pg-types "^2.1.0"
-    pgpass "1.x"
-  optionalDependencies:
-    pg-cloudflare "^1.1.1"
-
 pgpass@1.x:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
@@ -20729,16 +20696,7 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20829,7 +20787,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20843,13 +20801,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22805,7 +22756,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22823,15 +22774,6 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
@@ -23218,4 +23160,4 @@ zip-stream@^6.0.1:
   dependencies:
     archiver-utils "^5.0.0"
     compress-commons "^6.0.2"
-    readable-stream "^4.0.0"
+    readable-stream "^4.0.0"
\ No newline at end of file

From 1b109db61f5bc2ab510beed074bc91a216ccdca6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 2 Oct 2024 15:42:07 +0100
Subject: [PATCH 196/455] Revert yarn.lock

---
 yarn.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn.lock b/yarn.lock
index 8b7fd5ccf72..0cddbf981fd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -23160,4 +23160,4 @@ zip-stream@^6.0.1:
   dependencies:
     archiver-utils "^5.0.0"
     compress-commons "^6.0.2"
-    readable-stream "^4.0.0"
\ No newline at end of file
+    readable-stream "^4.0.0"

From 2c5fe77740942093bb784d6441b18b6f52d70de1 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 2 Oct 2024 16:11:52 +0100
Subject: [PATCH 197/455] don't run over all historical cells

---
 .../src/components/grid/cells/AICell.svelte   | 106 ++++++++++++++++++
 .../src/components/grid/lib/renderers.js      |   3 +-
 .../src/sdk/app/tables/internal/index.ts      |   3 +-
 3 files changed, 109 insertions(+), 3 deletions(-)
 create mode 100644 packages/frontend-core/src/components/grid/cells/AICell.svelte

diff --git a/packages/frontend-core/src/components/grid/cells/AICell.svelte b/packages/frontend-core/src/components/grid/cells/AICell.svelte
new file mode 100644
index 00000000000..4e11fb7ebd3
--- /dev/null
+++ b/packages/frontend-core/src/components/grid/cells/AICell.svelte
@@ -0,0 +1,106 @@
+<script>
+  import { onMount } from "svelte"
+  import { clickOutside } from "@budibase/bbui"
+  import GridPopover from "../overlays/GridPopover.svelte"
+
+  export let value
+  export let focused = false
+  export let api
+
+  let textarea
+  let isOpen = false
+  let anchor
+
+  $: {
+    if (!focused) {
+      isOpen = false
+    }
+  }
+
+  const onKeyDown = () => {
+    return isOpen
+  }
+
+  const open = async () => {
+    isOpen = true
+  }
+
+  const close = () => {
+    textarea?.blur()
+    isOpen = false
+  }
+
+  onMount(() => {
+    api = {
+      focus: () => open(),
+      blur: () => close(),
+      isActive: () => isOpen,
+      onKeyDown,
+    }
+  })
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<div
+  class="long-form-cell"
+  on:click={open}
+  bind:this={anchor}
+>
+  <div class="value">
+    {value || ""}
+  </div>
+</div>
+
+{#if isOpen}
+  <GridPopover {anchor} on:close={close}>
+    <textarea
+      disabled
+      bind:this={textarea}
+      value={value || ""}
+      on:wheel|stopPropagation
+      spellcheck="false"
+      use:clickOutside={close}
+    />
+  </GridPopover>
+{/if}
+
+<style>
+  .long-form-cell {
+    flex: 1 1 auto;
+    padding: var(--cell-padding);
+    align-self: stretch;
+    display: flex;
+    align-items: flex-start;
+    overflow: hidden;
+  }
+  .long-form-cell.editable:hover {
+    cursor: text;
+  }
+  .value {
+    display: -webkit-box;
+    -webkit-line-clamp: var(--content-lines);
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+    line-height: 20px;
+  }
+  textarea {
+    border: none;
+    width: 320px;
+    flex: 1 1 auto;
+    height: var(--max-cell-render-overflow);
+    padding: var(--cell-padding);
+    margin: 0;
+    background: var(--cell-background);
+    font-size: var(--cell-font-size);
+    font-family: var(--font-sans);
+    color: inherit;
+    z-index: 1;
+    resize: none;
+    line-height: 20px;
+    overflow: auto;
+  }
+  textarea:focus {
+    outline: none;
+  }
+</style>
diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index 8939103fe0a..f9e26d920a8 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -9,6 +9,7 @@ import TextCell from "../cells/TextCell.svelte"
 import LongFormCell from "../cells/LongFormCell.svelte"
 import BooleanCell from "../cells/BooleanCell.svelte"
 import FormulaCell from "../cells/FormulaCell.svelte"
+import AICell from "../cells/AICell.svelte"
 import JSONCell from "../cells/JSONCell.svelte"
 import AttachmentCell from "../cells/AttachmentCell.svelte"
 import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
@@ -30,7 +31,7 @@ const TypeComponentMap = {
   [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
-  [FieldType.AI]: LongFormCell,
+  [FieldType.AI]: AICell,
   [FieldType.JSON]: JSONCell,
   [FieldType.BB_REFERENCE]: BBReferenceCell,
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts
index 6e2a7df276b..80b86b3b76c 100644
--- a/packages/server/src/sdk/app/tables/internal/index.ts
+++ b/packages/server/src/sdk/app/tables/internal/index.ts
@@ -16,7 +16,6 @@ import { EventType, updateLinks } from "../../../../db/linkedRows"
 import { cloneDeep } from "lodash/fp"
 import isEqual from "lodash/isEqual"
 import {
-  runAIColumnChecks,
   runStaticFormulaChecks,
 } from "../../../../api/controllers/table/bulkFormula"
 import { context } from "@budibase/backend-core"
@@ -136,7 +135,7 @@ export async function save(
   }
   // has to run after, make sure it has _id
   await runStaticFormulaChecks(table, { oldTable, deletion: false })
-  await runAIColumnChecks(table, { oldTable, deletion: false })
+  // await runAIColumnChecks(table, { oldTable, deletion: false })
   return { table }
 }
 

From c1a86433c168d68becc98444a9f8426be1b7e804 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 2 Oct 2024 16:49:04 +0100
Subject: [PATCH 198/455] don't backfill old rows on AI column creation

---
 packages/frontend-core/src/constants.js       |  2 +-
 .../src/api/controllers/row/aiColumn.ts       | 43 -------------------
 .../src/api/controllers/table/bulkFormula.ts  | 19 --------
 .../src/sdk/app/tables/internal/index.ts      |  1 -
 4 files changed, 1 insertion(+), 64 deletions(-)
 delete mode 100644 packages/server/src/api/controllers/row/aiColumn.ts

diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index dfdcaf5c014..d70bb32fee9 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -163,7 +163,7 @@ export const TypeIconMap = {
   [FieldType.AI]: "MagicWand",
   [FieldType.JSON]: "Brackets",
   [FieldType.BIGINT]: "TagBold",
-  [FieldType.AUTO]: "MagicWand",
+  [FieldType.AUTO]: "Shapes",
   [FieldType.BB_REFERENCE]: {
     [BBReferenceFieldSubType.USER]: "UserGroup",
     [BBReferenceFieldSubType.USERS]: "UserGroup",
diff --git a/packages/server/src/api/controllers/row/aiColumn.ts b/packages/server/src/api/controllers/row/aiColumn.ts
deleted file mode 100644
index fc543c34870..00000000000
--- a/packages/server/src/api/controllers/row/aiColumn.ts
+++ /dev/null
@@ -1,43 +0,0 @@
-import { getRowParams } from "../../../db/utils"
-import {
-  outputProcessing,
-  processAIColumns,
-} from "../../../utilities/rowProcessor"
-import { context } from "@budibase/backend-core"
-import { Table, Row } from "@budibase/types"
-import isEqual from "lodash/isEqual"
-import { cloneDeep } from "lodash/fp"
-
-export async function updateAllAIColumnsInTable(table: Table) {
-  const db = context.getAppDB()
-  // start by getting the raw rows (which will be written back to DB after update)
-  let rows = (
-    await db.allDocs<Row>(
-      getRowParams(table._id, null, {
-        include_docs: true,
-      })
-    )
-  ).rows.map(row => row.doc!)
-  // now enrich the rows, note the clone so that we have the base state of the
-  // rows so that we don't write any of the enriched information back
-  let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
-    squash: false,
-  })
-  const updatedRows = []
-  for (let row of rows) {
-    // find the enriched row, if found process the formulas
-    const enrichedRow = enrichedRows.find(
-      (enriched: Row) => enriched._id === row._id
-    )
-    if (enrichedRow) {
-      let processed = await processAIColumns(table, cloneDeep(row), {
-        contextRows: [enrichedRow],
-      })
-      // values have changed, need to add to bulk docs to update
-      if (!isEqual(processed, row)) {
-        updatedRows.push(processed)
-      }
-    }
-  }
-  await db.bulkDocs(updatedRows)
-}
diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts
index fcaf8f1fcda..40555aa3f8d 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.ts
+++ b/packages/server/src/api/controllers/table/bulkFormula.ts
@@ -14,7 +14,6 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { isRelationshipColumn } from "../../../db/utils"
-import { updateAllAIColumnsInTable } from "../row/aiColumn"
 
 function isStaticFormula(
   column: FieldSchema
@@ -199,21 +198,3 @@ export async function runStaticFormulaChecks(
     await checkIfFormulaUpdated(table, { oldTable })
   }
 }
-
-export async function runAIColumnChecks(
-  table: Table,
-  { oldTable }: { oldTable?: Table }
-) {
-  // look to see if any AI column values have changed
-  const shouldUpdate = Object.values(table.schema).find(
-    column =>
-      column.type === FieldType.AI &&
-      (!oldTable ||
-        !oldTable.schema[column.name] ||
-        !isEqual(oldTable.schema[column.name], column))
-  )
-  // if an AI column has updated, then need to run the update
-  if (shouldUpdate != null) {
-    await updateAllAIColumnsInTable(table)
-  }
-}
diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts
index 80b86b3b76c..57b4cb0c80d 100644
--- a/packages/server/src/sdk/app/tables/internal/index.ts
+++ b/packages/server/src/sdk/app/tables/internal/index.ts
@@ -135,7 +135,6 @@ export async function save(
   }
   // has to run after, make sure it has _id
   await runStaticFormulaChecks(table, { oldTable, deletion: false })
-  // await runAIColumnChecks(table, { oldTable, deletion: false })
   return { table }
 }
 

From 786bfdb0e2584c9e03ad54befa7202e5bd8354b5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 2 Oct 2024 18:09:53 +0100
Subject: [PATCH 199/455] Merging v3-backport branch

---
 .../server/src/api/controllers/row/views.ts   |  38 +++--
 .../src/api/controllers/view/viewsV2.ts       |   4 +-
 packages/server/src/sdk/app/rows/search.ts    |  47 +++---
 .../server/src/sdk/app/rows/search/utils.ts   |   4 +-
 packages/shared-core/src/filters.ts           |  21 +--
 packages/shared-core/src/utils.ts             | 157 +++++++++---------
 packages/types/src/api/web/app/view.ts        |   5 +-
 packages/types/src/api/web/searchFilter.ts    |   5 +-
 packages/types/src/documents/app/view.ts      |   8 +-
 9 files changed, 155 insertions(+), 134 deletions(-)

diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts
index f804c650f91..b8d01424f21 100644
--- a/packages/server/src/api/controllers/row/views.ts
+++ b/packages/server/src/api/controllers/row/views.ts
@@ -3,6 +3,8 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
+  RequiredKeys,
+  RowSearchParams,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
@@ -20,30 +22,34 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
+  const viewFields = Object.entries(view.schema || {})
+    .filter(([_, value]) => value.visible)
+    .map(([key]) => key)
   const { body } = ctx.request
 
   await context.ensureSnippetContext(true)
 
-  const result = await sdk.rows.search(
-    {
-      viewId: view.id,
-      tableId: view.tableId,
-      query: body.query,
-      ...getSortOptions(body, view),
-      limit: body.limit,
-      bookmark: body.bookmark,
-      paginate: body.paginate,
-      countRows: body.countRows,
-    },
-    {
-      user: sdk.users.getUserContextBindings(ctx.user),
-    }
-  )
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
+    RequiredKeys<
+      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
+    > = {
+    tableId: view.tableId,
+    viewId: view.id,
+    query: body.query,
+    fields: viewFields,
+    ...getSortOptions(body, view),
+    limit: body.limit,
+    bookmark: body.bookmark,
+    paginate: body.paginate,
+    countRows: body.countRows,
+  }
 
+  const result = await sdk.rows.search(searchOptions, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
-
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
   if (request.sort) {
     return {
diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index 7f6f6385419..3df7172de2a 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -99,7 +99,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const schema = await parseSchema(view)
 
-  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
+  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version" | "queryUI"> = {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
@@ -132,7 +132,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const { tableId } = view
 
   const schema = await parseSchema(view)
-  const parsedView: RequiredKeys<ViewV2> = {
+  const parsedView: RequiredKeys<Omit<ViewV2, "queryUI">> = {
     id: view.id,
     name: view.name,
     version: view.version,
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index dae24c6bc0b..7e73a518897 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -1,12 +1,10 @@
 import {
   EmptyFilterOption,
+  LegacyFilter,
   LogicalOperator,
   Row,
   RowSearchParams,
-  SearchFilter,
-  SearchFilterGroup,
   SearchFilterKey,
-  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -63,7 +61,6 @@ export async function search(
     if (options.viewId) {
       source = await sdk.views.get(options.viewId)
       table = await sdk.views.getTable(source)
-      options = searchInputMapping(table, options)
     } else if (options.tableId) {
       source = await sdk.tables.getTable(options.tableId)
       table = source
@@ -76,7 +73,7 @@ export async function search(
     if (options.query) {
       const visibleFields = (
         options.fields || Object.keys(table.schema)
-      ).filter(field => table.schema[field].visible !== false)
+      ).filter(field => table.schema[field]?.visible !== false)
 
       const queryableFields = await getQueryableFields(table, visibleFields)
       options.query = removeInvalidFilters(options.query, queryableFields)
@@ -84,38 +81,52 @@ export async function search(
       options.query = {}
     }
 
+    // need to make sure filters in correct shape before checking for view
+    options = searchInputMapping(table, options)
+
     if (options.viewId) {
-      const view = await sdk.views.get(options.viewId)
+      // Delete extraneous search params that cannot be overridden
+      delete options.query.onEmptyFilter
+
+      const view = source as ViewV2
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQuery(view.query || [])
-
-      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
-        // Lucene does not accept conditional filters, so we need to keep the old logic
-        const query: SearchFilters = viewQuery || {}
-        const viewFilters = view.query as SearchFilter[]
+      let viewQuery = dataFilters.buildQueryLegacy(view.query || [])
+      delete viewQuery?.onEmptyFilter
+
+      const sqsEnabled = await features.flags.isEnabled("SQS")
+      const supportsLogicalOperators =
+        isExternalTableID(view.tableId) || sqsEnabled
+      if (!supportsLogicalOperators) {
+        // In the unlikely event that a Grouped Filter is in a non-SQS environment
+        // It needs to be ignored entirely
+        let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+          ? view.query
+          : []
 
         // Extract existing fields
         const existingFields =
-          viewFilters
+          queryFilters
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
+        viewQuery ??= {}
         // Carry over filters for unused fields
-        Object.keys(options.query || {}).forEach(key => {
+        Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              query[operator]![field] = options.query[operator]![field]
+              viewQuery![operator]![field] = options.query[operator]![field]
             }
           })
         })
-        options.query = query
+        options.query = viewQuery
       } else {
+        const conditions = viewQuery ? [viewQuery] : []
         options.query = {
           $and: {
-            conditions: [viewQuery as SearchFilterGroup, options.query],
+            conditions: [...conditions, options.query],
           },
         }
       }
@@ -146,8 +157,6 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    options = searchInputMapping(table, options)
-
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })
diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts
index 1dba420a286..90303a6ca7b 100644
--- a/packages/server/src/sdk/app/rows/search/utils.ts
+++ b/packages/server/src/sdk/app/rows/search/utils.ts
@@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
     }
     return dataFilters.recurseLogicalOperators(filters, checkFilters)
   }
-  options.query = checkFilters(options.query)
+  if (options.query) {
+    options.query = checkFilters(options.query)
+  }
   return options
 }
 
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 23d7c7590cd..fce355f7b91 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -3,7 +3,7 @@ import {
   BBReferenceFieldSubType,
   FieldType,
   FormulaType,
-  SearchFilter,
+  LegacyFilter,
   SearchFilters,
   SearchQueryFields,
   ArrayOperator,
@@ -127,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters?.[logical]) {
+    if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )
@@ -163,9 +163,6 @@ export function recurseSearchFilters(
  * https://github.com/Budibase/budibase/issues/10118
  */
 export const cleanupQuery = (query: SearchFilters) => {
-  if (!query) {
-    return query
-  }
   for (let filterField of NoEmptyFilterStrings) {
     if (!query[filterField]) {
       continue
@@ -311,7 +308,7 @@ export class ColumnSplitter {
  * @param filter the builder filter structure
  */
 
-const buildCondition = (expression: SearchFilter) => {
+const buildCondition = (expression: LegacyFilter) => {
   // Filter body
   let query: SearchFilters = {
     string: {},
@@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => {
 }
 
 export const buildQueryLegacy = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: LegacyFilter[] | SearchFilters
 ): SearchFilters | undefined => {
+  // this is of type SearchFilters or is undefined
+  if (!Array.isArray(filter)) {
+    return filter
+  }
+
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -572,7 +574,7 @@ export const buildQueryLegacy = (
  */
 
 export const buildQuery = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: SearchFilterGroup | LegacyFilter[]
 ): SearchFilters | undefined => {
   const parsedFilter: SearchFilterGroup | undefined =
     processSearchFilters(filter)
@@ -594,7 +596,7 @@ export const buildQuery = (
   const globalOperator: LogicalOperator =
     operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
 
-  const coreRequest: SearchFilters = {
+  return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
       conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@@ -608,7 +610,6 @@ export const buildQuery = (
       }),
     },
   }
-  return coreRequest
 }
 
 // The frontend can send single values for array fields sometimes, so to handle
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 5b4d4399847..14b3c844253 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -1,5 +1,5 @@
 import {
-  SearchFilter,
+  LegacyFilter,
   SearchFilterGroup,
   FilterGroupLogicalOperator,
   SearchFilters,
@@ -9,6 +9,10 @@ import {
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
 
+// an array of keys from filter type to properties that are in the type
+// this can then be converted using .fromEntries to an object
+type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+
 export function unreachable(
   value: never,
   message = `No such case in exhaustive switch: ${value}`
@@ -87,16 +91,42 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
   return result
 }
 
+export function isSupportedUserSearch(query: SearchFilters) {
+  const allowed = [
+    { op: BasicOperator.STRING, key: "email" },
+    { op: BasicOperator.EQUAL, key: "_id" },
+    { op: ArrayOperator.ONE_OF, key: "_id" },
+  ]
+  for (let [key, operation] of Object.entries(query)) {
+    if (typeof operation !== "object") {
+      return false
+    }
+    const fields = Object.keys(operation || {})
+    // this filter doesn't contain options - ignore
+    if (fields.length === 0) {
+      continue
+    }
+    const allowedOperation = allowed.find(
+      allow =>
+        allow.op === key && fields.length === 1 && fields[0] === allow.key
+    )
+    if (!allowedOperation) {
+      return false
+    }
+  }
+  return true
+}
+
 /**
  * Processes the filter config. Filters are migrated from
  * SearchFilter[] to SearchFilterGroup
  *
  * If filters is not an array, the migration is skipped
  *
- * @param {SearchFilter[] | SearchFilterGroup} filters
+ * @param {LegacyFilter[] | SearchFilterGroup} filters
  */
 export const processSearchFilters = (
-  filters: SearchFilter[] | SearchFilterGroup | undefined
+  filters: LegacyFilter[] | SearchFilterGroup | undefined
 ): SearchFilterGroup | undefined => {
   if (!filters) {
     return
@@ -108,7 +138,7 @@ export const processSearchFilters = (
     groups: [],
   }
 
-  const filterWhitelistKeys = [
+  const filterAllowedKeys = [
     "field",
     "operator",
     "value",
@@ -125,90 +155,59 @@ export const processSearchFilters = (
       logicalOperator: FilterGroupLogicalOperator.ALL,
     }
 
-    const migratedSetting: SearchFilterGroup = filters.reduce(
-      (acc: SearchFilterGroup, filter: SearchFilter) => {
-        // Sort the properties for easier debugging
-        const filterEntries = Object.entries(filter)
-          .sort((a, b) => {
-            return a[0].localeCompare(b[0])
-          })
-          .filter(x => x[1] ?? false)
-
-        if (filterEntries.length == 1) {
-          const [key, value] = filterEntries[0]
-          // Global
-          if (key === "onEmptyFilter") {
-            // unset otherwise
-            acc.onEmptyFilter = value
-          } else if (key === "operator" && value === "allOr") {
-            // Group 1 logical operator
-            baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
-          }
-
-          return acc
+    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+      // Sort the properties for easier debugging
+      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
+        .sort((a, b) => {
+          return a.localeCompare(b)
+        })
+        .filter(key => key in filter)
+
+      if (filterPropertyKeys.length == 1) {
+        const key = filterPropertyKeys[0],
+          value = filter[key]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
         }
 
-        const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
-          (acc: [string, any][], entry: [string, any]) => {
-            const [key, value] = entry
-
-            if (filterWhitelistKeys.includes(key)) {
-              if (key === "field") {
-                acc.push([key, removeKeyNumbering(value)])
-              } else {
-                acc.push([key, value])
-              }
+        return acc
+      }
+
+      const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
+        (acc: AllowedFilters, key) => {
+          const value = filter[key]
+          if (filterAllowedKeys.includes(key)) {
+            if (key === "field") {
+              acc.push([key, removeKeyNumbering(value)])
+            } else {
+              acc.push([key, value])
             }
-            return acc
-          },
-          []
-        )
-
-        const migratedFilter: SearchFilter = Object.fromEntries(
-          whiteListedFilterSettings
-        ) as SearchFilter
+          }
+          return acc
+        },
+        []
+      )
 
-        baseGroup.filters!.push(migratedFilter)
+      const migratedFilter: LegacyFilter = Object.fromEntries(
+        allowedFilterSettings
+      ) as LegacyFilter
 
-        if (!acc.groups || !acc.groups.length) {
-          // init the base group
-          acc.groups = [baseGroup]
-        }
+      baseGroup.filters!.push(migratedFilter)
 
-        return acc
-      },
-      defaultCfg
-    )
+      if (!acc.groups || !acc.groups.length) {
+        // init the base group
+        acc.groups = [baseGroup]
+      }
 
-    return migratedSetting
+      return acc
+    }, defaultCfg)
   } else if (!filters?.groups) {
     return
   }
   return filters
 }
-
-export function isSupportedUserSearch(query: SearchFilters) {
-  const allowed = [
-    { op: BasicOperator.STRING, key: "email" },
-    { op: BasicOperator.EQUAL, key: "_id" },
-    { op: ArrayOperator.ONE_OF, key: "_id" },
-  ]
-  for (let [key, operation] of Object.entries(query)) {
-    if (typeof operation !== "object") {
-      return false
-    }
-    const fields = Object.keys(operation || {})
-    // this filter doesn't contain options - ignore
-    if (fields.length === 0) {
-      continue
-    }
-    const allowedOperation = allowed.find(
-      allow =>
-        allow.op === key && fields.length === 1 && fields[0] === allow.key
-    )
-    if (!allowedOperation) {
-      return false
-    }
-  }
-  return true
-}
diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts
index a6be5e29868..a99f2938aba 100644
--- a/packages/types/src/api/web/app/view.ts
+++ b/packages/types/src/api/web/app/view.ts
@@ -9,6 +9,7 @@ export interface ViewResponseEnriched {
   data: ViewV2Enriched
 }
 
-export interface CreateViewRequest extends Omit<ViewV2, "version" | "id"> {}
+export interface CreateViewRequest
+  extends Omit<ViewV2, "version" | "id" | "queryUI"> {}
 
-export interface UpdateViewRequest extends ViewV2 {}
+export interface UpdateViewRequest extends Omit<ViewV2, "queryUI"> {}
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index b3d577f0c87..23c599027e5 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -5,7 +5,7 @@ import {
   SearchFilters,
 } from "../../sdk"
 
-export type SearchFilter = {
+export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
   onEmptyFilter?: EmptyFilterOption
   field: string
@@ -14,9 +14,10 @@ export type SearchFilter = {
   externalType?: string
 }
 
+// this is a type purely used by the UI
 export type SearchFilterGroup = {
   logicalOperator: FilterGroupLogicalOperator
   onEmptyFilter?: EmptyFilterOption
   groups?: SearchFilterGroup[]
-  filters?: SearchFilter[]
+  filters?: LegacyFilter[]
 }
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 16817f177ac..87667a71e00 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -1,7 +1,7 @@
-import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
+import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
-import { DBView } from "../../sdk"
+import { DBView, SearchFilters } from "../../sdk"
 
 export type ViewTemplateOpts = {
   field: string
@@ -65,7 +65,9 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilter[] | SearchFilterGroup
+  query?: LegacyFilter[] | SearchFilters
+  // duplicate to store UI information about filters
+  queryUI?: SearchFilterGroup
   sort?: {
     field: string
     order?: SortOrder

From 2fed6008fa81ec462532f72b7c23a9a58cb10900 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 2 Oct 2024 18:41:27 +0100
Subject: [PATCH 200/455] Updating view stores.

---
 .../builder/src/stores/builder/viewsV2.js     | 22 ++++++++++++++++++-
 .../grid/stores/datasources/viewV2.js         | 12 +++++++++-
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index 9bd32f4a247..a3ceb623a93 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -1,6 +1,23 @@
 import { writable, derived, get } from "svelte/store"
 import { tables } from "./tables"
 import { API } from "api"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
+
+function convertToSearchFilterGroup(view) {
+  if (view.queryUI) {
+    view.query = view.queryUI
+  }
+  return view
+}
 
 export function createViewsV2Store() {
   const store = writable({
@@ -12,7 +29,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views)
+      list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
     })
     return {
       ...$store,
@@ -34,6 +51,7 @@ export function createViewsV2Store() {
   }
 
   const create = async view => {
+    view = convertToSearchFilters(view)
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
     replaceView(savedView.id, savedView)
@@ -41,6 +59,7 @@ export function createViewsV2Store() {
   }
 
   const save = async view => {
+    view = convertToSearchFilters(view)
     const res = await API.viewV2.update(view)
     const savedView = res?.data
     replaceView(view.id, savedView)
@@ -51,6 +70,7 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
+    view = convertToSearchFilterGroup(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index bf5e7c26285..13c46f9d34f 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -1,4 +1,14 @@
 import { get } from "svelte/store"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
 
 const SuppressErrors = true
 
@@ -6,7 +16,7 @@ export const createActions = context => {
   const { API, datasource, columns } = context
 
   const saveDefinition = async newDefinition => {
-    await API.viewV2.update(newDefinition)
+    await API.viewV2.update(convertToSearchFilters(newDefinition))
   }
 
   const saveRow = async row => {

From bcb940b7eb3cc2ebe40f2788c3494f9f861e9122 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 2 Oct 2024 18:45:47 +0100
Subject: [PATCH 201/455] Merge issue.

---
 packages/server/src/api/controllers/view/viewsV2.ts | 4 ++--
 packages/types/src/api/web/app/view.ts              | 5 ++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index 0d74a2f5d05..0257c86ded5 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -99,7 +99,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const schema = await parseSchema(view)
 
-  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version" | "queryUI"> = {
+  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
@@ -133,7 +133,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const { tableId } = view
 
   const schema = await parseSchema(view)
-  const parsedView: RequiredKeys<Omit<ViewV2, "queryUI">> = {
+  const parsedView: RequiredKeys<ViewV2> = {
     id: view.id,
     name: view.name,
     version: view.version,
diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts
index a99f2938aba..a6be5e29868 100644
--- a/packages/types/src/api/web/app/view.ts
+++ b/packages/types/src/api/web/app/view.ts
@@ -9,7 +9,6 @@ export interface ViewResponseEnriched {
   data: ViewV2Enriched
 }
 
-export interface CreateViewRequest
-  extends Omit<ViewV2, "version" | "id" | "queryUI"> {}
+export interface CreateViewRequest extends Omit<ViewV2, "version" | "id"> {}
 
-export interface UpdateViewRequest extends Omit<ViewV2, "queryUI"> {}
+export interface UpdateViewRequest extends ViewV2 {}

From 932413c4c1df3e99ee64c5ecbccd68d222f0e6fc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 2 Oct 2024 18:57:40 +0100
Subject: [PATCH 202/455] Getting re-loading of view queries working again.

---
 packages/builder/src/stores/builder/viewsV2.js                  | 1 +
 .../src/components/grid/stores/datasources/viewV2.js            | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index a3ceb623a93..0c7a03414ed 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -15,6 +15,7 @@ function convertToSearchFilters(view) {
 function convertToSearchFilterGroup(view) {
   if (view.queryUI) {
     view.query = view.queryUI
+    delete view.queryUI
   }
   return view
 }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 13c46f9d34f..8e4321be967 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -135,7 +135,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          filter.set($definition.query)
+          filter.set($definition.queryUI || $definition.query)
         }
       })
     )

From 9a4a788283c92ff1ec30e318b1cafbcc4855ed13 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 2 Oct 2024 21:15:42 +0100
Subject: [PATCH 203/455] dont show AI type if AI not enabled

---
 .../backend/DataTable/modals/CreateEditColumn.svelte          | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index f594a3ecf4f..67c7927be8e 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -25,6 +25,7 @@
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
+  import { licensing } from "stores/portal"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -105,6 +106,7 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
+  $: aiEnabled = $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
@@ -432,7 +434,7 @@
         FIELDS.BOOLEAN,
         FIELDS.DATETIME,
         FIELDS.LINK,
-        FIELDS.AI,
+        ...(aiEnabled ? [FIELDS.AI] : []),
         FIELDS.LONGFORM,
         FIELDS.USER,
         FIELDS.USERS,

From 2dff149aa9ed4fb4f05dc8f56928ffe9b66d1ea9 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 2 Oct 2024 22:36:36 +0100
Subject: [PATCH 204/455] lint

---
 .../DataTable/modals/CreateEditColumn.svelte  |  3 +-
 .../common/AIFieldConfiguration.svelte        | 73 ++-----------------
 .../src/components/grid/cells/AICell.svelte   |  6 +-
 .../src/sdk/app/tables/internal/index.ts      |  4 +-
 .../src/utilities/rowProcessor/utils.ts       | 14 ++--
 packages/shared-core/src/constants/ai.ts      | 68 +++++++++++++++++
 packages/shared-core/src/constants/index.ts   |  1 +
 packages/types/src/documents/app/index.ts     |  1 -
 .../types/src/documents/app/table/schema.ts   |  2 +-
 .../types/src/{documents/app => sdk}/ai.ts    | 15 +---
 packages/types/src/sdk/index.ts               |  1 +
 11 files changed, 92 insertions(+), 96 deletions(-)
 create mode 100644 packages/shared-core/src/constants/ai.ts
 rename packages/types/src/{documents/app => sdk}/ai.ts (78%)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 67c7927be8e..348c74dec11 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -106,7 +106,8 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
-  $: aiEnabled = $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
+  $: aiEnabled =
+    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte
index 8afe78c73c7..0a8a2850c58 100644
--- a/packages/builder/src/components/common/AIFieldConfiguration.svelte
+++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte
@@ -1,71 +1,12 @@
 <script>
-  import { Input, Helpers, Multiselect, Select, TextArea } from "@budibase/bbui"
+  import { Helpers, Multiselect, Select } from "@budibase/bbui"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
-
-  const AIOperations = {
-    SUMMARISE_TEXT: {
-      label: "Summarise Text",
-      value: "SUMMARISE_TEXT",
-    },
-    CLEAN_DATA: {
-      label: "Clean Data",
-      value: "CLEAN_DATA",
-    },
-    TRANSLATE: {
-      label: "Translate",
-      value: "TRANSLATE",
-    },
-    CATEGORISE_TEXT: {
-      label: "Categorise Text",
-      value: "CATEGORISE_TEXT",
-    },
-    SENTIMENT_ANALYSIS: {
-      label: "Sentiment Analysis",
-      value: "SENTIMENT_ANALYSIS",
-    },
-    PROMPT: {
-      label: "Prompt",
-      value: "PROMPT",
-    },
-    SEARCH_WEB: {
-      label: "Search Web",
-      value: "SEARCH_WEB",
-    },
-  }
-
-  const OperationFieldTypes = {
-    MULTI_COLUMN: "columns",
-    COLUMN: "column",
-    BINDABLE_TEXT: "prompt",
-    // LANGUAGE: "language",
-  }
-
-  const OperationFields = {
-    SUMMARISE_TEXT: {
-      columns: OperationFieldTypes.MULTI_COLUMN,
-    },
-    CLEAN_DATA: {
-      column: OperationFieldTypes.COLUMN,
-    },
-    TRANSLATE: {
-      column: OperationFieldTypes.COLUMN,
-      language: OperationFieldTypes.BINDABLE_TEXT,
-    },
-    CATEGORISE_TEXT: {
-      columns: OperationFieldTypes.MULTI_COLUMN,
-      categories: OperationFieldTypes.BINDABLE_TEXT,
-    },
-    SENTIMENT_ANALYSIS: {
-      column: OperationFieldTypes.COLUMN,
-    },
-    PROMPT: {
-      prompt: OperationFieldTypes.BINDABLE_TEXT,
-    },
-    SEARCH_WEB: {
-      columns: OperationFieldTypes.MULTI_COLUMN,
-    },
-  }
+  import {
+    AIOperations,
+    OperationFields,
+    OperationFieldTypes,
+  } from "@budibase/shared-core"
 
   const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({
     label: AIOperations[key].label,
@@ -78,8 +19,6 @@
   export let aiField = {}
 
   $: OperationField = OperationFields[aiField.operation] || null
-  $: console.log(aiField)
-  $: console.log(schema)
   $: schemaWithoutRelations = Object.keys(schema).filter(
     key => schema[key].type !== "link"
   )
diff --git a/packages/frontend-core/src/components/grid/cells/AICell.svelte b/packages/frontend-core/src/components/grid/cells/AICell.svelte
index 4e11fb7ebd3..421c028d047 100644
--- a/packages/frontend-core/src/components/grid/cells/AICell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AICell.svelte
@@ -42,11 +42,7 @@
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
-<div
-  class="long-form-cell"
-  on:click={open}
-  bind:this={anchor}
->
+<div class="long-form-cell" on:click={open} bind:this={anchor}>
   <div class="value">
     {value || ""}
   </div>
diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts
index 57b4cb0c80d..c0beed0db80 100644
--- a/packages/server/src/sdk/app/tables/internal/index.ts
+++ b/packages/server/src/sdk/app/tables/internal/index.ts
@@ -15,9 +15,7 @@ import {
 import { EventType, updateLinks } from "../../../../db/linkedRows"
 import { cloneDeep } from "lodash/fp"
 import isEqual from "lodash/isEqual"
-import {
-  runStaticFormulaChecks,
-} from "../../../../api/controllers/table/bulkFormula"
+import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
 import { context } from "@budibase/backend-core"
 import { findDuplicateInternalColumns } from "@budibase/shared-core"
 import { getTable } from "../getters"
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 0b46eef6c5c..1df2ff2fec4 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -8,7 +8,10 @@ import {
   FormulaType,
   AutoFieldSubType,
   FieldType,
+  OperationFieldTypeEnum,
+  AIFieldMetadata,
 } from "@budibase/types"
+import { OperationFields } from "@budibase/shared-core"
 import tracer from "dd-trace"
 import { context } from "@budibase/backend-core"
 import * as pro from "@budibase/pro"
@@ -116,12 +119,13 @@ export async function processAIColumns<T extends Row | Row[]>(
 
         const rowUpdates = rows.map((row, i) => {
           const contextRow = contextRows ? contextRows[i] : row
-          // TODO: Map the prompts with string-templates
-          // grab the operation based on the schema
-          // then check the types in the fields, and decide whether to pass them through string templates
-          // TODO: cleaner way to map to the schema, move things into BB types and check against the AI schema
+
+          // Check if the type is bindable and pass through HBS if so
+          const operationField =
+            OperationFields[(schema as AIFieldMetadata).operation]
           for (const key in schema) {
-            if (["prompt", "categories"].includes(key)) {
+            const fieldType = operationField[key]
+            if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
               schema[key] = processStringSync(schema[key], contextRow)
             }
           }
diff --git a/packages/shared-core/src/constants/ai.ts b/packages/shared-core/src/constants/ai.ts
new file mode 100644
index 00000000000..e3f035e72bc
--- /dev/null
+++ b/packages/shared-core/src/constants/ai.ts
@@ -0,0 +1,68 @@
+import {
+  AIOperationEnum,
+  OperationFieldsType,
+  OperationFieldTypeEnum,
+} from "@budibase/types"
+
+const AIOperations = {
+  SUMMARISE_TEXT: {
+    label: "Summarise Text",
+    value: "SUMMARISE_TEXT",
+  },
+  CLEAN_DATA: {
+    label: "Clean Data",
+    value: "CLEAN_DATA",
+  },
+  TRANSLATE: {
+    label: "Translate",
+    value: "TRANSLATE",
+  },
+  CATEGORISE_TEXT: {
+    label: "Categorise Text",
+    value: "CATEGORISE_TEXT",
+  },
+  SENTIMENT_ANALYSIS: {
+    label: "Sentiment Analysis",
+    value: "SENTIMENT_ANALYSIS",
+  },
+  PROMPT: {
+    label: "Prompt",
+    value: "PROMPT",
+  },
+  SEARCH_WEB: {
+    label: "Search Web",
+    value: "SEARCH_WEB",
+  },
+}
+
+export const OperationFieldTypes = {
+  MULTI_COLUMN: "columns",
+  COLUMN: "column",
+  BINDABLE_TEXT: "prompt",
+}
+
+export const OperationFields: OperationFieldsType = {
+  [AIOperationEnum.SUMMARISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+  },
+  [AIOperationEnum.CLEAN_DATA]: {
+    column: OperationFieldTypeEnum.COLUMN,
+  },
+  [AIOperationEnum.TRANSLATE]: {
+    column: OperationFieldTypeEnum.COLUMN,
+    language: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.CATEGORISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+    categories: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: {
+    column: OperationFieldTypeEnum.COLUMN,
+  },
+  [AIOperationEnum.PROMPT]: {
+    prompt: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.SEARCH_WEB]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+  },
+}
diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts
index 5a42fc56772..6c49625937f 100644
--- a/packages/shared-core/src/constants/index.ts
+++ b/packages/shared-core/src/constants/index.ts
@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./api"
 export * from "./fields"
 export * from "./rows"
diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts
index 0290e9ed2ef..2b13676ba13 100644
--- a/packages/types/src/documents/app/index.ts
+++ b/packages/types/src/documents/app/index.ts
@@ -1,4 +1,3 @@
-export * from "./ai"
 export * from "./app"
 export * from "./automation"
 export * from "./datasource"
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index d84ba5f8ecb..c62921a3ea2 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -9,7 +9,7 @@ import {
   JsonFieldSubType,
   RelationshipType,
 } from "./constants"
-import { AIOperationEnum } from "../ai"
+import { AIOperationEnum } from "../../../sdk/ai"
 
 export interface UIFieldMetadata {
   order?: number
diff --git a/packages/types/src/documents/app/ai.ts b/packages/types/src/sdk/ai.ts
similarity index 78%
rename from packages/types/src/documents/app/ai.ts
rename to packages/types/src/sdk/ai.ts
index 7e33eeb651a..efcf8193b5b 100644
--- a/packages/types/src/documents/app/ai.ts
+++ b/packages/types/src/sdk/ai.ts
@@ -8,13 +8,13 @@ export enum AIOperationEnum {
   SEARCH_WEB = "SEARCH_WEB",
 }
 
-enum OperationFieldTypeEnum {
+export enum OperationFieldTypeEnum {
   MULTI_COLUMN = "columns",
   COLUMN = "column",
   BINDABLE_TEXT = "prompt",
 }
 
-type OperationFieldsType = {
+export type OperationFieldsType = {
   [AIOperationEnum.SUMMARISE_TEXT]: {
     columns: OperationFieldTypeEnum.MULTI_COLUMN
   }
@@ -40,17 +40,6 @@ type OperationFieldsType = {
   }
 }
 
-// Define the AI operations with just their labels (no need for separate 'value' property)
-const AIOperations: { [key in AIOperationEnum]: string } = {
-  [AIOperationEnum.SUMMARISE_TEXT]: "Summarise Text",
-  [AIOperationEnum.CLEAN_DATA]: "Clean Data",
-  [AIOperationEnum.TRANSLATE]: "Translate",
-  [AIOperationEnum.CATEGORISE_TEXT]: "Categorise Text",
-  [AIOperationEnum.SENTIMENT_ANALYSIS]: "Sentiment Analysis",
-  [AIOperationEnum.PROMPT]: "Prompt",
-  [AIOperationEnum.SEARCH_WEB]: "Search Web",
-}
-
 type BaseSchema = {
   operation: AIOperationEnum
 }
diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts
index d87ec58b0ca..86eb5b1a244 100644
--- a/packages/types/src/sdk/index.ts
+++ b/packages/types/src/sdk/index.ts
@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./automations"
 export * from "./hosting"
 export * from "./context"

From 4de91d4e3aaf19488388fe1795e2c7b07948d53e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 09:27:04 +0100
Subject: [PATCH 205/455] Add animated curly brackets

---
 .../backend/RoleEditor/BracketEdge.svelte     | 63 +++++++++++++++
 .../backend/RoleEditor/RoleFlow.svelte        | 12 +--
 .../backend/RoleEditor/RoleNode.svelte        | 62 +++-----------
 .../backend/RoleEditor/constants.js           |  2 +-
 .../components/backend/RoleEditor/utils.js    | 81 +++++++++++--------
 5 files changed, 131 insertions(+), 89 deletions(-)
 create mode 100644 packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte

diff --git a/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte b/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
new file mode 100644
index 00000000000..643fe28532b
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
@@ -0,0 +1,63 @@
+<script>
+  import { BaseEdge } from "@xyflow/svelte"
+  import { NodeWidth, GridResolution } from "./constants"
+  import { getContext } from "svelte"
+
+  export let sourceX
+  export let sourceY
+
+  const { bounds } = getContext("flow")
+  const BracketWidth = GridResolution * 2
+  const BracketHeight = $bounds.height / 2 + GridResolution * 2
+
+  $: path = getCurlyBracePath(
+    sourceX + BracketWidth,
+    sourceY - BracketHeight,
+    sourceX + BracketWidth,
+    sourceY + BracketHeight
+  )
+
+  const getCurlyBracePath = (x1, y1, x2, y2) => {
+    const w = 2 // Thickness
+    const q = 2 // Intensity
+    const i = 28 // Inner radius strenth (lower is stronger)
+    const j = 32 // Outer radius strength (higher is stronger)
+
+    // Calculate unit vector
+    var dx = x1 - x2
+    var dy = y1 - y2
+    var len = Math.sqrt(dx * dx + dy * dy)
+    dx = dx / len
+    dy = dy / len
+
+    // Path control points
+    const qx1 = x1 + q * w * dy - j
+    const qy1 = y1 - q * w * dx
+    const qx2 = x1 - 0.25 * len * dx + (1 - q) * w * dy - i
+    const qy2 = y1 - 0.25 * len * dy - (1 - q) * w * dx
+    const tx1 = x1 - 0.5 * len * dx + w * dy - BracketWidth
+    const ty1 = y1 - 0.5 * len * dy - w * dx
+    const qx3 = x2 + q * w * dy - j
+    const qy3 = y2 - q * w * dx
+    const qx4 = x1 - 0.75 * len * dx + (1 - q) * w * dy - i
+    const qy4 = y1 - 0.75 * len * dy - (1 - q) * w * dx
+
+    return `M ${x1} ${y1} Q ${qx1} ${qy1} ${qx2} ${qy2} T ${tx1} ${ty1} M ${x2} ${y2} Q ${qx3} ${qy3} ${qx4} ${qy4} T ${tx1} ${ty1}`
+  }
+</script>
+
+<BaseEdge
+  {...$$props}
+  {path}
+  style="--width:{NodeWidth}px; --x:{sourceX}px; --y:{sourceY}px;"
+/>
+
+<style>
+  :global(#basic-bracket) {
+    animation-timing-function: linear(1, 0);
+  }
+  :global(#admin-bracket) {
+    transform: scale(-1, 1) translateX(calc(var(--width) + 8px));
+    transform-origin: var(--x) var(--y);
+  }
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 7a9280d679d..1e894f0cc98 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -11,11 +11,13 @@
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
   import RoleEdge from "./RoleEdge.svelte"
+  import BracketEdge from "./BracketEdge.svelte"
   import {
     autoLayout,
     getAdminPosition,
     getBasicPosition,
     rolesToLayout,
+    nodeToRole,
   } from "./utils"
   import { setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
@@ -81,7 +83,7 @@
       name: roleId,
       uiMetadata: {
         displayName: getSequentialName($roles, "New role ", {
-          getName: x => x.uiMetadata.displayName,
+          getName: role => role.uiMetadata.displayName,
         }),
         color: "var(--spectrum-global-color-gray-700)",
         description: "Custom role",
@@ -108,7 +110,7 @@
     if (metadata) {
       flow.updateNodeData(roleId, metadata)
     }
-    await roles.save(nodeToRole(node))
+    await roles.save(nodeToRole({ node, edges: $edges }))
   }
 
   const deleteRole = async roleId => {
@@ -119,11 +121,11 @@
   }
 
   const deleteEdge = async edgeId => {
-    const edge = $edges.find(x => x.id === edgeId)
+    const edge = $edges.find(edge => edge.id === edgeId)
     if (!edge) {
       return
     }
-    edges.set($edges.filter(x => x.id !== edgeId))
+    edges.set($edges.filter(edge => edge.id !== edgeId))
     await updateRole(edge.target)
   }
 
@@ -158,7 +160,7 @@
     {edges}
     snapGrid={[GridResolution, GridResolution]}
     nodeTypes={{ role: RoleNode }}
-    edgeTypes={{ role: RoleEdge }}
+    edgeTypes={{ role: RoleEdge, bracket: BracketEdge }}
     proOptions={{ hideAttribution: true }}
     fitViewOptions={{ maxZoom: MaxAutoZoom }}
     defaultEdgeOptions={{ type: "role", animated: true, selectable: false }}
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 57ef54ee5ce..6f1c39d67ea 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -8,8 +8,7 @@
     ModalContent,
     FieldLabel,
   } from "@budibase/bbui"
-  import { Roles } from "constants/backend"
-  import { NodeWidth, NodeHeight, NodeVSpacing } from "./constants"
+  import { NodeWidth, NodeHeight } from "./constants"
   import { getContext } from "svelte"
   import { roles } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -70,6 +69,7 @@
 <div
   class="node"
   class:selected
+  class:custom={data.custom}
   style={`--color:${data.color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
   bind:this={anchor}
 >
@@ -92,24 +92,14 @@
       </div>
     {/if}
   </div>
-  {#if isConnectable}
-    <Handle
-      type="target"
-      position={Position.Left}
-      class={targetClasses}
-      isConnectable={$dragging}
-    />
-    <Handle type="source" position={Position.Right} />
-  {/if}
-</div>
-
-{#if id === Roles.BASIC || id === Roles.ADMIN}
-  <div
-    class="bounds"
-    class:flip={id === Roles.ADMIN}
-    style="--height:{$bounds.height}px; --spacing:{NodeVSpacing}px;"
+  <Handle
+    type="target"
+    position={Position.Left}
+    class={targetClasses}
+    isConnectable={isConnectable && $dragging}
   />
-{/if}
+  <Handle type="source" position={Position.Right} {isConnectable} />
+</div>
 
 <ConfirmDialog
   bind:this={deleteModal}
@@ -222,7 +212,7 @@
     color: var(--spectrum-global-color-gray-600);
   }
 
-  /* Handlers */
+  /* Handles */
   .node :global(.svelte-flow__handle) {
     width: 6px;
     height: 6px;
@@ -235,34 +225,8 @@
     opacity: 0;
     pointer-events: none;
   }
-
-  /* Bounds brackets */
-  .bounds {
-    height: calc(var(--height) + 2 * var(--spacing));
-    position: absolute;
-    width: calc(var(--spacing) / 1.5);
-    border: 2px dashed var(--edge-color);
-    border-right: none;
-    top: 50%;
-    left: calc(100% + var(--spacing));
-    transform: translateY(-50%);
-    border-top-left-radius: 8px;
-    border-bottom-left-radius: 8px;
-  }
-  .bounds.flip {
-    left: calc(-1 * var(--spacing));
-    transform: translateY(-50%) translateX(-100%) scale(-1, 1);
-  }
-  .bounds::after {
-    content: "";
-    display: block;
-    position: absolute;
-    top: 50%;
-    left: calc(-1 * var(--spacing));
-    height: 0;
-    border-top: 2px dashed var(--edge-color);
-    width: var(--spacing);
-    box-sizing: border-box;
-    transform: translateY(-50%);
+  .node:not(.custom) :global(.svelte-flow__handle) {
+    visibility: hidden;
+    pointer-events: none;
   }
 </style>
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index ba43e349b51..aa4b1f26f37 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -3,5 +3,5 @@ export const MaxAutoZoom = 1.2
 export const GridResolution = 20
 export const NodeHeight = GridResolution * 3
 export const NodeWidth = GridResolution * 12
-export const NodeHSpacing = GridResolution * 4
+export const NodeHSpacing = GridResolution * 6
 export const NodeVSpacing = GridResolution * 2
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index edd4cbe3356..ad0c2e110e6 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -8,19 +8,36 @@ import {
 } from "./constants"
 import { getNodesBounds, Position } from "@xyflow/svelte"
 import { Roles } from "constants/backend"
+import { roles } from "stores/builder"
+import { get } from "svelte/store"
 
 // Gets the position of the basic role
 export const getBasicPosition = bounds => ({
-  x: bounds.x - NodeHSpacing - NodeWidth,
+  x: bounds.x - GridResolution * 4 - NodeWidth,
   y: bounds.y + bounds.height / 2 - NodeHeight / 2,
 })
 
 // Gets the position of the admin role
 export const getAdminPosition = bounds => ({
-  x: bounds.x + bounds.width + NodeHSpacing,
+  x: bounds.x + bounds.width + GridResolution * 4,
   y: bounds.y + bounds.height / 2 - NodeHeight / 2,
 })
 
+// Filters out invalid nodes and edges
+const preProcessLayout = ({ nodes, edges }) => {
+  const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
+  const edglessRoles = [...ignoredRoles, Roles.BASIC, Roles.ADMIN]
+  return {
+    nodes: nodes.filter(node => !ignoredRoles.includes(node.id)),
+    edges: edges.filter(edge => {
+      return (
+        !edglessRoles.includes(edge.source) &&
+        !edglessRoles.includes(edge.target)
+      )
+    }),
+  }
+}
+
 // Updates positions of nodes and edges into a nice graph structure
 export const dagreLayout = ({ nodes, edges }) => {
   const dagreGraph = new dagre.graphlib.Graph()
@@ -46,33 +63,34 @@ export const dagreLayout = ({ nodes, edges }) => {
       y: Math.round((pos.y - NodeHeight / 2) / GridResolution) * GridResolution,
     }
   })
+  return { nodes, edges }
+}
 
+const postProcessLayout = ({ nodes, edges }) => {
   // Reposition basic and admin to bound the custom nodes
   const bounds = getNodesBounds(nodes.filter(node => node.data.custom))
   nodes.find(x => x.id === Roles.BASIC).position = getBasicPosition(bounds)
   nodes.find(x => x.id === Roles.ADMIN).position = getAdminPosition(bounds)
 
+  // Add custom edges for basic and admin brackets
+  edges.push({
+    id: "basic-bracket",
+    source: Roles.BASIC,
+    target: Roles.ADMIN,
+    type: "bracket",
+  })
+  edges.push({
+    id: "admin-bracket",
+    source: Roles.ADMIN,
+    target: Roles.BASIC,
+    type: "bracket",
+  })
   return { nodes, edges }
 }
 
-// Adds additional edges as needed to the flow structure to ensure compatibility with BB role logic
-const sanitiseLayout = ({ nodes, edges }) => {
-  const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
-  const edglessRoles = [...ignoredRoles, Roles.BASIC, Roles.ADMIN]
-  return {
-    nodes: nodes.filter(node => !ignoredRoles.includes(node.id)),
-    edges: edges.filter(edge => {
-      return (
-        !edglessRoles.includes(edge.source) &&
-        !edglessRoles.includes(edge.target)
-      )
-    }),
-  }
-}
-
 // Automatically lays out the graph, sanitising and enriching the structure
 export const autoLayout = ({ nodes, edges }) => {
-  return dagreLayout(sanitiseLayout({ nodes, edges }))
+  return postProcessLayout(dagreLayout(preProcessLayout({ nodes, edges })))
 }
 
 // Converts a role doc into a node structure
@@ -99,27 +117,22 @@ export const roleToNode = role => {
 }
 
 // Converts a node structure back into a role doc
-export const nodeToRole = node => {
-  const role = $roles.find(x => x._id === node.id)
-  const inherits = $edges.filter(x => x.target === node.id).map(x => x.source)
-  // TODO save inherits array
-  return {
-    ...role,
-    inherits,
-    uiMetadata: {
-      displayName: node.data.displayName,
-      color: node.data.color,
-      description: node.data.description,
-    },
-  }
-}
+export const nodeToRole = ({ node, edges }) => ({
+  ...get(roles).find(role => role._id === node.id),
+  inherits: edges.filter(x => x.target === node.id).map(x => x.source),
+  uiMetadata: {
+    displayName: node.data.displayName,
+    color: node.data.color,
+    description: node.data.description,
+  },
+})
 
-// Builds a layout from an array of roles
+// Builds a default layout from an array of roles
 export const rolesToLayout = roles => {
   let nodes = []
   let edges = []
 
-  // Remove some builtins
+  // Add all nodes and edges
   for (let role of roles) {
     // Add node for this role
     nodes.push(roleToNode(role))

From f430de53692bf27735b67ba80e7d37b2adc2964b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 09:36:18 +0100
Subject: [PATCH 206/455] Add more edge validation

---
 .../components/backend/RoleEditor/utils.js    | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index ad0c2e110e6..9156ff81590 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -28,12 +28,26 @@ const preProcessLayout = ({ nodes, edges }) => {
   const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
   const edglessRoles = [...ignoredRoles, Roles.BASIC, Roles.ADMIN]
   return {
-    nodes: nodes.filter(node => !ignoredRoles.includes(node.id)),
+    nodes: nodes.filter(node => {
+      // Filter out ignored roles
+      if (ignoredRoles.includes(node.id)) {
+        return false
+      }
+      return true
+    }),
     edges: edges.filter(edge => {
-      return (
-        !edglessRoles.includes(edge.source) &&
-        !edglessRoles.includes(edge.target)
-      )
+      // Filter out edges from ignored roles
+      if (
+        edglessRoles.includes(edge.source) ||
+        edglessRoles.includes(edge.target)
+      ) {
+        return false
+      }
+      // Filter out edges which have the same source and target
+      if (edge.source === edge.target) {
+        return false
+      }
+      return true
     }),
   }
 }

From fbdbefddf729eac7eff28956f56ec7d1d1fafef7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 11:21:30 +0100
Subject: [PATCH 207/455] Fix resizing issue with brackes

---
 .../backend/RoleEditor/BracketEdge.svelte          | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte b/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
index 643fe28532b..37aa772a22d 100644
--- a/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
@@ -7,14 +7,14 @@
   export let sourceY
 
   const { bounds } = getContext("flow")
-  const BracketWidth = GridResolution * 2
-  const BracketHeight = $bounds.height / 2 + GridResolution * 2
 
+  $: bracketWidth = GridResolution * 3
+  $: bracketHeight = $bounds.height / 2 + GridResolution * 2
   $: path = getCurlyBracePath(
-    sourceX + BracketWidth,
-    sourceY - BracketHeight,
-    sourceX + BracketWidth,
-    sourceY + BracketHeight
+    sourceX + bracketWidth,
+    sourceY - bracketHeight,
+    sourceX + bracketWidth,
+    sourceY + bracketHeight
   )
 
   const getCurlyBracePath = (x1, y1, x2, y2) => {
@@ -35,7 +35,7 @@
     const qy1 = y1 - q * w * dx
     const qx2 = x1 - 0.25 * len * dx + (1 - q) * w * dy - i
     const qy2 = y1 - 0.25 * len * dy - (1 - q) * w * dx
-    const tx1 = x1 - 0.5 * len * dx + w * dy - BracketWidth
+    const tx1 = x1 - 0.5 * len * dx + w * dy - bracketWidth
     const ty1 = y1 - 0.5 * len * dy - w * dx
     const qx3 = x2 + q * w * dy - j
     const qy3 = y2 - q * w * dx

From 72f1db8c1026003a8b9fce16b9ca1af7e73a8799 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 11:24:22 +0100
Subject: [PATCH 208/455] Update spacing

---
 packages/builder/src/components/backend/RoleEditor/utils.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index 9156ff81590..8aa8d734437 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -13,13 +13,13 @@ import { get } from "svelte/store"
 
 // Gets the position of the basic role
 export const getBasicPosition = bounds => ({
-  x: bounds.x - GridResolution * 4 - NodeWidth,
+  x: bounds.x - NodeHSpacing - NodeWidth,
   y: bounds.y + bounds.height / 2 - NodeHeight / 2,
 })
 
 // Gets the position of the admin role
 export const getAdminPosition = bounds => ({
-  x: bounds.x + bounds.width + GridResolution * 4,
+  x: bounds.x + bounds.width + NodeHSpacing,
   y: bounds.y + bounds.height / 2 - NodeHeight / 2,
 })
 

From 303f4001398b7467fa16e7a48b0535d61d80f2d4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 11:28:41 +0100
Subject: [PATCH 209/455] Prevent selection of basic and admin roles

---
 .../src/components/backend/RoleEditor/RoleNode.svelte      | 7 ++++---
 .../builder/src/components/backend/RoleEditor/utils.js     | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 6f1c39d67ea..d8e69df0812 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -70,6 +70,7 @@
   class="node"
   class:selected
   class:custom={data.custom}
+  class:selectable={isConnectable}
   style={`--color:${data.color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
   bind:this={anchor}
 >
@@ -146,13 +147,13 @@
     display: flex;
     flex-direction: row;
     box-sizing: border-box;
-    cursor: pointer;
     transition: background 130ms ease-out;
   }
-  .node:hover {
+  .node.selectable:hover {
+    cursor: pointer;
     background: var(--spectrum-global-color-gray-200);
   }
-  .node.selected {
+  .node.selectable.selected {
     background: var(--spectrum-global-color-blue-100);
     cursor: grab;
   }
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index 8aa8d734437..bf02cfd5af4 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -127,6 +127,7 @@ export const roleToNode = role => {
     deletable: custom,
     draggable: custom,
     connectable: custom,
+    selectable: custom,
   }
 }
 

From a342aa3c31c915a4547951fc668be446f24ba4fc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 13:08:47 +0100
Subject: [PATCH 210/455] Add empty state node to role editor

---
 .../backend/RoleEditor/Controls.svelte        | 11 ++---
 .../backend/RoleEditor/EmptyStateNode.svelte  | 24 ++++++++++
 .../backend/RoleEditor/RoleFlow.svelte        |  8 ++--
 .../components/backend/RoleEditor/utils.js    | 44 ++++++++++++++++++-
 4 files changed, 75 insertions(+), 12 deletions(-)
 create mode 100644 packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 919093bee56..0b5d70217cf 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -7,10 +7,13 @@
 
   const { nodes, edges, createRole } = getContext("flow")
   const flow = useSvelteFlow()
+  const autoFit = () =>
+    flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })
 
   const addRole = async () => {
     await createRole()
     doAutoLayout()
+    autoFit()
   }
 
   const doAutoLayout = () => {
@@ -33,13 +36,7 @@
       on:click={() => flow.zoomOut({ duration: ZoomDuration })}
     />
   </div>
-  <Button
-    secondary
-    on:click={() =>
-      flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })}
-  >
-    Zoom to fit
-  </Button>
+  <Button secondary on:click={autoFit}>Zoom to fit</Button>
   <Button secondary on:click={doAutoLayout}>Auto layout</Button>
 </div>
 <div class="control bottom-right">
diff --git a/packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte b/packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte
new file mode 100644
index 00000000000..2c949ed0f9f
--- /dev/null
+++ b/packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte
@@ -0,0 +1,24 @@
+<script>
+  import { NodeWidth, NodeHeight } from "./constants"
+</script>
+
+<div class="node" style={`--width:${NodeWidth}px; --height:${NodeHeight}px;`}>
+  Add custom roles for more granular control over permissions
+</div>
+
+<style>
+  .node {
+    border-radius: 4px;
+    width: var(--width);
+    height: var(--height);
+    display: grid;
+    place-items: center;
+    font-size: 16px;
+    text-align: center;
+    color: var(--spectrum-global-color-gray-800);
+    text-shadow: 4px 4px 10px var(--background-color),
+      4px -4px 10px var(--background-color),
+      -4px 4px 10px var(--background-color),
+      -4px -4px 10px var(--background-color);
+  }
+</style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 1e894f0cc98..0bb1b04ddc6 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -10,6 +10,7 @@
   } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
+  import EmptyStateNode from "./EmptyStateNode.svelte"
   import RoleEdge from "./RoleEdge.svelte"
   import BracketEdge from "./BracketEdge.svelte"
   import {
@@ -18,6 +19,7 @@
     getBasicPosition,
     rolesToLayout,
     nodeToRole,
+    getBounds,
   } from "./utils"
   import { setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
@@ -38,9 +40,7 @@
   })
 
   // Derive the bounds of all custom role nodes
-  const bounds = derivedMemo(nodes, $nodes => {
-    return getNodesBounds($nodes.filter(node => node.data.custom))
-  })
+  const bounds = derivedMemo(nodes, getBounds)
 
   $: handleExternalRoleChanges($roles)
   $: updateBuiltins($bounds)
@@ -159,7 +159,7 @@
     {nodes}
     {edges}
     snapGrid={[GridResolution, GridResolution]}
-    nodeTypes={{ role: RoleNode }}
+    nodeTypes={{ role: RoleNode, empty: EmptyStateNode }}
     edgeTypes={{ role: RoleEdge, bracket: BracketEdge }}
     proOptions={{ hideAttribution: true }}
     fitViewOptions={{ maxZoom: MaxAutoZoom }}
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index bf02cfd5af4..fce8e9516c1 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -11,6 +11,22 @@ import { Roles } from "constants/backend"
 import { roles } from "stores/builder"
 import { get } from "svelte/store"
 
+// Calculates the bounds of all custom nodes
+export const getBounds = nodes => {
+  const customNodes = nodes.filter(node => node.data.custom)
+
+  // Empty state bounds which line up with bounds after adding first node
+  if (!customNodes.length) {
+    return {
+      x: 0,
+      y: 6.5 * GridResolution,
+      width: 12 * GridResolution,
+      height: 10 * GridResolution,
+    }
+  }
+  return getNodesBounds(customNodes)
+}
+
 // Gets the position of the basic role
 export const getBasicPosition = bounds => ({
   x: bounds.x - NodeHSpacing - NodeWidth,
@@ -33,6 +49,10 @@ const preProcessLayout = ({ nodes, edges }) => {
       if (ignoredRoles.includes(node.id)) {
         return false
       }
+      // Filter out empty state
+      if (node.id === "empty") {
+        return false
+      }
       return true
     }),
     edges: edges.filter(edge => {
@@ -82,7 +102,7 @@ export const dagreLayout = ({ nodes, edges }) => {
 
 const postProcessLayout = ({ nodes, edges }) => {
   // Reposition basic and admin to bound the custom nodes
-  const bounds = getNodesBounds(nodes.filter(node => node.data.custom))
+  const bounds = getBounds(nodes)
   nodes.find(x => x.id === Roles.BASIC).position = getBasicPosition(bounds)
   nodes.find(x => x.id === Roles.ADMIN).position = getAdminPosition(bounds)
 
@@ -99,6 +119,28 @@ const postProcessLayout = ({ nodes, edges }) => {
     target: Roles.BASIC,
     type: "bracket",
   })
+
+  // Add empty state node if required
+  if (!nodes.filter(node => node.data.custom).length) {
+    nodes.push({
+      id: "empty",
+      type: "empty",
+      position: {
+        x: bounds.x + bounds.width / 2 - NodeWidth / 2,
+        y: bounds.y + bounds.height / 2 - NodeHeight / 2,
+      },
+      data: {},
+      measured: {
+        width: NodeWidth,
+        height: NodeHeight,
+      },
+      deletable: false,
+      draggable: false,
+      connectable: false,
+      selectable: false,
+    })
+  }
+
   return { nodes, edges }
 }
 

From a4aac8aae057597a335cbc8c8c669ef5e5c86125 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 13:59:50 +0100
Subject: [PATCH 211/455] Deal with power users

---
 .../backend/RoleEditor/RoleNode.svelte        | 14 ++---
 .../components/backend/RoleEditor/utils.js    | 53 +++++++++++--------
 2 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index d8e69df0812..3dbab9537b7 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -12,6 +12,7 @@
   import { getContext } from "svelte"
   import { roles } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { Roles } from "constants/backend"
 
   export let data
   export let id
@@ -68,7 +69,9 @@
 
 <div
   class="node"
+  class:dragging={$dragging}
   class:selected
+  class:interactive={data.interactive}
   class:custom={data.custom}
   class:selectable={isConnectable}
   style={`--color:${data.color}; --width:${NodeWidth}px; --height:${NodeHeight}px;`}
@@ -96,8 +99,7 @@
   <Handle
     type="target"
     position={Position.Left}
-    class={targetClasses}
-    isConnectable={isConnectable && $dragging}
+    isConnectable={isConnectable && $dragging && data.custom}
   />
   <Handle type="source" position={Position.Right} {isConnectable} />
 </div>
@@ -222,11 +224,9 @@
   .node :global(.svelte-flow__handle.target) {
     background: var(--background-color);
   }
-  .node :global(.svelte-flow__handle.hidden) {
-    opacity: 0;
-    pointer-events: none;
-  }
-  .node:not(.custom) :global(.svelte-flow__handle) {
+  .node:not(.dragging) :global(.svelte-flow__handle.target),
+  .node:not(.interactive) :global(.svelte-flow__handle),
+  .node:not(.custom) :global(.svelte-flow__handle.target) {
     visibility: hidden;
     pointer-events: none;
   }
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index fce8e9516c1..9d23d46a753 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -13,18 +13,18 @@ import { get } from "svelte/store"
 
 // Calculates the bounds of all custom nodes
 export const getBounds = nodes => {
-  const customNodes = nodes.filter(node => node.data.custom)
+  const interactiveNodes = nodes.filter(node => node.data.interactive)
 
   // Empty state bounds which line up with bounds after adding first node
-  if (!customNodes.length) {
+  if (!interactiveNodes.length) {
     return {
       x: 0,
-      y: 6.5 * GridResolution,
+      y: -3.5 * GridResolution,
       width: 12 * GridResolution,
       height: 10 * GridResolution,
     }
   }
-  return getNodesBounds(customNodes)
+  return getNodesBounds(interactiveNodes)
 }
 
 // Gets the position of the basic role
@@ -41,25 +41,21 @@ export const getAdminPosition = bounds => ({
 
 // Filters out invalid nodes and edges
 const preProcessLayout = ({ nodes, edges }) => {
-  const ignoredRoles = [Roles.PUBLIC, Roles.POWER]
-  const edglessRoles = [...ignoredRoles, Roles.BASIC, Roles.ADMIN]
+  const ignoredIds = [Roles.PUBLIC, Roles.BASIC, Roles.ADMIN, "empty"]
+  const targetlessIds = [Roles.POWER]
   return {
     nodes: nodes.filter(node => {
-      // Filter out ignored roles
-      if (ignoredRoles.includes(node.id)) {
-        return false
-      }
-      // Filter out empty state
-      if (node.id === "empty") {
+      // Filter out ignored IDs
+      if (ignoredIds.includes(node.id)) {
         return false
       }
       return true
     }),
     edges: edges.filter(edge => {
-      // Filter out edges from ignored roles
+      // Filter out edges from ignored IDs
       if (
-        edglessRoles.includes(edge.source) ||
-        edglessRoles.includes(edge.target)
+        ignoredIds.includes(edge.source) ||
+        ignoredIds.includes(edge.target)
       ) {
         return false
       }
@@ -67,6 +63,10 @@ const preProcessLayout = ({ nodes, edges }) => {
       if (edge.source === edge.target) {
         return false
       }
+      // Filter out edges which target targetless roles
+      if (targetlessIds.includes(edge.target)) {
+        return false
+      }
       return true
     }),
   }
@@ -101,10 +101,17 @@ export const dagreLayout = ({ nodes, edges }) => {
 }
 
 const postProcessLayout = ({ nodes, edges }) => {
-  // Reposition basic and admin to bound the custom nodes
+  // Add basic and admin nodes at each edge
   const bounds = getBounds(nodes)
-  nodes.find(x => x.id === Roles.BASIC).position = getBasicPosition(bounds)
-  nodes.find(x => x.id === Roles.ADMIN).position = getAdminPosition(bounds)
+  const $roles = get(roles)
+  nodes.push({
+    ...roleToNode($roles.find(role => role._id === Roles.BASIC)),
+    position: getBasicPosition(bounds),
+  })
+  nodes.push({
+    ...roleToNode($roles.find(role => role._id === Roles.ADMIN)),
+    position: getAdminPosition(bounds),
+  })
 
   // Add custom edges for basic and admin brackets
   edges.push({
@@ -121,7 +128,7 @@ const postProcessLayout = ({ nodes, edges }) => {
   })
 
   // Add empty state node if required
-  if (!nodes.filter(node => node.data.custom).length) {
+  if (!nodes.some(node => node.data.interactive)) {
     nodes.push({
       id: "empty",
       type: "empty",
@@ -152,6 +159,7 @@ export const autoLayout = ({ nodes, edges }) => {
 // Converts a role doc into a node structure
 export const roleToNode = role => {
   const custom = !role._id.match(/[A-Z]+/)
+  const interactive = custom || role._id === Roles.POWER
   return {
     id: role._id,
     sourcePosition: Position.Right,
@@ -161,15 +169,16 @@ export const roleToNode = role => {
     data: {
       ...role.uiMetadata,
       custom,
+      interactive,
     },
     measured: {
       width: NodeWidth,
       height: NodeHeight,
     },
     deletable: custom,
-    draggable: custom,
-    connectable: custom,
-    selectable: custom,
+    draggable: interactive,
+    connectable: interactive,
+    selectable: interactive,
   }
 }
 

From 818732250b0e327d6f3711ad082e7f063aca7104 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 14:13:22 +0100
Subject: [PATCH 212/455] Add auto layout and fit when doing any role operation
 other than updating metadata

---
 .../backend/RoleEditor/Controls.svelte        | 24 ++++---------------
 .../backend/RoleEditor/RoleFlow.svelte        | 21 +++++++++++-----
 2 files changed, 19 insertions(+), 26 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 0b5d70217cf..e20fab29955 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -2,25 +2,10 @@
   import { Button, ActionButton } from "@budibase/bbui"
   import { useSvelteFlow } from "@xyflow/svelte"
   import { getContext } from "svelte"
-  import { autoLayout } from "./utils"
-  import { MaxAutoZoom, ZoomDuration } from "./constants"
+  import { ZoomDuration } from "./constants"
 
-  const { nodes, edges, createRole } = getContext("flow")
+  const { createRole, layoutAndFit } = getContext("flow")
   const flow = useSvelteFlow()
-  const autoFit = () =>
-    flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })
-
-  const addRole = async () => {
-    await createRole()
-    doAutoLayout()
-    autoFit()
-  }
-
-  const doAutoLayout = () => {
-    const layout = autoLayout({ nodes: $nodes, edges: $edges })
-    nodes.set(layout.nodes)
-    edges.set(layout.edges)
-  }
 </script>
 
 <div class="control top-left">
@@ -36,11 +21,10 @@
       on:click={() => flow.zoomOut({ duration: ZoomDuration })}
     />
   </div>
-  <Button secondary on:click={autoFit}>Zoom to fit</Button>
-  <Button secondary on:click={doAutoLayout}>Auto layout</Button>
+  <Button secondary on:click={layoutAndFit}>Auto layout</Button>
 </div>
 <div class="control bottom-right">
-  <Button icon="Add" cta on:click={addRole}>Add role</Button>
+  <Button icon="Add" cta on:click={createRole}>Add role</Button>
 </div>
 
 <style>
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 0bb1b04ddc6..a7c2818da08 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -6,7 +6,6 @@
     Background,
     BackgroundVariant,
     useSvelteFlow,
-    getNodesBounds,
   } from "@xyflow/svelte"
   import "@xyflow/svelte/dist/style.css"
   import RoleNode from "./RoleNode.svelte"
@@ -23,7 +22,7 @@
   } from "./utils"
   import { setContext, tick } from "svelte"
   import Controls from "./Controls.svelte"
-  import { GridResolution, MaxAutoZoom } from "./constants"
+  import { GridResolution, MaxAutoZoom, ZoomDuration } from "./constants"
   import { roles } from "stores/builder"
   import { Roles } from "constants/backend"
   import { getSequentialName } from "helpers/duplicate"
@@ -77,6 +76,14 @@
     })
   }
 
+  // Automatically lays out all roles and edges and zooms to fit them
+  const layoutAndFit = () => {
+    const layout = autoLayout({ nodes: $nodes, edges: $edges })
+    nodes.set(layout.nodes)
+    edges.set(layout.edges)
+    flow.fitView({ maxZoom: MaxAutoZoom, duration: ZoomDuration })
+  }
+
   const createRole = async () => {
     const roleId = Helpers.uuid()
     await roles.save({
@@ -91,6 +98,7 @@
       permissionId: "write",
     })
     await tick()
+    layoutAndFit()
 
     // Select the new node
     nodes.update($nodes => {
@@ -114,6 +122,8 @@
   }
 
   const deleteRole = async roleId => {
+    nodes.set($nodes.filter(node => node.id !== roleId))
+    layoutAndFit()
     const role = $roles.find(role => role._id === roleId)
     if (role) {
       roles.delete(role)
@@ -122,15 +132,13 @@
 
   const deleteEdge = async edgeId => {
     const edge = $edges.find(edge => edge.id === edgeId)
-    if (!edge) {
-      return
-    }
     edges.set($edges.filter(edge => edge.id !== edgeId))
+    layoutAndFit()
     await updateRole(edge.target)
   }
 
-  // Updates roles which have had a new connection
   const onConnect = async connection => {
+    layoutAndFit()
     await updateRole(connection.target)
   }
 
@@ -144,6 +152,7 @@
     updateRole,
     deleteRole,
     deleteEdge,
+    layoutAndFit,
   })
 </script>
 

From 303bb94eec851cce019e9f24a65a58dea3266d73 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 14:28:43 +0100
Subject: [PATCH 213/455] Fix top nav being off-center

---
 .../builder/src/pages/builder/app/[application]/_layout.svelte | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte
index be8337b376f..8e109d3145d 100644
--- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte
@@ -228,7 +228,7 @@
   .top-nav {
     flex: 0 0 60px;
     background: var(--background);
-    padding-left: var(--spacing-xl);
+    padding: 0 var(--spacing-xl);
     display: grid;
     grid-template-columns: 1fr auto 1fr;
     flex-direction: row;
@@ -269,6 +269,7 @@
     flex-direction: row;
     justify-content: flex-end;
     align-items: center;
+    margin-right: calc(-1 * var(--spacing-xl));
   }
 
   .toprightnav :global(.avatars) {

From c51c7cd2a6112511b7d5bae72297a5234b0343d6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 15:47:05 +0100
Subject: [PATCH 214/455] Make role editor full screen

---
 .../backend/RoleEditor/Controls.svelte        | 20 ++++++------
 .../backend/RoleEditor/RoleFlow.svelte        | 31 ++++++++++---------
 2 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index e20fab29955..9c851193e81 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -8,7 +8,7 @@
   const flow = useSvelteFlow()
 </script>
 
-<div class="control top-left">
+<div class="control top-right">
   <div class="group">
     <ActionButton
       icon="Add"
@@ -36,25 +36,25 @@
     align-items: center;
     gap: 12px;
   }
-  .top-left {
+  .top-right {
     top: 20px;
-    left: 20px;
+    right: 20px;
   }
   .bottom-right {
     bottom: 20px;
     right: 20px;
   }
-  .top-left :global(.spectrum-Button),
-  .top-left :global(.spectrum-ActionButton),
-  .top-left :global(.spectrum-Icon) {
+  .top-right :global(.spectrum-Button),
+  .top-right :global(.spectrum-ActionButton),
+  .top-right :global(.spectrum-Icon) {
     color: var(--spectrum-global-color-gray-900) !important;
   }
-  .top-left :global(.spectrum-Button),
-  .top-left :global(.spectrum-ActionButton) {
+  .top-right :global(.spectrum-Button),
+  .top-right :global(.spectrum-ActionButton) {
     background: var(--spectrum-global-color-gray-200) !important;
   }
-  .top-left :global(.spectrum-Button:hover),
-  .top-left :global(.spectrum-ActionButton:hover) {
+  .top-right :global(.spectrum-Button:hover),
+  .top-right :global(.spectrum-ActionButton:hover) {
     background: var(--spectrum-global-color-gray-300) !important;
   }
   .group {
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index a7c2818da08..9e688fd4e56 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -157,10 +157,7 @@
 </script>
 
 <div class="title">
-  <div class="heading">
-    <Heading size="S">Manage roles</Heading>
-  </div>
-  <div class="description">Roles inherit permissions from each other.</div>
+  <div class="heading" />
 </div>
 <div class="flow">
   <SvelteFlow
@@ -180,22 +177,17 @@
   >
     <Background variant={BackgroundVariant.Dots} />
     <Controls />
+    <div class="title">
+      <Heading size="S">Manage roles</Heading>
+    </div>
+    <div class="footer">Roles inherit permissions from each other</div>
   </SvelteFlow>
 </div>
 
 <style>
-  .heading {
-    border-bottom: 1px solid var(--spectrum-global-color-gray-300);
-    margin-bottom: 20px;
-    padding-bottom: 12px;
-  }
-  .description {
-    color: var(--spectrum-global-color-gray-600);
-    margin-bottom: calc(20px - var(--spacing-l));
-  }
   .flow {
+    margin: -28px -40px -40px -40px;
     flex: 1 1 auto;
-    border-radius: 8px;
     overflow: hidden;
     position: relative;
     --background-color: var(--spectrum-global-color-gray-50);
@@ -204,6 +196,17 @@
     --handle-color: var(--spectrum-global-color-gray-600);
     --selected-color: var(--spectrum-global-color-blue-400);
   }
+  .title {
+    position: absolute;
+    top: 20px;
+    left: 20px;
+  }
+  .footer {
+    position: absolute;
+    left: 20px;
+    bottom: 20px;
+    color: var(--spectrum-global-color-gray-600);
+  }
 
   /* Customise svelte-flow theme */
   .flow :global(.svelte-flow) {

From aa3da2bcf2074c89a6cfdb5f8a66b03fcee32f28 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 3 Oct 2024 15:49:44 +0100
Subject: [PATCH 215/455] Fix z-index

---
 .../builder/src/components/backend/RoleEditor/Controls.svelte   | 2 +-
 .../builder/src/components/backend/RoleEditor/RoleFlow.svelte   | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
index 9c851193e81..b695a71a430 100644
--- a/packages/builder/src/components/backend/RoleEditor/Controls.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte
@@ -30,7 +30,7 @@
 <style>
   .control {
     position: absolute;
-    z-index: 999;
+    z-index: 10;
     display: flex;
     flex-direction: row;
     align-items: center;
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 9e688fd4e56..5507e5ee870 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -200,12 +200,14 @@
     position: absolute;
     top: 20px;
     left: 20px;
+    z-index: 10;
   }
   .footer {
     position: absolute;
     left: 20px;
     bottom: 20px;
     color: var(--spectrum-global-color-gray-600);
+    z-index: 10;
   }
 
   /* Customise svelte-flow theme */

From ddea2d137ac5b7f00e955b386fc753b947d02d9a Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 3 Oct 2024 16:59:59 +0100
Subject: [PATCH 216/455] don't run AI column functionality if the feature
 isn't enabled

---
 packages/server/specs/openapi.json             | 12 +++++++++---
 packages/server/specs/openapi.yaml             |  6 ++++++
 .../src/api/controllers/row/staticFormula.ts   | 18 ++++++++++++------
 .../src/api/routes/tests/datasource.spec.ts    |  6 ++++++
 .../server/src/integrations/utils/utils.ts     |  1 +
 packages/types/src/documents/app/row.ts        |  5 ++++-
 6 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json
index b21554505bf..f9f5a364ae3 100644
--- a/packages/server/specs/openapi.json
+++ b/packages/server/specs/openapi.json
@@ -833,7 +833,8 @@
                       "type": "string",
                       "enum": [
                         "static",
-                        "dynamic"
+                        "dynamic",
+                        "ai"
                       ],
                       "description": "Defines whether this is a static or dynamic formula."
                     }
@@ -857,6 +858,7 @@
                         "link",
                         "formula",
                         "auto",
+                        "ai",
                         "json",
                         "internal",
                         "barcodeqr",
@@ -1042,7 +1044,8 @@
                           "type": "string",
                           "enum": [
                             "static",
-                            "dynamic"
+                            "dynamic",
+                            "ai"
                           ],
                           "description": "Defines whether this is a static or dynamic formula."
                         }
@@ -1066,6 +1069,7 @@
                             "link",
                             "formula",
                             "auto",
+                            "ai",
                             "json",
                             "internal",
                             "barcodeqr",
@@ -1262,7 +1266,8 @@
                             "type": "string",
                             "enum": [
                               "static",
-                              "dynamic"
+                              "dynamic",
+                              "ai"
                             ],
                             "description": "Defines whether this is a static or dynamic formula."
                           }
@@ -1286,6 +1291,7 @@
                               "link",
                               "formula",
                               "auto",
+                              "ai",
                               "json",
                               "internal",
                               "barcodeqr",
diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml
index 6a2ae89c61b..d4b2df54a51 100644
--- a/packages/server/specs/openapi.yaml
+++ b/packages/server/specs/openapi.yaml
@@ -761,6 +761,7 @@ components:
                     enum:
                       - static
                       - dynamic
+                      - ai
                     description: Defines whether this is a static or dynamic formula.
               - type: object
                 properties:
@@ -779,6 +780,7 @@ components:
                       - link
                       - formula
                       - auto
+                      - ai
                       - json
                       - internal
                       - barcodeqr
@@ -929,6 +931,7 @@ components:
                         enum:
                           - static
                           - dynamic
+                          - ai
                         description: Defines whether this is a static or dynamic formula.
                   - type: object
                     properties:
@@ -947,6 +950,7 @@ components:
                           - link
                           - formula
                           - auto
+                          - ai
                           - json
                           - internal
                           - barcodeqr
@@ -1104,6 +1108,7 @@ components:
                           enum:
                             - static
                             - dynamic
+                            - ai
                           description: Defines whether this is a static or dynamic formula.
                     - type: object
                       properties:
@@ -1122,6 +1127,7 @@ components:
                             - link
                             - formula
                             - auto
+                            - ai
                             - json
                             - internal
                             - barcodeqr
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index 266d45d270d..eb1283cc9e2 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -10,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
+import * as pro from "@budibase/pro"
 
 /**
  * This function runs through a list of enriched rows, looks at the rows which
@@ -143,9 +144,12 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
-  row = await processAIColumns(table, row, {
-    contextRows: [enrichedRow],
-  })
+  const aiEnabled = await pro.features.isBudibaseAIEnabled() || await pro.features.isAICustomConfigsEnabled()
+  if (aiEnabled) {
+    row = await processAIColumns(table, row, {
+      contextRows: [enrichedRow],
+    })
+  }
 
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
@@ -153,9 +157,11 @@ export async function finaliseRow(
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
-  enrichedRow = await processAIColumns(table, row, {
-    contextRows: [enrichedRow],
-  })
+  if (aiEnabled) {
+    enrichedRow = await processAIColumns(table, row, {
+      contextRows: [enrichedRow],
+    })
+  }
 
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts
index 75452531813..768dfb32ded 100644
--- a/packages/server/src/api/routes/tests/datasource.spec.ts
+++ b/packages/server/src/api/routes/tests/datasource.spec.ts
@@ -343,6 +343,12 @@ describe("/datasources", () => {
             type: FieldType.FORMULA,
             formula: "any formula",
           },
+          [FieldType.AI]: {
+            name: "ai",
+            type: FieldType.AI,
+            operation: "PROMPT",
+            prompt: "test prompt",
+          },
           [FieldType.BARCODEQR]: {
             name: "barcodeqr",
             type: FieldType.BARCODEQR,
diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts
index 43302de36f9..73b2f637f8d 100644
--- a/packages/server/src/integrations/utils/utils.ts
+++ b/packages/server/src/integrations/utils/utils.ts
@@ -242,6 +242,7 @@ function copyExistingPropsOver(
       let shouldKeepSchema = false
       switch (existingColumnType) {
         case FieldType.FORMULA:
+        case FieldType.AI:
         case FieldType.AUTO:
         case FieldType.INTERNAL:
           shouldKeepSchema = true
diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index cb4b1d560d0..6d7e0c754f4 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -77,7 +77,10 @@ export enum FieldType {
    */
   AUTO = "auto",
   /**
-   * A complex type, called an AI column within Budibase. This type has a... TODO: fill out
+   * A complex type, called an AI column within Budibase. This type is only supported against internal tables
+   * and calculates the output based on a chosen operation (summarise text, translation etc) which passes to
+   * the configured Budibase Large Language Model to retrieve the output and write it back into the row.
+   * AI fields function in a similar fashion to static formulas, and possess many of the same characteristics.
    */
   AI = "ai",
   /**

From f2b4ef2ac8ba44130f37ddec9cf26af199962625 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Oct 2024 09:23:14 +0100
Subject: [PATCH 217/455] Lint

---
 .../src/components/backend/RoleEditor/RoleEdge.svelte     | 8 +-------
 .../src/components/backend/RoleEditor/RoleNode.svelte     | 4 +---
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
index 14b1e03efa5..84badeb6b2d 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte
@@ -1,10 +1,5 @@
 <script>
-  import {
-    getBezierPath,
-    BaseEdge,
-    EdgeLabelRenderer,
-    useSvelteFlow,
-  } from "@xyflow/svelte"
+  import { getBezierPath, BaseEdge, EdgeLabelRenderer } from "@xyflow/svelte"
   import { Icon, TooltipPosition } from "@budibase/bbui"
   import { getContext, onMount } from "svelte"
   import { roles } from "stores/builder"
@@ -19,7 +14,6 @@
   export let source
   export let target
 
-  const flow = useSvelteFlow()
   const { deleteEdge, selectedNodes } = getContext("flow")
 
   let iconHovered = false
diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
index 3dbab9537b7..32a95f4278c 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte
@@ -12,14 +12,13 @@
   import { getContext } from "svelte"
   import { roles } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { Roles } from "constants/backend"
 
   export let data
   export let id
   export let selected
   export let isConnectable
 
-  const { dragging, updateRole, deleteRole, bounds } = getContext("flow")
+  const { dragging, updateRole, deleteRole } = getContext("flow")
 
   let anchor
   let modal
@@ -31,7 +30,6 @@
   $: nameError = validateName(tempDisplayName, $roles)
   $: descriptionError = validateDescription(tempDescription)
   $: invalid = nameError || descriptionError
-  $: targetClasses = `target${!$dragging ? " hidden" : ""}`
 
   const validateName = (name, roles) => {
     if (!name?.length) {

From 3cb14d596ad64049a63ed254cbbaf764bc9199d9 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 4 Oct 2024 10:18:27 +0100
Subject: [PATCH 218/455] fixes an issue where id and revision weren't passed
 to row action automations

---
 packages/server/src/sdk/app/rowActions.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index 21c256eacbc..0fbd7f664e2 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -220,6 +220,8 @@ export async function run(tableId: any, rowActionId: any, rowId: string) {
     automation,
     {
       fields: {
+        id: row._id,
+        revision: row._rev,
         row,
         table,
       },

From 9a47db4a13dec6785a04606cffcaf66a858d08d9 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 4 Oct 2024 10:39:17 +0100
Subject: [PATCH 219/455] fix test

---
 packages/server/src/api/routes/tests/rowAction.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index 4fe248984a7..a58053d6cd2 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -698,6 +698,8 @@ describe("/rowsActions", () => {
             inputs: null,
             outputs: {
               fields: {},
+              id: rowId,
+              revision: (await config.api.row.get(tableId, rowId))._rev,
               row: await config.api.row.get(tableId, rowId),
               table: {
                 ...(await config.api.table.get(tableId)),

From b6aebd67a81c4731f8eb1e8e043ddad0acb78c8c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Oct 2024 12:55:50 +0200
Subject: [PATCH 220/455] Rename allowedViews to allowedSources

---
 .../backend/DataTable/buttons/grid/GridRowActionsButton.svelte  | 2 +-
 packages/builder/src/stores/builder/rowActions.js               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
index 3f0d6f11c5a..f61e19c19d8 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
@@ -95,7 +95,7 @@
             {#if isView}
               <span>
                 <Toggle
-                  value={action.allowedViews?.includes(viewId)}
+                  value={action.allowedSources?.includes(viewId)}
                   on:change={e => toggleAction(action, e.detail)}
                 />
               </span>
diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index a8532b0c720..21de2501b30 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -131,7 +131,7 @@ const derivedStore = derived(store, $store => {
   Object.keys($store || {}).forEach(tableId => {
     map[tableId] = $store[tableId]
     for (let action of $store[tableId]) {
-      for (let viewId of action.allowedViews || []) {
+      for (let viewId of action.allowedSources || []) {
         if (!map[viewId]) {
           map[viewId] = []
         }

From 349637733eccacf9d2d4efd5c1fdf43a01d5d67c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Oct 2024 13:21:21 +0200
Subject: [PATCH 221/455] Return allowed run from table

---
 .../data/table/[tableId]/index.svelte            | 16 +++++++++-------
 .../builder/src/stores/builder/rowActions.js     | 11 ++++++-----
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 24851c723d7..1ee96bf624a 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -63,13 +63,15 @@
   $: rowActions.refreshRowActions(id)
 
   const makeRowActionButtons = actions => {
-    return (actions || []).map(action => ({
-      text: action.name,
-      onClick: async row => {
-        await rowActions.trigger(id, action.id, row._id)
-        notifications.success("Row action triggered successfully")
-      },
-    }))
+    return (actions || [])
+      .filter(action => action.allowedSources?.includes(id))
+      .map(action => ({
+        text: action.name,
+        onClick: async row => {
+          await rowActions.trigger(id, action.id, row._id)
+          notifications.success("Row action triggered successfully")
+        },
+      }))
   }
 
   const relationshipSupport = datasource => {
diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index 21de2501b30..714f1a47a46 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -131,11 +131,12 @@ const derivedStore = derived(store, $store => {
   Object.keys($store || {}).forEach(tableId => {
     map[tableId] = $store[tableId]
     for (let action of $store[tableId]) {
-      for (let viewId of action.allowedSources || []) {
-        if (!map[viewId]) {
-          map[viewId] = []
-        }
-        map[viewId].push(action)
+      const otherSources = (action.allowedSources || []).filter(
+        sourceId => sourceId !== tableId
+      )
+      for (let source of otherSources) {
+        map[source] ??= []
+        map[source].push(action)
       }
     }
   })

From 6f62b8671df35a8e7d210ddd46d5fcc27bff6417 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 4 Oct 2024 15:54:07 +0200
Subject: [PATCH 222/455] Add comment

---
 packages/builder/src/stores/builder/rowActions.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/stores/builder/rowActions.js b/packages/builder/src/stores/builder/rowActions.js
index 714f1a47a46..b1f4e7067f3 100644
--- a/packages/builder/src/stores/builder/rowActions.js
+++ b/packages/builder/src/stores/builder/rowActions.js
@@ -129,6 +129,7 @@ const derivedStore = derived(store, $store => {
 
   // Generate an entry for every view as well
   Object.keys($store || {}).forEach(tableId => {
+    // We need to have all the actions for the table in order to be displayed in the crud section
     map[tableId] = $store[tableId]
     for (let action of $store[tableId]) {
       const otherSources = (action.allowedSources || []).filter(

From d5324388d6687b1672b0b5c61be3ad9fdf6cfc49 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 4 Oct 2024 17:20:30 +0100
Subject: [PATCH 223/455] allow regeneration of AI columns from context menu

---
 .../DataTable/modals/CreateEditRow.svelte     |  3 ++-
 .../grid/overlays/MenuOverlay.svelte          | 16 +++++++++++++
 .../src/components/grid/stores/rows.js        | 24 +++++++++++++++++++
 packages/shared-core/src/constants/ai.ts      |  2 +-
 4 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte
index 277fe63d6af..eb99e994d14 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte
@@ -7,6 +7,7 @@
   import { FIELDS } from "constants/backend"
 
   const FORMULA_TYPE = FIELDS.FORMULA.type
+  const AI_TYPE = FIELDS.AI.type
 
   export let row = {}
 
@@ -60,7 +61,7 @@
     }}
   >
     {#each tableSchema as [key, meta]}
-      {#if !meta.autocolumn && meta.type !== FORMULA_TYPE}
+      {#if !meta.autocolumn && meta.type !== FORMULA_TYPE && meta.type !== AI_TYPE}
         <div>
           <RowFieldControl error={errors[key]} {meta} bind:value={row[key]} />
         </div>
diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
index ec9ea981e51..6ae4dc2fb09 100644
--- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
@@ -53,6 +53,13 @@
     await Helpers.copyToClipboard(value)
     $notifications.success("Copied to clipboard")
   }
+
+  const generateAIColumns = async () => {
+    await rows.actions.saveRow({ rowId: $focusedRowId })
+    await rows.actions.refreshData()
+    menu.actions.close()
+    $notifications.success("Generated AI columns")
+  }
 </script>
 
 <div bind:this={anchor} {style} class="menu-anchor" />
@@ -161,6 +168,15 @@
           >
             Delete row
           </MenuItem>
+            <MenuItem
+              icon="MagicWand"
+              disabled={isNewRow ||
+                !$focusedRow?._rev ||
+                !$hasBudibaseIdentifiers}
+              on:click={generateAIColumns}
+            >
+              Generate AI Columns
+            </MenuItem>
         {/if}
       </Menu>
     </GridPopover>
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 4cfaa1cfd59..94fbdba5fac 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -465,6 +465,29 @@ export const createActions = context => {
     return true
   }
 
+  const saveRow = async ({ rowId }) => {
+    const $rowLookupMap = get(rowLookupMap)
+    const row = $rowLookupMap[rowId]
+    if (row == null) {
+      return
+    }
+    let savedRow
+
+    // Save row
+    try {
+      const newRow = cleanRow(row)
+      savedRow = await datasource.actions.updateRow(newRow)
+
+      if (savedRow?.id) {
+        // Handle users table edge case
+        await refreshRow(savedRow.id)
+      }
+    } catch (error) {
+      handleValidationError(rowId, error)
+    }
+    return savedRow
+  }
+
   // Saves any pending changes to a row, as well as any additional changes
   // specified
   const applyRowChanges = async ({
@@ -685,6 +708,7 @@ export const createActions = context => {
       ...rows,
       actions: {
         addRow,
+        saveRow,
         duplicateRow,
         bulkDuplicate,
         updateValue,
diff --git a/packages/shared-core/src/constants/ai.ts b/packages/shared-core/src/constants/ai.ts
index e3f035e72bc..e9ad0344695 100644
--- a/packages/shared-core/src/constants/ai.ts
+++ b/packages/shared-core/src/constants/ai.ts
@@ -4,7 +4,7 @@ import {
   OperationFieldTypeEnum,
 } from "@budibase/types"
 
-const AIOperations = {
+export const AIOperations = {
   SUMMARISE_TEXT: {
     label: "Summarise Text",
     value: "SUMMARISE_TEXT",

From b9a106954f43e33c977ae285affda0d07bc022e3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 7 Oct 2024 10:45:36 +0100
Subject: [PATCH 224/455] Improve RBAC edutor UI for small flows

---
 .../components/backend/RoleEditor/BracketEdge.svelte  |  2 +-
 .../src/components/backend/RoleEditor/constants.js    |  1 +
 .../src/components/backend/RoleEditor/utils.js        | 11 ++++++++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte b/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
index 37aa772a22d..182b56f1223 100644
--- a/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte
@@ -19,7 +19,7 @@
 
   const getCurlyBracePath = (x1, y1, x2, y2) => {
     const w = 2 // Thickness
-    const q = 2 // Intensity
+    const q = 1 // Intensity
     const i = 28 // Inner radius strenth (lower is stronger)
     const j = 32 // Outer radius strength (higher is stronger)
 
diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index aa4b1f26f37..dacd37d745e 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -5,3 +5,4 @@ export const NodeHeight = GridResolution * 3
 export const NodeWidth = GridResolution * 12
 export const NodeHSpacing = GridResolution * 6
 export const NodeVSpacing = GridResolution * 2
+export const MinHeight = GridResolution * 10
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index 9d23d46a753..1cbe6a17af7 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -5,6 +5,7 @@ import {
   GridResolution,
   NodeHSpacing,
   NodeVSpacing,
+  MinHeight,
 } from "./constants"
 import { getNodesBounds, Position } from "@xyflow/svelte"
 import { Roles } from "constants/backend"
@@ -24,7 +25,15 @@ export const getBounds = nodes => {
       height: 10 * GridResolution,
     }
   }
-  return getNodesBounds(interactiveNodes)
+  let bounds = getNodesBounds(interactiveNodes)
+
+  // Enforce a min size
+  if (bounds.height < MinHeight) {
+    const diff = MinHeight - bounds.height
+    bounds.height = MinHeight
+    bounds.y -= diff / 2
+  }
+  return bounds
 }
 
 // Gets the position of the basic role

From 3e86e0265664001c723598c8955e7a13578e6b93 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 8 Oct 2024 14:27:44 +0100
Subject: [PATCH 225/455] Add support for creating and displaying calculation
 views

---
 .../data/table/[tableId]/[viewId]/index.svelte    |  3 ++-
 .../[tableId]/_components/CreateViewButton.svelte |  5 ++---
 .../[tableId]/_components/DeleteViewModal.svelte  |  1 +
 .../src/components/grid/lib/renderers.js          |  3 +++
 .../src/components/grid/stores/columns.js         |  1 +
 .../src/components/grid/stores/config.js          | 15 ++++++++++++---
 packages/frontend-core/src/utils/schema.js        |  5 +++--
 7 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index b48184514de..985c81add61 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -14,11 +14,12 @@
   import GridScreensButton from "components/backend/DataTable/buttons/grid/GridScreensButton.svelte"
   import GridRowActionsButton from "components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte"
   import GridViewCalculationButton from "components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte"
+  import { ViewV2Type } from "@budibase/types"
 
   let generateButton
 
   $: view = $viewsV2.selected
-  $: calculation = view?.uiMetadata?.calculation
+  $: calculation = view?.type === ViewV2Type.CALCULATION
   $: id = view?.id
   $: datasource = {
     type: "viewV2",
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index c420bb508e0..253c386b9ef 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -3,6 +3,7 @@
   import { Input, notifications, Button, Icon, ListItem } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { viewsV2 } from "stores/builder"
+  import { CalculationType, ViewV2Type } from "@budibase/types"
 
   export let table
   export let firstView = false
@@ -40,9 +41,7 @@
         tableId: table._id,
         schema: enrichSchema(table.schema),
         primaryDisplay: calculation ? undefined : table.primaryDisplay,
-        uiMetadata: {
-          calculation,
-        },
+        type: calculation ? ViewV2Type.CALCULATION : undefined,
       })
       notifications.success(`View ${name} created`)
       $goto(`./${newView.id}`)
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
index 774a2f987a4..0b5f18b70bd 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
@@ -20,6 +20,7 @@
       }
       notifications.success("View deleted")
     } catch (error) {
+      console.error(error)
       notifications.error("Error deleting view")
     }
   }
diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index 6af21e65f07..feed991da21 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -35,6 +35,9 @@ const TypeComponentMap = {
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
 }
 export const getCellRenderer = column => {
+  if (column.calculation) {
+    return NumberCell
+  }
   return (
     TypeComponentMap[column?.schema?.cellRenderType] ||
     TypeComponentMap[column?.schema?.type] ||
diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 4823f2159c3..36fb6659a9b 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -160,6 +160,7 @@ export const initialise = context => {
             order: fieldSchema.order ?? oldColumn?.order,
             conditions: fieldSchema.conditions,
             related: fieldSchema.related,
+            calculation: fieldSchema.calculationType != null,
           }
           // Override a few properties for primary display
           if (field === primaryDisplay) {
diff --git a/packages/frontend-core/src/components/grid/stores/config.js b/packages/frontend-core/src/components/grid/stores/config.js
index fc0435f92d6..d0cda1e114d 100644
--- a/packages/frontend-core/src/components/grid/stores/config.js
+++ b/packages/frontend-core/src/components/grid/stores/config.js
@@ -1,5 +1,6 @@
 import { derivedMemo } from "../../../utils"
 import { derived } from "svelte/store"
+import { ViewV2Type } from "@budibase/types"
 
 export const createStores = context => {
   const { props } = context
@@ -30,12 +31,12 @@ export const createStores = context => {
 }
 
 export const deriveStores = context => {
-  const { props, hasNonAutoColumn } = context
+  const { props, definition, hasNonAutoColumn } = context
 
   // Derive features
   const config = derived(
-    [props, hasNonAutoColumn],
-    ([$props, $hasNonAutoColumn]) => {
+    [props, definition, hasNonAutoColumn],
+    ([$props, $definition, $hasNonAutoColumn]) => {
       let config = { ...$props }
       const type = $props.datasource?.type
 
@@ -59,6 +60,14 @@ export const deriveStores = context => {
         config.canEditColumns = false
       }
 
+      // Disable features for calculation views
+      if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
+        config.canAddRows = false
+        config.canEditRows = false
+        config.canDeleteRows = false
+        config.canExpandRows = false
+      }
+
       return config
     }
   )
diff --git a/packages/frontend-core/src/utils/schema.js b/packages/frontend-core/src/utils/schema.js
index 7185b30d109..b98ae1c375c 100644
--- a/packages/frontend-core/src/utils/schema.js
+++ b/packages/frontend-core/src/utils/schema.js
@@ -2,14 +2,15 @@ import { helpers } from "@budibase/shared-core"
 import { TypeIconMap } from "../constants"
 
 export const getColumnIcon = column => {
+  if (column.calculation) {
+    return "Calculator"
+  }
   if (column.schema.autocolumn) {
     return "MagicWand"
   }
-
   if (helpers.schema.isDeprecatedSingleUserColumn(column.schema)) {
     return "User"
   }
-
   const { type, subtype } = column.schema
   const result =
     typeof TypeIconMap[type] === "object" && subtype

From c1333c66dbc3cb0e7da1d0ef567deff926c2bc6d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 8 Oct 2024 14:30:10 +0100
Subject: [PATCH 226/455] Fix error when deleting views

---
 .../data/table/[tableId]/_components/DeleteViewModal.svelte   | 1 +
 packages/builder/src/stores/builder/viewsV2.js                | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
index 774a2f987a4..0b5f18b70bd 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte
@@ -20,6 +20,7 @@
       }
       notifications.success("View deleted")
     } catch (error) {
+      console.error(error)
       notifications.error("Error deleting view")
     }
   }
diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index 0c7a03414ed..b1db8165e2c 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -5,7 +5,7 @@ import { dataFilters } from "@budibase/shared-core"
 
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
-  if (view.query) {
+  if (view?.query) {
     view.queryUI = view.query
     view.query = dataFilters.buildQuery(view.query)
   }
@@ -13,7 +13,7 @@ function convertToSearchFilters(view) {
 }
 
 function convertToSearchFilterGroup(view) {
-  if (view.queryUI) {
+  if (view?.queryUI) {
     view.query = view.queryUI
     delete view.queryUI
   }

From 6e0b5341b144f087643ba2c5accd9054259344b4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 8 Oct 2024 15:54:42 +0100
Subject: [PATCH 227/455] Handle lack of IDs from calculation views

---
 ....timestamp-1728399274731-fb0dcfaf9cacc.mjs | 146 ++++++++++++++++++
 .../src/components/grid/stores/config.js      |  20 +--
 .../src/components/grid/stores/datasource.js  |  20 ++-
 .../frontend-core/src/fetch/ViewV2Fetch.js    |  12 --
 4 files changed, 168 insertions(+), 30 deletions(-)
 create mode 100644 packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs

diff --git a/packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs b/packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs
new file mode 100644
index 00000000000..8a28a7ca60c
--- /dev/null
+++ b/packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs
@@ -0,0 +1,146 @@
+// vite.config.mjs
+import { svelte } from "file:///Users/aptk/dev/budibase/node_modules/@sveltejs/vite-plugin-svelte/dist/index.js";
+import replace from "file:///Users/aptk/dev/budibase/node_modules/@rollup/plugin-replace/dist/es/index.js";
+import { defineConfig, loadEnv } from "file:///Users/aptk/dev/budibase/node_modules/vite/dist/node/index.js";
+import { viteStaticCopy } from "file:///Users/aptk/dev/budibase/node_modules/vite-plugin-static-copy/dist/index.js";
+import path from "path";
+var ignoredWarnings = [
+  "unused-export-let",
+  "css-unused-selector",
+  "module-script-reactive-declaration",
+  "a11y-no-onchange",
+  "a11y-click-events-have-key-events"
+];
+var copyFonts = (dest) => viteStaticCopy({
+  targets: [
+    {
+      src: "../../node_modules/@fontsource/source-sans-pro",
+      dest
+    },
+    {
+      src: "../../node_modules/remixicon/fonts/*",
+      dest
+    }
+  ]
+});
+var vite_config_default = defineConfig(({ mode }) => {
+  const isProduction = mode === "production";
+  const env = loadEnv(mode, process.cwd());
+  const devOnlyPlugins = [
+    // Copy fonts to an additional path so that svelte's automatic
+    // prefixing of the base URL path can still resolve assets
+    copyFonts("builder/fonts")
+  ];
+  return {
+    test: {
+      setupFiles: ["./vitest.setup.js"],
+      globals: true,
+      environment: "jsdom"
+    },
+    server: {
+      fs: {
+        strict: false
+      },
+      hmr: {
+        protocol: env.VITE_HMR_PROTOCOL || "ws",
+        clientPort: env.VITE_HMR_CLIENT_PORT || 3e3,
+        path: env.VITE_HMR_PATH || "/"
+      },
+      port: 3e3
+    },
+    base: "/builder/",
+    build: {
+      minify: isProduction,
+      outDir: "../server/builder",
+      sourcemap: !isProduction
+    },
+    plugins: [
+      svelte({
+        hot: !isProduction,
+        emitCss: true,
+        onwarn: (warning, handler) => {
+          if (!ignoredWarnings.includes(warning.code)) {
+            handler(warning);
+          }
+        }
+      }),
+      replace({
+        preventAssignment: true,
+        "process.env.NODE_ENV": JSON.stringify(
+          isProduction ? "production" : "development"
+        ),
+        "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN)
+      }),
+      copyFonts("fonts"),
+      ...isProduction ? [] : devOnlyPlugins
+    ],
+    optimizeDeps: {
+      exclude: ["@roxi/routify", "fsevents"]
+    },
+    resolve: {
+      dedupe: ["@roxi/routify"],
+      alias: [
+        {
+          find: "assets",
+          replacement: path.resolve("./assets")
+        },
+        {
+          find: "components",
+          replacement: path.resolve("./src/components")
+        },
+        {
+          find: "pages",
+          replacement: path.resolve("./src/pages")
+        },
+        {
+          find: "templates",
+          replacement: path.resolve("./src/templates")
+        },
+        {
+          find: "stores",
+          replacement: path.resolve("./src/stores")
+        },
+        {
+          find: "dataBinding",
+          replacement: path.resolve("./src/dataBinding.js")
+        },
+        {
+          find: "api",
+          replacement: path.resolve("./src/api.js")
+        },
+        {
+          find: "constants",
+          replacement: path.resolve("./src/constants")
+        },
+        {
+          find: "analytics",
+          replacement: path.resolve("./src/analytics")
+        },
+        {
+          find: "actions",
+          replacement: path.resolve("./src/actions")
+        },
+        {
+          find: "helpers",
+          replacement: path.resolve("./src/helpers")
+        },
+        {
+          find: "@budibase/types",
+          replacement: path.resolve("../types/src")
+        },
+        {
+          find: "@budibase/shared-core",
+          replacement: path.resolve("../shared-core/src")
+        },
+        {
+          find: "@budibase/bbui",
+          replacement: path.resolve("../bbui/src")
+        }
+      ]
+    }
+  };
+});
+export {
+  vite_config_default as default
+};
+//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsidml0ZS5jb25maWcubWpzIl0sCiAgInNvdXJjZXNDb250ZW50IjogWyJjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfZGlybmFtZSA9IFwiL1VzZXJzL2FwdGsvZGV2L2J1ZGliYXNlL3BhY2thZ2VzL2J1aWxkZXJcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfZmlsZW5hbWUgPSBcIi9Vc2Vycy9hcHRrL2Rldi9idWRpYmFzZS9wYWNrYWdlcy9idWlsZGVyL3ZpdGUuY29uZmlnLm1qc1wiO2NvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9pbXBvcnRfbWV0YV91cmwgPSBcImZpbGU6Ly8vVXNlcnMvYXB0ay9kZXYvYnVkaWJhc2UvcGFja2FnZXMvYnVpbGRlci92aXRlLmNvbmZpZy5tanNcIjtpbXBvcnQgeyBzdmVsdGUgfSBmcm9tIFwiQHN2ZWx0ZWpzL3ZpdGUtcGx1Z2luLXN2ZWx0ZVwiXG5pbXBvcnQgcmVwbGFjZSBmcm9tIFwiQHJvbGx1cC9wbHVnaW4tcmVwbGFjZVwiXG5pbXBvcnQgeyBkZWZpbmVDb25maWcsIGxvYWRFbnYgfSBmcm9tIFwidml0ZVwiXG5pbXBvcnQgeyB2aXRlU3RhdGljQ29weSB9IGZyb20gXCJ2aXRlLXBsdWdpbi1zdGF0aWMtY29weVwiXG5pbXBvcnQgcGF0aCBmcm9tIFwicGF0aFwiXG5cbmNvbnN0IGlnbm9yZWRXYXJuaW5ncyA9IFtcbiAgXCJ1bnVzZWQtZXhwb3J0LWxldFwiLFxuICBcImNzcy11bnVzZWQtc2VsZWN0b3JcIixcbiAgXCJtb2R1bGUtc2NyaXB0LXJlYWN0aXZlLWRlY2xhcmF0aW9uXCIsXG4gIFwiYTExeS1uby1vbmNoYW5nZVwiLFxuICBcImExMXktY2xpY2stZXZlbnRzLWhhdmUta2V5LWV2ZW50c1wiLFxuXVxuXG5jb25zdCBjb3B5Rm9udHMgPSBkZXN0ID0+XG4gIHZpdGVTdGF0aWNDb3B5KHtcbiAgICB0YXJnZXRzOiBbXG4gICAgICB7XG4gICAgICAgIHNyYzogXCIuLi8uLi9ub2RlX21vZHVsZXMvQGZvbnRzb3VyY2Uvc291cmNlLXNhbnMtcHJvXCIsXG4gICAgICAgIGRlc3QsXG4gICAgICB9LFxuICAgICAge1xuICAgICAgICBzcmM6IFwiLi4vLi4vbm9kZV9tb2R1bGVzL3JlbWl4aWNvbi9mb250cy8qXCIsXG4gICAgICAgIGRlc3QsXG4gICAgICB9LFxuICAgIF0sXG4gIH0pXG5cbmV4cG9ydCBkZWZhdWx0IGRlZmluZUNvbmZpZygoeyBtb2RlIH0pID0+IHtcbiAgY29uc3QgaXNQcm9kdWN0aW9uID0gbW9kZSA9PT0gXCJwcm9kdWN0aW9uXCJcbiAgY29uc3QgZW52ID0gbG9hZEVudihtb2RlLCBwcm9jZXNzLmN3ZCgpKVxuXG4gIC8vIFBsdWdpbnMgdG8gb25seSBydW4gaW4gZGV2XG4gIGNvbnN0IGRldk9ubHlQbHVnaW5zID0gW1xuICAgIC8vIENvcHkgZm9udHMgdG8gYW4gYWRkaXRpb25hbCBwYXRoIHNvIHRoYXQgc3ZlbHRlJ3MgYXV0b21hdGljXG4gICAgLy8gcHJlZml4aW5nIG9mIHRoZSBiYXNlIFVSTCBwYXRoIGNhbiBzdGlsbCByZXNvbHZlIGFzc2V0c1xuICAgIGNvcHlGb250cyhcImJ1aWxkZXIvZm9udHNcIiksXG5dXG5cbiAgcmV0dXJuIHtcbiAgICB0ZXN0OiB7XG4gICAgICBzZXR1cEZpbGVzOiBbXCIuL3ZpdGVzdC5zZXR1cC5qc1wiXSxcbiAgICAgIGdsb2JhbHM6IHRydWUsXG4gICAgICBlbnZpcm9ubWVudDogXCJqc2RvbVwiLFxuICAgIH0sXG4gICAgc2VydmVyOiB7XG4gICAgICBmczoge1xuICAgICAgICBzdHJpY3Q6IGZhbHNlLFxuICAgICAgfSxcbiAgICAgIGhtcjoge1xuICAgICAgICBwcm90b2NvbDogZW52LlZJVEVfSE1SX1BST1RPQ09MIHx8IFwid3NcIixcbiAgICAgICAgY2xpZW50UG9ydDogZW52LlZJVEVfSE1SX0NMSUVOVF9QT1JUIHx8IDMwMDAsXG4gICAgICAgIHBhdGg6IGVudi5WSVRFX0hNUl9QQVRIIHx8IFwiL1wiLFxuICAgICAgfSxcbiAgICAgIHBvcnQ6IDMwMDAsXG4gICAgfSxcbiAgICBiYXNlOiBcIi9idWlsZGVyL1wiLFxuICAgIGJ1aWxkOiB7XG4gICAgICBtaW5pZnk6IGlzUHJvZHVjdGlvbixcbiAgICAgIG91dERpcjogXCIuLi9zZXJ2ZXIvYnVpbGRlclwiLFxuICAgICAgc291cmNlbWFwOiAhaXNQcm9kdWN0aW9uLFxuICAgIH0sXG4gICAgcGx1Z2luczogW1xuICAgICAgc3ZlbHRlKHtcbiAgICAgICAgaG90OiAhaXNQcm9kdWN0aW9uLFxuICAgICAgICBlbWl0Q3NzOiB0cnVlLFxuICAgICAgICBvbndhcm46ICh3YXJuaW5nLCBoYW5kbGVyKSA9PiB7XG4gICAgICAgICAgLy8gSWdub3JlIHNvbWUgd2FybmluZ3NcbiAgICAgICAgICBpZiAoIWlnbm9yZWRXYXJuaW5ncy5pbmNsdWRlcyh3YXJuaW5nLmNvZGUpKSB7XG4gICAgICAgICAgICBoYW5kbGVyKHdhcm5pbmcpXG4gICAgICAgICAgfVxuICAgICAgICB9LFxuICAgICAgfSksXG4gICAgICByZXBsYWNlKHtcbiAgICAgICAgcHJldmVudEFzc2lnbm1lbnQ6IHRydWUsXG4gICAgICAgIFwicHJvY2Vzcy5lbnYuTk9ERV9FTlZcIjogSlNPTi5zdHJpbmdpZnkoXG4gICAgICAgICAgaXNQcm9kdWN0aW9uID8gXCJwcm9kdWN0aW9uXCIgOiBcImRldmVsb3BtZW50XCJcbiAgICAgICAgKSxcbiAgICAgICAgXCJwcm9jZXNzLmVudi5QT1NUSE9HX1RPS0VOXCI6IEpTT04uc3RyaW5naWZ5KHByb2Nlc3MuZW52LlBPU1RIT0dfVE9LRU4pLFxuICAgICAgfSksXG4gICAgICBjb3B5Rm9udHMoXCJmb250c1wiKSxcbiAgICAgIC4uLihpc1Byb2R1Y3Rpb24gPyBbXSA6IGRldk9ubHlQbHVnaW5zKSxcbiAgICBdLFxuICAgIG9wdGltaXplRGVwczoge1xuICAgICAgZXhjbHVkZTogW1wiQHJveGkvcm91dGlmeVwiLCBcImZzZXZlbnRzXCJdLFxuICAgIH0sXG4gICAgcmVzb2x2ZToge1xuICAgICAgZGVkdXBlOiBbXCJAcm94aS9yb3V0aWZ5XCJdLFxuICAgICAgYWxpYXM6IFtcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYXNzZXRzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vYXNzZXRzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJjb21wb25lbnRzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2NvbXBvbmVudHNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcInBhZ2VzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL3BhZ2VzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJ0ZW1wbGF0ZXNcIixcbiAgICAgICAgICByZXBsYWNlbWVudDogcGF0aC5yZXNvbHZlKFwiLi9zcmMvdGVtcGxhdGVzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJzdG9yZXNcIixcbiAgICAgICAgICByZXBsYWNlbWVudDogcGF0aC5yZXNvbHZlKFwiLi9zcmMvc3RvcmVzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJkYXRhQmluZGluZ1wiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuL3NyYy9kYXRhQmluZGluZy5qc1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYXBpXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2FwaS5qc1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiY29uc3RhbnRzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2NvbnN0YW50c1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYW5hbHl0aWNzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2FuYWx5dGljc1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYWN0aW9uc1wiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuL3NyYy9hY3Rpb25zXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJoZWxwZXJzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2hlbHBlcnNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcIkBidWRpYmFzZS90eXBlc1wiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuLi90eXBlcy9zcmNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcIkBidWRpYmFzZS9zaGFyZWQtY29yZVwiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuLi9zaGFyZWQtY29yZS9zcmNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcIkBidWRpYmFzZS9iYnVpXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4uL2JidWkvc3JjXCIpLFxuICAgICAgICB9LFxuICAgICAgXSxcbiAgICB9LFxuICB9XG59KVxuIl0sCiAgIm1hcHBpbmdzIjogIjtBQUErUyxTQUFTLGNBQWM7QUFDdFUsT0FBTyxhQUFhO0FBQ3BCLFNBQVMsY0FBYyxlQUFlO0FBQ3RDLFNBQVMsc0JBQXNCO0FBQy9CLE9BQU8sVUFBVTtBQUVqQixJQUFNLGtCQUFrQjtBQUFBLEVBQ3RCO0FBQUEsRUFDQTtBQUFBLEVBQ0E7QUFBQSxFQUNBO0FBQUEsRUFDQTtBQUNGO0FBRUEsSUFBTSxZQUFZLFVBQ2hCLGVBQWU7QUFBQSxFQUNiLFNBQVM7QUFBQSxJQUNQO0FBQUEsTUFDRSxLQUFLO0FBQUEsTUFDTDtBQUFBLElBQ0Y7QUFBQSxJQUNBO0FBQUEsTUFDRSxLQUFLO0FBQUEsTUFDTDtBQUFBLElBQ0Y7QUFBQSxFQUNGO0FBQ0YsQ0FBQztBQUVILElBQU8sc0JBQVEsYUFBYSxDQUFDLEVBQUUsS0FBSyxNQUFNO0FBQ3hDLFFBQU0sZUFBZSxTQUFTO0FBQzlCLFFBQU0sTUFBTSxRQUFRLE1BQU0sUUFBUSxJQUFJLENBQUM7QUFHdkMsUUFBTSxpQkFBaUI7QUFBQTtBQUFBO0FBQUEsSUFHckIsVUFBVSxlQUFlO0FBQUEsRUFDN0I7QUFFRSxTQUFPO0FBQUEsSUFDTCxNQUFNO0FBQUEsTUFDSixZQUFZLENBQUMsbUJBQW1CO0FBQUEsTUFDaEMsU0FBUztBQUFBLE1BQ1QsYUFBYTtBQUFBLElBQ2Y7QUFBQSxJQUNBLFFBQVE7QUFBQSxNQUNOLElBQUk7QUFBQSxRQUNGLFFBQVE7QUFBQSxNQUNWO0FBQUEsTUFDQSxLQUFLO0FBQUEsUUFDSCxVQUFVLElBQUkscUJBQXFCO0FBQUEsUUFDbkMsWUFBWSxJQUFJLHdCQUF3QjtBQUFBLFFBQ3hDLE1BQU0sSUFBSSxpQkFBaUI7QUFBQSxNQUM3QjtBQUFBLE1BQ0EsTUFBTTtBQUFBLElBQ1I7QUFBQSxJQUNBLE1BQU07QUFBQSxJQUNOLE9BQU87QUFBQSxNQUNMLFFBQVE7QUFBQSxNQUNSLFFBQVE7QUFBQSxNQUNSLFdBQVcsQ0FBQztBQUFBLElBQ2Q7QUFBQSxJQUNBLFNBQVM7QUFBQSxNQUNQLE9BQU87QUFBQSxRQUNMLEtBQUssQ0FBQztBQUFBLFFBQ04sU0FBUztBQUFBLFFBQ1QsUUFBUSxDQUFDLFNBQVMsWUFBWTtBQUU1QixjQUFJLENBQUMsZ0JBQWdCLFNBQVMsUUFBUSxJQUFJLEdBQUc7QUFDM0Msb0JBQVEsT0FBTztBQUFBLFVBQ2pCO0FBQUEsUUFDRjtBQUFBLE1BQ0YsQ0FBQztBQUFBLE1BQ0QsUUFBUTtBQUFBLFFBQ04sbUJBQW1CO0FBQUEsUUFDbkIsd0JBQXdCLEtBQUs7QUFBQSxVQUMzQixlQUFlLGVBQWU7QUFBQSxRQUNoQztBQUFBLFFBQ0EsNkJBQTZCLEtBQUssVUFBVSxRQUFRLElBQUksYUFBYTtBQUFBLE1BQ3ZFLENBQUM7QUFBQSxNQUNELFVBQVUsT0FBTztBQUFBLE1BQ2pCLEdBQUksZUFBZSxDQUFDLElBQUk7QUFBQSxJQUMxQjtBQUFBLElBQ0EsY0FBYztBQUFBLE1BQ1osU0FBUyxDQUFDLGlCQUFpQixVQUFVO0FBQUEsSUFDdkM7QUFBQSxJQUNBLFNBQVM7QUFBQSxNQUNQLFFBQVEsQ0FBQyxlQUFlO0FBQUEsTUFDeEIsT0FBTztBQUFBLFFBQ0w7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLFVBQVU7QUFBQSxRQUN0QztBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGtCQUFrQjtBQUFBLFFBQzlDO0FBQUEsUUFDQTtBQUFBLFVBQ0UsTUFBTTtBQUFBLFVBQ04sYUFBYSxLQUFLLFFBQVEsYUFBYTtBQUFBLFFBQ3pDO0FBQUEsUUFDQTtBQUFBLFVBQ0UsTUFBTTtBQUFBLFVBQ04sYUFBYSxLQUFLLFFBQVEsaUJBQWlCO0FBQUEsUUFDN0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxjQUFjO0FBQUEsUUFDMUM7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxzQkFBc0I7QUFBQSxRQUNsRDtBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGNBQWM7QUFBQSxRQUMxQztBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGlCQUFpQjtBQUFBLFFBQzdDO0FBQUEsUUFDQTtBQUFBLFVBQ0UsTUFBTTtBQUFBLFVBQ04sYUFBYSxLQUFLLFFBQVEsaUJBQWlCO0FBQUEsUUFDN0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxlQUFlO0FBQUEsUUFDM0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxlQUFlO0FBQUEsUUFDM0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxjQUFjO0FBQUEsUUFDMUM7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxvQkFBb0I7QUFBQSxRQUNoRDtBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGFBQWE7QUFBQSxRQUN6QztBQUFBLE1BQ0Y7QUFBQSxJQUNGO0FBQUEsRUFDRjtBQUNGLENBQUM7IiwKICAibmFtZXMiOiBbXQp9Cg==
diff --git a/packages/frontend-core/src/components/grid/stores/config.js b/packages/frontend-core/src/components/grid/stores/config.js
index d0cda1e114d..d4ef44545b6 100644
--- a/packages/frontend-core/src/components/grid/stores/config.js
+++ b/packages/frontend-core/src/components/grid/stores/config.js
@@ -43,6 +43,14 @@ export const deriveStores = context => {
       // Disable some features if we're editing a view
       if (type === "viewV2") {
         config.canEditColumns = false
+
+        // Disable features for calculation views
+        if ($definition?.type === ViewV2Type.CALCULATION) {
+          config.canAddRows = false
+          config.canEditRows = false
+          config.canDeleteRows = false
+          config.canExpandRows = false
+        }
       }
 
       // Disable adding rows if we don't have any valid columns
@@ -60,19 +68,7 @@ export const deriveStores = context => {
         config.canEditColumns = false
       }
 
-      // Disable features for calculation views
-      if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
-        config.canAddRows = false
-        config.canEditRows = false
-        config.canDeleteRows = false
-        config.canExpandRows = false
-      }
-
       return config
     }
   )
-
-  return {
-    config,
-  }
 }
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 869cd567304..6aa607f7edc 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -2,6 +2,7 @@ import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
 import { cloneDeep } from "lodash"
+import { ViewV2Type } from "@budibase/types"
 
 export const createStores = () => {
   const definition = memo(null)
@@ -81,13 +82,20 @@ export const deriveStores = context => {
     }
   )
 
-  const hasBudibaseIdentifiers = derived(datasource, $datasource => {
-    let type = $datasource?.type
-    if (type === "provider") {
-      type = $datasource.value?.datasource?.type
+  const hasBudibaseIdentifiers = derived(
+    [datasource, definition],
+    ([$datasource, $definition]) => {
+      let type = $datasource?.type
+      if (type === "provider") {
+        type = $datasource.value?.datasource?.type
+      }
+      // Handle calculation views
+      if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) {
+        return false
+      }
+      return ["table", "viewV2", "link"].includes(type)
     }
-    return ["table", "viewV2", "link"].includes(type)
-  })
+  )
 
   return {
     schema,
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 191cb87b54c..40135746df3 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -39,18 +39,6 @@ export default class ViewV2Fetch extends DataFetch {
       this.options
     const { cursor, query, definition } = get(this.store)
 
-    // If this is a calculation view and there are no schema fields then do nothing
-    console.log(definition)
-    if (
-      definition.calculation &&
-      !Object.keys(definition.schema || {}).length
-    ) {
-      return {
-        rows: [],
-        hasNextPage: false,
-      }
-    }
-
     // If sort/filter params are not defined, update options to store the
     // params built in to this view. This ensures that we can accurately
     // compare old and new params and skip a redundant API call.

From d35fce699af5f730451859856d0a2f40ab016271 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 8 Oct 2024 15:59:21 +0100
Subject: [PATCH 228/455] Fix config store

---
 packages/frontend-core/src/components/grid/stores/config.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/frontend-core/src/components/grid/stores/config.js b/packages/frontend-core/src/components/grid/stores/config.js
index d4ef44545b6..4a603706902 100644
--- a/packages/frontend-core/src/components/grid/stores/config.js
+++ b/packages/frontend-core/src/components/grid/stores/config.js
@@ -71,4 +71,8 @@ export const deriveStores = context => {
       return config
     }
   )
+
+  return {
+    config,
+  }
 }

From 608331018ae016246e5ef36addab3c3b93c52ff3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Oct 2024 08:10:02 +0100
Subject: [PATCH 229/455] Add progress on calculation editor

---
 .../grid/GridViewCalculationButton.svelte     | 222 +++++++++++++++++-
 1 file changed, 212 insertions(+), 10 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 33931cdd3e2..96a7d0ef8ce 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -1,29 +1,161 @@
 <script>
-  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
-  import { CalculationType } from "@budibase/types"
-  import { API } from "api"
-  import { getContext, createEventDispatcher } from "svelte"
+  import {
+    ActionButton,
+    Modal,
+    ModalContent,
+    Select,
+    Icon,
+  } from "@budibase/bbui"
+  import { CalculationType, FieldType } from "@budibase/types"
+  import { getContext } from "svelte"
 
-  const { definition } = getContext("grid")
-  const dispatch = createEventDispatcher()
+  const { definition, datasource, rows } = getContext("grid")
+  const calculationTypeOptions = [
+    {
+      label: "Average (mean)",
+      value: CalculationType.AVG,
+    },
+    {
+      label: "Sum",
+      value: CalculationType.SUM,
+    },
+    {
+      label: "Minimum",
+      value: CalculationType.MIN,
+    },
+    {
+      label: "Maximum",
+      value: CalculationType.MAX,
+    },
+    {
+      label: "Count",
+      value: CalculationType.COUNT,
+    },
+  ]
 
   let modal
+  let calculations = []
+  let groupings = []
+
+  $: schema = $definition?.schema || {}
+
+  const open = () => {
+    calculations = extractCalculations(schema)
+    groupings = extractGroupings(schema)
+    modal?.show()
+  }
+
+  const extractCalculations = schema => {
+    if (!schema) {
+      return []
+    }
+    return Object.keys(schema)
+      .filter(field => {
+        return schema[field].calculationType != null
+      })
+      .map(field => ({
+        type: schema[field].calculationType,
+        field: schema[field].field,
+      }))
+  }
+
+  const extractGroupings = schema => {
+    if (!schema) {
+      return []
+    }
+    return Object.keys(schema)
+      .filter(field => {
+        return schema[field].calculationType == null && schema[field].visible
+      })
+      .map(field => ({ field }))
+  }
+
+  // Gets the available types for a given calculation
+  const getTypeOptions = (self, calculations) => {
+    return calculationTypeOptions.filter(option => {
+      return !calculations.some(
+        calc =>
+          calc !== self &&
+          calc.field === self.field &&
+          calc.type === option.value
+      )
+    })
+  }
+
+  // Gets the available fields for a given calculation
+  const getFieldOptions = (self, calculations, schema) => {
+    return Object.entries(schema)
+      .filter(([field, fieldSchema]) => {
+        // Only allow numeric fields that are not calculations themselves
+        if (
+          fieldSchema.calculationType ||
+          fieldSchema.type !== FieldType.NUMBER
+        ) {
+          return false
+        }
+        // Don't allow duplicates
+        return !calculations.some(calc => {
+          return (
+            calc !== self && calc.type === self.type && calc.field === field
+          )
+        })
+      })
+      .map(([field]) => field)
+  }
+
+  // Gets the available fields to group by for a given grouping
+  const getGroupingOptions = (self, groupings, schema) => {
+    return Object.entries(schema)
+      .filter(([field, fieldSchema]) => {
+        // Don't allow grouping by calculations
+        if (fieldSchema.calculationType) {
+          return false
+        }
+        // Don't allow duplicates
+        return !groupings.some(grouping => {
+          return grouping !== self && grouping.field === field
+        })
+      })
+      .map(([field]) => field)
+  }
+
+  const addCalc = () => {
+    calculations = [...calculations, {}]
+  }
+
+  const deleteCalc = idx => {
+    calculations = calculations.toSpliced(idx, 1)
+  }
+
+  const addGrouping = () => {
+    groupings = [...groupings, {}]
+  }
+
+  const deleteGrouping = idx => {
+    groupings = groupings.toSpliced(idx, 1)
+  }
 
   const save = async () => {
-    await API.viewV2.update({
+    let schema = {}
+
+    const rand = ("" + Math.random()).substring(2)
+    await datasource.actions.saveDefinition({
       ...$definition,
+      primaryDisplay: null,
       schema: {
-        "Average game length": {
+        ["Average game length " + rand]: {
           visible: true,
           calculationType: CalculationType.AVG,
           field: "Game Length",
+          width: 300,
         },
       },
     })
+    await rows.actions.refreshData()
   }
 </script>
 
-<ActionButton icon="WebPage" quiet on:click={modal?.show}>
+<ActionButton icon="WebPage" quiet on:click={open}>
   Configure calculations
 </ActionButton>
 
@@ -34,6 +166,76 @@
     size="L"
     onConfirm={save}
   >
-    Show calculations which are based on
+    {#if !calculations.length}
+      <div>
+        <ActionButton quiet icon="Add">Add your first calculation</ActionButton>
+      </div>
+    {:else}
+      <div class="calculations">
+        {#each calculations as calc, idx}
+          <span>{idx === 0 ? "Calculate" : "and"} the</span>
+          <Select
+            options={getTypeOptions(calc, calculations)}
+            bind:value={calc.type}
+            placeholder="Function"
+          />
+          <span>of</span>
+          <Select
+            options={getFieldOptions(calc, calculations, schema)}
+            bind:value={calc.field}
+            placeholder="Column"
+          />
+          <Icon
+            hoverable
+            name="Close"
+            size="S"
+            on:click={() => deleteCalc(idx)}
+            color="var(--spectrum-global-color-gray-700)"
+          />
+        {/each}
+        {#each groupings as group, idx}
+          <span>{idx === 0 ? "Group by" : "and"}</span>
+          <Select
+            options={getGroupingOptions(group, groupings, schema)}
+            bind:value={group.field}
+            placeholder="Column"
+          />
+          <Icon
+            hoverable
+            name="Close"
+            size="S"
+            on:click={() => deleteGrouping(idx)}
+            color="var(--spectrum-global-color-gray-700)"
+          />
+          <span />
+          <span />
+        {/each}
+      </div>
+      <div class="buttons">
+        <ActionButton quiet icon="Add" on:click={addCalc}>
+          Add calculation
+        </ActionButton>
+        <ActionButton quiet icon="Add" on:click={addGrouping}>
+          Group by
+        </ActionButton>
+      </div>
+    {/if}
   </ModalContent>
 </Modal>
+
+<style>
+  .calculations {
+    display: grid;
+    grid-template-columns: auto 1fr auto 1fr auto;
+    align-items: center;
+    column-gap: var(--spacing-m);
+    row-gap: var(--spacing-m);
+  }
+  .buttons {
+    display: flex;
+    flex-direction: row;
+  }
+  span {
+    text-align: right;
+  }
+</style>

From 6d294be646543fce3b2094502fe5037b280e3679 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Oct 2024 09:45:47 +0100
Subject: [PATCH 230/455] Add saving of calculation view updates

---
 .../grid/GridViewCalculationButton.svelte     | 32 ++++++++++++-------
 .../_components/CreateViewButton.svelte       |  2 +-
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 96a7d0ef8ce..3644a846ba7 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -41,7 +41,7 @@
 
   const open = () => {
     calculations = extractCalculations(schema)
-    groupings = extractGroupings(schema)
+    groupings = calculations.length ? extractGroupings(schema) : []
     modal?.show()
   }
 
@@ -138,18 +138,26 @@
   const save = async () => {
     let schema = {}
 
-    const rand = ("" + Math.random()).substring(2)
+    // Add calculations
+    for (let calc of calculations) {
+      const name = `${calc.type} of ${calc.field}`
+      schema[name] = {
+        calculationType: calc.type,
+        field: calc.field,
+        visible: true,
+      }
+    }
+
+    // Add groupings
+    for (let grouping of groupings) {
+      schema[grouping.field] = {
+        visible: true,
+      }
+    }
     await datasource.actions.saveDefinition({
       ...$definition,
       primaryDisplay: null,
-      schema: {
-        ["Average game length " + rand]: {
-          visible: true,
-          calculationType: CalculationType.AVG,
-          field: "Game Length",
-          width: 300,
-        },
-      },
+      schema,
     })
     await rows.actions.refreshData()
   }
@@ -168,7 +176,9 @@
   >
     {#if !calculations.length}
       <div>
-        <ActionButton quiet icon="Add">Add your first calculation</ActionButton>
+        <ActionButton quiet icon="Add" on:click={addCalc}>
+          Add your first calculation
+        </ActionButton>
       </div>
     {:else}
       <div class="calculations">
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index 253c386b9ef..e531e54d87b 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -39,7 +39,7 @@
       const newView = await viewsV2.create({
         name: trimmedName,
         tableId: table._id,
-        schema: enrichSchema(table.schema),
+        schema: calculation ? {} : enrichSchema(table.schema),
         primaryDisplay: calculation ? undefined : table.primaryDisplay,
         type: calculation ? ViewV2Type.CALCULATION : undefined,
       })

From 6916269cbb87850a46d4020719f15d5c5e320e31 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Oct 2024 09:59:05 +0100
Subject: [PATCH 231/455] Fix duplicate API call spam due to incorrect
 comparison of view filters

---
 .../src/components/grid/stores/datasources/viewV2.js            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 8e4321be967..84d5eb153ee 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -191,7 +191,7 @@ export const initialise = context => {
         if ($view?.id !== $datasource.id) {
           return
         }
-        if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
+        if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) {
           await datasource.actions.saveDefinition({
             ...$view,
             query: $filter,

From 72dd3a1d59c9ea0078f4bcf452db3f3638b83b35 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Oct 2024 18:57:47 +0100
Subject: [PATCH 232/455] Fix resolution of builderSocket

---
 packages/server/src/api/controllers/role.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 2b225bf0cd5..ababadbcede 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -21,7 +21,7 @@ import {
 } from "@budibase/types"
 import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
-import { builderSocket } from "src/websockets"
+import { builderSocket } from "../../websockets"
 
 const UpdateRolesOptions = {
   CREATED: "created",

From f3b601d2943e8e4425aaf0039d873c3a8cf768f4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 10 Oct 2024 14:37:10 +0100
Subject: [PATCH 233/455] Improve view calculation configuration

---
 .../grid/GridViewCalculationButton.svelte     | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 3644a846ba7..6ccabaaf430 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -125,6 +125,11 @@
 
   const deleteCalc = idx => {
     calculations = calculations.toSpliced(idx, 1)
+
+    // Remove any groupings if clearing the last calculation
+    if (!calculations.length) {
+      groupings = []
+    }
   }
 
   const addGrouping = () => {
@@ -138,6 +143,10 @@
   const save = async () => {
     let schema = {}
 
+    // Prune empty stuff
+    calculations = calculations.filter(calc => calc.type && calc.field)
+    groupings = groupings.filter(grouping => grouping.field)
+
     // Add calculations
     for (let calc of calculations) {
       const name = `${calc.type} of ${calc.field}`
@@ -154,9 +163,18 @@
         visible: true,
       }
     }
+
+    // Ensure primary display is visible
+    let primaryDisplay = $definition.primaryDisplay
+    if (!primaryDisplay || !schema[primaryDisplay]) {
+      primaryDisplay = groupings[0]?.field
+    }
+    console.log("pd", primaryDisplay, groupings)
+
+    // Save changes
     await datasource.actions.saveDefinition({
       ...$definition,
-      primaryDisplay: null,
+      primaryDisplay,
       schema,
     })
     await rows.actions.refreshData()

From bf1bf1956adc1eba06150b43f9625f86ac494784 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 10 Oct 2024 14:42:52 +0100
Subject: [PATCH 234/455] Fix edge cases when saving view calcultion schema

---
 .../buttons/grid/GridViewCalculationButton.svelte        | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 6ccabaaf430..f147179bc23 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -12,7 +12,7 @@
   const { definition, datasource, rows } = getContext("grid")
   const calculationTypeOptions = [
     {
-      label: "Average (mean)",
+      label: "Average",
       value: CalculationType.AVG,
     },
     {
@@ -149,7 +149,8 @@
 
     // Add calculations
     for (let calc of calculations) {
-      const name = `${calc.type} of ${calc.field}`
+      const typeOption = calculationTypeOptions.find(x => x.value === calc.type)
+      const name = `${typeOption.label} ${calc.field}`
       schema[name] = {
         calculationType: calc.type,
         field: calc.field,
@@ -160,16 +161,16 @@
     // Add groupings
     for (let grouping of groupings) {
       schema[grouping.field] = {
+        ...$definition.schema[grouping.field],
         visible: true,
       }
     }
 
-    // Ensure primary display is visible
+    // Ensure primary display is valid
     let primaryDisplay = $definition.primaryDisplay
     if (!primaryDisplay || !schema[primaryDisplay]) {
       primaryDisplay = groupings[0]?.field
     }
-    console.log("pd", primaryDisplay, groupings)
 
     // Save changes
     await datasource.actions.saveDefinition({

From 1cbae366834fc0380d3e1617f950ad8b0fb4fb36 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 10 Oct 2024 14:52:13 +0100
Subject: [PATCH 235/455] Don't bother fetching data if no calculations defined

---
 .../src/components/grid/stores/columns.js         |  1 -
 packages/frontend-core/src/fetch/ViewV2Fetch.js   | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 36fb6659a9b..4358deeb2c7 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -164,7 +164,6 @@ export const initialise = context => {
           }
           // Override a few properties for primary display
           if (field === primaryDisplay) {
-            column.visible = true
             column.order = 0
             column.primaryDisplay = true
           }
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 40135746df3..6bfef0927fe 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -1,3 +1,4 @@
+import { ViewV2Type } from "@budibase/types"
 import DataFetch from "./DataFetch.js"
 import { get } from "svelte/store"
 
@@ -39,6 +40,19 @@ export default class ViewV2Fetch extends DataFetch {
       this.options
     const { cursor, query, definition } = get(this.store)
 
+    // If this is a calculation view and we have no calculations, return nothing
+    if (
+      definition.type === ViewV2Type.CALCULATION &&
+      !Object.values(definition.schema || {}).some(x => x.calculationType)
+    ) {
+      return {
+        rows: [],
+        hasNextPage: false,
+        cursor: null,
+        error: null,
+      }
+    }
+
     // If sort/filter params are not defined, update options to store the
     // params built in to this view. This ensures that we can accurately
     // compare old and new params and skip a redundant API call.
@@ -67,6 +81,7 @@ export default class ViewV2Fetch extends DataFetch {
       return {
         rows: [],
         hasNextPage: false,
+        cursor: null,
         error,
       }
     }

From 88b2f423f781016b5270b5c5ab171f002aee8872 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 10 Oct 2024 16:05:02 +0100
Subject: [PATCH 236/455] Improve UX when editing view calculations

---
 packages/bbui/src/Modal/ModalContent.svelte   |   3 +
 .../grid/GridViewCalculationButton.svelte     | 119 +++++++++---------
 .../_components/Component/InfoDisplay.svelte  |  12 +-
 3 files changed, 69 insertions(+), 65 deletions(-)

diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte
index 22a14c358f2..8a80b2fdb99 100644
--- a/packages/bbui/src/Modal/ModalContent.svelte
+++ b/packages/bbui/src/Modal/ModalContent.svelte
@@ -147,6 +147,9 @@
   .spectrum-Dialog--extraLarge {
     width: 1000px;
   }
+  .spectrum-Dialog--medium {
+    width: 540px;
+  }
 
   .content-grid {
     display: grid;
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index f147179bc23..b61163c5cef 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -5,8 +5,10 @@
     ModalContent,
     Select,
     Icon,
+    Multiselect,
   } from "@budibase/bbui"
-  import { CalculationType, FieldType } from "@budibase/types"
+  import { CalculationType, canGroupBy, FieldType } from "@budibase/types"
+  import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
   import { getContext } from "svelte"
 
   const { definition, datasource, rows } = getContext("grid")
@@ -35,13 +37,16 @@
 
   let modal
   let calculations = []
-  let groupings = []
+  let groupBy = []
+  let schema = {}
 
   $: schema = $definition?.schema || {}
+  $: count = extractCalculations($definition?.schema || {}).length
+  $: groupByOptions = getGroupByOptions(schema)
 
   const open = () => {
     calculations = extractCalculations(schema)
-    groupings = calculations.length ? extractGroupings(schema) : []
+    groupBy = calculations.length ? extractGroupBy(schema) : []
     modal?.show()
   }
 
@@ -59,15 +64,13 @@
       }))
   }
 
-  const extractGroupings = schema => {
+  const extractGroupBy = schema => {
     if (!schema) {
       return []
     }
-    return Object.keys(schema)
-      .filter(field => {
-        return schema[field].calculationType == null && schema[field].visible
-      })
-      .map(field => ({ field }))
+    return Object.keys(schema).filter(field => {
+      return schema[field].calculationType == null && schema[field].visible
+    })
   }
 
   // Gets the available types for a given calculation
@@ -103,18 +106,16 @@
       .map(([field]) => field)
   }
 
-  // Gets the available fields to group by for a given grouping
-  const getGroupingOptions = (self, groupings, schema) => {
+  // Gets the available fields to group by
+  const getGroupByOptions = schema => {
     return Object.entries(schema)
       .filter(([field, fieldSchema]) => {
         // Don't allow grouping by calculations
         if (fieldSchema.calculationType) {
           return false
         }
-        // Don't allow duplicates
-        return !groupings.some(grouping => {
-          return grouping !== self && grouping.field === field
-        })
+        // Don't allow complex types
+        return canGroupBy(fieldSchema.type)
       })
       .map(([field]) => field)
   }
@@ -126,71 +127,65 @@
   const deleteCalc = idx => {
     calculations = calculations.toSpliced(idx, 1)
 
-    // Remove any groupings if clearing the last calculation
+    // Remove any grouping if clearing the last calculation
     if (!calculations.length) {
-      groupings = []
+      groupBy = []
     }
   }
 
-  const addGrouping = () => {
-    groupings = [...groupings, {}]
-  }
-
-  const deleteGrouping = idx => {
-    groupings = groupings.toSpliced(idx, 1)
-  }
-
   const save = async () => {
-    let schema = {}
-
-    // Prune empty stuff
-    calculations = calculations.filter(calc => calc.type && calc.field)
-    groupings = groupings.filter(grouping => grouping.field)
+    let newSchema = {}
 
     // Add calculations
     for (let calc of calculations) {
+      if (!calc.type || !calc.field) {
+        continue
+      }
       const typeOption = calculationTypeOptions.find(x => x.value === calc.type)
       const name = `${typeOption.label} ${calc.field}`
-      schema[name] = {
+      newSchema[name] = {
         calculationType: calc.type,
         field: calc.field,
         visible: true,
       }
     }
 
-    // Add groupings
-    for (let grouping of groupings) {
-      schema[grouping.field] = {
-        ...$definition.schema[grouping.field],
-        visible: true,
+    // Add other fields
+    for (let field of Object.keys(schema)) {
+      if (schema[field].calculationType) {
+        continue
+      }
+      newSchema[field] = {
+        ...schema[field],
+        visible: groupBy.includes(field),
       }
     }
 
     // Ensure primary display is valid
     let primaryDisplay = $definition.primaryDisplay
-    if (!primaryDisplay || !schema[primaryDisplay]) {
-      primaryDisplay = groupings[0]?.field
+    if (!primaryDisplay || !newSchema[primaryDisplay]?.visible) {
+      primaryDisplay = groupBy[0]
     }
 
     // Save changes
     await datasource.actions.saveDefinition({
       ...$definition,
       primaryDisplay,
-      schema,
+      schema: newSchema,
     })
     await rows.actions.refreshData()
   }
 </script>
 
 <ActionButton icon="WebPage" quiet on:click={open}>
-  Configure calculations
+  Configure calculations{count ? `: ${count}` : ""}
 </ActionButton>
 
 <Modal bind:this={modal}>
   <ModalContent
     title="Calculations"
     confirmText="Save"
-    size="L"
+    size="M"
     onConfirm={save}
   >
     {#if !calculations.length}
@@ -222,32 +217,30 @@
             color="var(--spectrum-global-color-gray-700)"
           />
         {/each}
-        {#each groupings as group, idx}
-          <span>{idx === 0 ? "Group by" : "and"}</span>
-          <Select
-            options={getGroupingOptions(group, groupings, schema)}
-            bind:value={group.field}
-            placeholder="Column"
+        <span>Group by</span>
+        <div class="group-by">
+          <Multiselect
+            options={groupByOptions}
+            bind:value={groupBy}
+            placeholder="None"
           />
-          <Icon
-            hoverable
-            name="Close"
-            size="S"
-            on:click={() => deleteGrouping(idx)}
-            color="var(--spectrum-global-color-gray-700)"
-          />
-          <span />
-          <span />
-        {/each}
+        </div>
       </div>
       <div class="buttons">
-        <ActionButton quiet icon="Add" on:click={addCalc}>
+        <ActionButton
+          quiet
+          icon="Add"
+          on:click={addCalc}
+          disabled={calculations.length >= 5}
+        >
           Add calculation
         </ActionButton>
-        <ActionButton quiet icon="Add" on:click={addGrouping}>
-          Group by
-        </ActionButton>
       </div>
+      <InfoDisplay
+        icon="Help"
+        quiet
+        body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
+      />
     {/if}
   </ModalContent>
 </Modal>
@@ -264,7 +257,9 @@
     display: flex;
     flex-direction: row;
   }
+  .group-by {
+    grid-column: 2 / 5;
+  }
   span {
-    text-align: right;
   }
 </style>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
index 03bf771beb8..60c908c7c59 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
@@ -4,9 +4,10 @@
   export let title
   export let body
   export let icon = "HelpOutline"
+  export let quiet = false
 </script>
 
-<div class="info" class:noTitle={!title}>
+<div class="info" class:noTitle={!title} class:quiet>
   {#if title}
     <div class="title">
       <Icon name={icon} />
@@ -40,15 +41,20 @@
     color: var(--spectrum-global-color-gray-600);
   }
   .info {
-    padding: var(--spacing-m) var(--spacing-l) var(--spacing-l) var(--spacing-l);
     background-color: var(--background-alt);
+    padding: var(--spacing-m) var(--spacing-l) var(--spacing-m) var(--spacing-l);
     border-radius: var(--border-radius-s);
     font-size: 13px;
   }
+  .quiet {
+    background: none;
+    color: var(--spectrum-global-color-gray-700);
+    padding: 0;
+  }
   .noTitle {
     display: flex;
     align-items: center;
-    gap: var(--spacing-m);
+    gap: var(--spacing-l);
   }
   .info :global(a) {
     color: inherit;

From 8be7f82ccb04ae04e5ab6b8590737571ef282f4c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 09:27:04 +0100
Subject: [PATCH 237/455] Improve view calculation config

---
 .../grid/GridViewCalculationButton.svelte     | 44 ++++++++-----------
 packages/frontend-core/src/utils/table.js     |  4 --
 2 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index b61163c5cef..f9833a981b8 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -121,7 +121,7 @@
   }
 
   const addCalc = () => {
-    calculations = [...calculations, {}]
+    calculations = [...calculations, { type: CalculationType.AVG }]
   }
 
   const deleteCalc = idx => {
@@ -188,20 +188,14 @@
     size="M"
     onConfirm={save}
   >
-    {#if !calculations.length}
-      <div>
-        <ActionButton quiet icon="Add" on:click={addCalc}>
-          Add your first calculation
-        </ActionButton>
-      </div>
-    {:else}
+    {#if calculations.length}
       <div class="calculations">
         {#each calculations as calc, idx}
           <span>{idx === 0 ? "Calculate" : "and"} the</span>
           <Select
             options={getTypeOptions(calc, calculations)}
             bind:value={calc.type}
-            placeholder="Function"
+            placeholder={false}
           />
           <span>of</span>
           <Select
@@ -211,7 +205,7 @@
           />
           <Icon
             hoverable
-            name="Close"
+            name="Delete"
             size="S"
             on:click={() => deleteCalc(idx)}
             color="var(--spectrum-global-color-gray-700)"
@@ -226,22 +220,22 @@
           />
         </div>
       </div>
-      <div class="buttons">
-        <ActionButton
-          quiet
-          icon="Add"
-          on:click={addCalc}
-          disabled={calculations.length >= 5}
-        >
-          Add calculation
-        </ActionButton>
-      </div>
-      <InfoDisplay
-        icon="Help"
-        quiet
-        body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
-      />
     {/if}
+    <div class="buttons">
+      <ActionButton
+        quiet
+        icon="Add"
+        on:click={addCalc}
+        disabled={calculations.length >= 5}
+      >
+        Add calculation
+      </ActionButton>
+    </div>
+    <InfoDisplay
+      icon="Help"
+      quiet
+      body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
+    />
   </ModalContent>
 </Modal>
 
diff --git a/packages/frontend-core/src/utils/table.js b/packages/frontend-core/src/utils/table.js
index 193848ec4d0..7dd8cfac7db 100644
--- a/packages/frontend-core/src/utils/table.js
+++ b/packages/frontend-core/src/utils/table.js
@@ -14,10 +14,6 @@ export function canBeDisplayColumn(column) {
 }
 
 export function canBeSortColumn(column) {
-  if (!sharedCore.canBeSortColumn(column.type)) {
-    return false
-  }
-
   if (column.related) {
     // If it's a related column (only available in the frontend), don't allow using it as display column
     return false

From ba6def8f73e3895c35516691c729e5bf8995be9d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 09:28:48 +0100
Subject: [PATCH 238/455] Prevent filtering on calculation fields

---
 .../src/components/CoreFilterBuilder.svelte            | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index c711a57e1c5..8a2a19b0dcb 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -69,10 +69,12 @@
 
   const context = getContext("context")
 
-  $: fieldOptions = (schemaFields || []).map(field => ({
-    label: field.displayName || field.name,
-    value: field.name,
-  }))
+  $: fieldOptions = (schemaFields || [])
+    .filter(field => !field.calculationType)
+    .map(field => ({
+      label: field.displayName || field.name,
+      value: field.name,
+    }))
 
   const onFieldChange = filter => {
     const previousType = filter.type

From 7474d3c987fd1515752c70011c2f3597230d560b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 09:39:08 +0100
Subject: [PATCH 239/455] Remove readonly fields from being able to be used in
 a save row button action

---
 .../settings/controls/ButtonActionEditor/actions/SaveRow.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
index c23edbeb58d..819ed108802 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
@@ -33,7 +33,7 @@
 
   const getSchemaFields = resourceId => {
     const { schema } = getSchemaForDatasourcePlus(resourceId)
-    return Object.values(schema || {})
+    return Object.values(schema || {}).filter(field => !field.readonly)
   }
 
   const onFieldsChanged = e => {

From bf3bb5a8474ad7d04fee6393173b38c3d649c083 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 09:45:57 +0100
Subject: [PATCH 240/455] Fix issue when deleting tables

---
 .../data/table/[tableId]/_components/ViewNavBar.svelte          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
index 69b792c6122..bf3d073f604 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte
@@ -196,7 +196,7 @@
     on:contextmenu={openTableContextMenu}
   >
     <div class="nav-item__title">
-      {table._id === TableNames.USERS ? "App users" : table.name}
+      {table?._id === TableNames.USERS ? "App users" : table?.name || ""}
     </div>
     {#if tableSelectedBy}
       <UserAvatars size="XS" users={tableSelectedBy} />

From 038bfd1360582a6ee18967d659946c3e194b86d1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 14:11:41 +0100
Subject: [PATCH 241/455] Add default value toggle for both single and multi
 user columns

---
 .../DataTable/modals/CreateEditColumn.svelte  | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 19cc4ec1c09..9a1b4645667 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -49,6 +49,9 @@
   import OptionsEditor from "./OptionsEditor.svelte"
   import { isEnabled } from "helpers/featureFlags"
   import { getUserBindings } from "dataBinding"
+  import { makePropSafe as safe } from "@budibase/string-templates"
+
+  export let field
 
   const AUTO_TYPE = FieldType.AUTO
   const FORMULA_TYPE = FieldType.FORMULA
@@ -57,12 +60,12 @@
   const NUMBER_TYPE = FieldType.NUMBER
   const JSON_TYPE = FieldType.JSON
   const DATE_TYPE = FieldType.DATETIME
+  const SINGLE_USER_DEFAULT = `{{ ${safe("user")}.${safe("_id")} }}`
+  const MULTI_USER_DEFAULT = `{{ js "cmV0dXJuIFskKCJbdXNlcl0uW19pZF0iKV0=" }}`
 
   const dispatch = createEventDispatcher()
   const { dispatch: gridDispatch, rows } = getContext("grid")
 
-  export let field
-
   let mounted = false
   let originalName
   let linkEditDisabled
@@ -835,6 +838,18 @@
           (editableColumn.default = e.detail?.length ? e.detail : undefined)}
         placeholder="None"
       />
+    {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
+      {@const defaultValue =
+        editableColumn.type === FieldType.BB_REFERENCE
+          ? SINGLE_USER_DEFAULT
+          : MULTI_USER_DEFAULT}
+      <Toggle
+        disabled={!canHaveDefault}
+        text="Default to current user"
+        value={editableColumn.default === defaultValue}
+        on:change={e =>
+          (editableColumn.default = e.detail ? defaultValue : undefined)}
+      />
     {:else}
       <ModalBindableInput
         disabled={!canHaveDefault}

From bd3e83789ec5f1e6e0b6223f76dc60fb3d0c3923 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:06:20 +0100
Subject: [PATCH 242/455] Use more readable bindings and remove extraneous
 aliases to enums

---
 .../DataTable/modals/CreateEditColumn.svelte  | 61 +++++++++----------
 1 file changed, 30 insertions(+), 31 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 9a1b4645667..9362fc5d5d3 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -53,18 +53,11 @@
 
   export let field
 
-  const AUTO_TYPE = FieldType.AUTO
-  const FORMULA_TYPE = FieldType.FORMULA
-  const LINK_TYPE = FieldType.LINK
-  const STRING_TYPE = FieldType.STRING
-  const NUMBER_TYPE = FieldType.NUMBER
-  const JSON_TYPE = FieldType.JSON
-  const DATE_TYPE = FieldType.DATETIME
-  const SINGLE_USER_DEFAULT = `{{ ${safe("user")}.${safe("_id")} }}`
-  const MULTI_USER_DEFAULT = `{{ js "cmV0dXJuIFskKCJbdXNlcl0uW19pZF0iKV0=" }}`
-
   const dispatch = createEventDispatcher()
   const { dispatch: gridDispatch, rows } = getContext("grid")
+  const SafeID = `${safe("user")}.${safe("_id")}`
+  const SingleUserDefault = `{{ ${SafeID} }}`
+  const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}`
 
   let mounted = false
   let originalName
@@ -113,7 +106,7 @@
   $: {
     // this parses any changes the user has made when creating a new internal relationship
     // into what we expect the schema to look like
-    if (editableColumn.type === LINK_TYPE) {
+    if (editableColumn.type === FieldType.LINK) {
       relationshipTableIdPrimary = table._id
       if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
         relationshipOpts2 = relationshipOpts2.filter(
@@ -150,7 +143,7 @@
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
   $: invalid =
     !editableColumn?.name ||
-    (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) ||
+    (editableColumn?.type === FieldType.LINK && !editableColumn?.tableId) ||
     Object.keys(errors).length !== 0 ||
     !optionsValid
   $: errors = checkErrors(editableColumn)
@@ -176,9 +169,9 @@
   $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
   $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
-    editableColumn?.type !== LINK_TYPE &&
+    editableColumn?.type !== FieldType.LINK &&
     !uneditable &&
-    editableColumn?.type !== AUTO_TYPE &&
+    editableColumn?.type !== FieldType.AUTO &&
     !editableColumn.autocolumn
   $: hasDefault =
     editableColumn?.default != null && editableColumn?.default !== ""
@@ -227,7 +220,7 @@
 
   function makeFieldId(type, subtype, autocolumn) {
     // don't make field IDs for auto types
-    if (type === AUTO_TYPE || autocolumn) {
+    if (type === FieldType.AUTO || autocolumn) {
       return type.toUpperCase()
     } else if (
       type === FieldType.BB_REFERENCE ||
@@ -252,7 +245,7 @@
       // Here we are setting the relationship values based on the editableColumn
       // This part of the code is used when viewing an existing field hence the check
       // for the tableId
-      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+      if (editableColumn.type === FieldType.LINK && editableColumn.tableId) {
         relationshipTableIdPrimary = table._id
         relationshipTableIdSecondary = editableColumn.tableId
         if (editableColumn.relationshipType in relationshipMap) {
@@ -293,14 +286,14 @@
 
     delete saveColumn.fieldId
 
-    if (saveColumn.type === AUTO_TYPE) {
+    if (saveColumn.type === FieldType.AUTO) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
         saveColumn.name,
         saveColumn.subtype
       )
     }
-    if (saveColumn.type !== LINK_TYPE) {
+    if (saveColumn.type !== FieldType.LINK) {
       delete saveColumn.fieldName
     }
 
@@ -387,9 +380,9 @@
     editableColumn.subtype = definition.subtype
 
     // Default relationships many to many
-    if (editableColumn.type === LINK_TYPE) {
+    if (editableColumn.type === FieldType.LINK) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
-    } else if (editableColumn.type === FORMULA_TYPE) {
+    } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
     }
   }
@@ -508,17 +501,23 @@
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
-    if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) {
+    if (
+      fieldToCheck.type === FieldType.STRING &&
+      !fieldToCheck.constraints.length
+    ) {
       fieldToCheck.constraints.length = {}
     }
     // some number types made server-side will be missing constraints
     if (
-      fieldToCheck.type === NUMBER_TYPE &&
+      fieldToCheck.type === FieldType.NUMBER &&
       !fieldToCheck.constraints.numericality
     ) {
       fieldToCheck.constraints.numericality = {}
     }
-    if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) {
+    if (
+      fieldToCheck.type === FieldType.DATETIME &&
+      !fieldToCheck.constraints.datetime
+    ) {
       fieldToCheck.constraints.datetime = {}
     }
   }
@@ -593,13 +592,13 @@
       on:input={e => {
         if (
           !uneditable &&
-          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+          !(linkEditDisabled && editableColumn.type === FieldType.LINK)
         ) {
           editableColumn.name = e.target.value
         }
       }}
       disabled={uneditable ||
-        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+        (linkEditDisabled && editableColumn.type === FieldType.LINK)}
       error={errors?.name}
     />
   {/if}
@@ -613,7 +612,7 @@
     getOptionValue={field => field.fieldId}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
-      if (option.type === AUTO_TYPE) {
+      if (option.type === FieldType.AUTO) {
         return availableAutoColumnKeys?.length > 0
       }
       return true
@@ -656,7 +655,7 @@
       bind:optionColors={editableColumn.optionColors}
       bind:valid={optionsValid}
     />
-  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
+  {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Earliest</Label>
@@ -743,7 +742,7 @@
       {tableOptions}
       {errors}
     />
-  {:else if editableColumn.type === FORMULA_TYPE}
+  {:else if editableColumn.type === FieldType.FORMULA}
     {#if !externalTable}
       <div class="split-label">
         <div class="label-length">
@@ -786,12 +785,12 @@
         />
       </div>
     </div>
-  {:else if editableColumn.type === JSON_TYPE}
+  {:else if editableColumn.type === FieldType.JSON}
     <Button primary text on:click={openJsonSchemaEditor}>
       Open schema editor
     </Button>
   {/if}
-  {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
+  {#if editableColumn.type === FieldType.AUTO || editableColumn.autocolumn}
     <Select
       label="Auto column type"
       value={editableColumn.subtype}
@@ -840,7 +839,7 @@
       />
     {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
       {@const defaultValue =
-        editableColumn.type === FieldType.BB_REFERENCE
+        editableColumn.type === FieldType.BB_REFERENCE_SINGLE
           ? SINGLE_USER_DEFAULT
           : MULTI_USER_DEFAULT}
       <Toggle

From 97cee504f04fa5328b104018900df9c84b2eb652 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:06:44 +0100
Subject: [PATCH 243/455] Fix references

---
 .../backend/DataTable/modals/CreateEditColumn.svelte          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 9362fc5d5d3..79f80974ac5 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -840,8 +840,8 @@
     {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
       {@const defaultValue =
         editableColumn.type === FieldType.BB_REFERENCE_SINGLE
-          ? SINGLE_USER_DEFAULT
-          : MULTI_USER_DEFAULT}
+          ? SingleUserDefault
+          : MultiUserDefault}
       <Toggle
         disabled={!canHaveDefault}
         text="Default to current user"

From 516f45e8ad9c727d584dbf19b4bf86f8ea6cd21c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:37:25 +0100
Subject: [PATCH 244/455] Re-add deleted code

---
 packages/frontend-core/src/utils/table.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/frontend-core/src/utils/table.js b/packages/frontend-core/src/utils/table.js
index 7dd8cfac7db..193848ec4d0 100644
--- a/packages/frontend-core/src/utils/table.js
+++ b/packages/frontend-core/src/utils/table.js
@@ -14,6 +14,10 @@ export function canBeDisplayColumn(column) {
 }
 
 export function canBeSortColumn(column) {
+  if (!sharedCore.canBeSortColumn(column.type)) {
+    return false
+  }
+
   if (column.related) {
     // If it's a related column (only available in the frontend), don't allow using it as display column
     return false

From 865eb3169fe5a9a716b1314887fcdaba9aae4328 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:38:35 +0100
Subject: [PATCH 245/455] Allow sorting by calculation columns

---
 packages/frontend-core/src/utils/table.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/utils/table.js b/packages/frontend-core/src/utils/table.js
index 193848ec4d0..b7abd572c73 100644
--- a/packages/frontend-core/src/utils/table.js
+++ b/packages/frontend-core/src/utils/table.js
@@ -4,24 +4,24 @@ export function canBeDisplayColumn(column) {
   if (!sharedCore.canBeDisplayColumn(column.type)) {
     return false
   }
-
+  // If it's a related column (only available in the frontend), don't allow using it as display column
   if (column.related) {
-    // If it's a related column (only available in the frontend), don't allow using it as display column
     return false
   }
-
   return true
 }
 
 export function canBeSortColumn(column) {
+  // Allow sorting by calculation columns
+  if (column.calculationType) {
+    return true
+  }
   if (!sharedCore.canBeSortColumn(column.type)) {
     return false
   }
-
+  // If it's a related column (only available in the frontend), don't allow using it as display column
   if (column.related) {
-    // If it's a related column (only available in the frontend), don't allow using it as display column
     return false
   }
-
   return true
 }

From 838f9e9c56aa85b8b276ae6f6568f616578e6c79 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:43:24 +0100
Subject: [PATCH 246/455] Improve sorting logic

---
 .../backend/DataTable/buttons/grid/GridSortButton.svelte      | 4 +---
 packages/frontend-core/src/components/grid/lib/renderers.js   | 2 +-
 packages/frontend-core/src/components/grid/stores/columns.js  | 2 +-
 packages/frontend-core/src/utils/schema.js                    | 2 +-
 4 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
index 5a1f6b221a0..4591b22b781 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
@@ -9,13 +9,11 @@
   let anchor
 
   $: columnOptions = $columns
+    .filter(col => canBeSortColumn(col.schema))
     .map(col => ({
       label: col.label || col.name,
       value: col.name,
-      type: col.schema?.type,
-      related: col.related,
     }))
-    .filter(col => canBeSortColumn(col))
   $: orderOptions = getOrderOptions($sort.column, columnOptions)
 
   const getOrderOptions = (column, columnOptions) => {
diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index feed991da21..547c397b4de 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -35,7 +35,7 @@ const TypeComponentMap = {
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
 }
 export const getCellRenderer = column => {
-  if (column.calculation) {
+  if (column.calculationType) {
     return NumberCell
   }
   return (
diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 4358deeb2c7..db09fa00c55 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -160,7 +160,7 @@ export const initialise = context => {
             order: fieldSchema.order ?? oldColumn?.order,
             conditions: fieldSchema.conditions,
             related: fieldSchema.related,
-            calculation: fieldSchema.calculationType != null,
+            calculationType: fieldSchema.calculationType,
           }
           // Override a few properties for primary display
           if (field === primaryDisplay) {
diff --git a/packages/frontend-core/src/utils/schema.js b/packages/frontend-core/src/utils/schema.js
index 30b0bd00792..e4e26dce396 100644
--- a/packages/frontend-core/src/utils/schema.js
+++ b/packages/frontend-core/src/utils/schema.js
@@ -5,7 +5,7 @@ export const getColumnIcon = column => {
   if (column.schema.icon) {
     return column.schema.icon
   }
-  if (column.calculation) {
+  if (column.calculationType) {
     return "Calculator"
   }
   if (column.schema.autocolumn) {

From b8cd40bf211f0703ff5056a56187d75df5c7d95b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:45:31 +0100
Subject: [PATCH 247/455] Delete random file

---
 ....timestamp-1728399274731-fb0dcfaf9cacc.mjs | 146 ------------------
 1 file changed, 146 deletions(-)
 delete mode 100644 packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs

diff --git a/packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs b/packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs
deleted file mode 100644
index 8a28a7ca60c..00000000000
--- a/packages/builder/vite.config.mjs.timestamp-1728399274731-fb0dcfaf9cacc.mjs
+++ /dev/null
@@ -1,146 +0,0 @@
-// vite.config.mjs
-import { svelte } from "file:///Users/aptk/dev/budibase/node_modules/@sveltejs/vite-plugin-svelte/dist/index.js";
-import replace from "file:///Users/aptk/dev/budibase/node_modules/@rollup/plugin-replace/dist/es/index.js";
-import { defineConfig, loadEnv } from "file:///Users/aptk/dev/budibase/node_modules/vite/dist/node/index.js";
-import { viteStaticCopy } from "file:///Users/aptk/dev/budibase/node_modules/vite-plugin-static-copy/dist/index.js";
-import path from "path";
-var ignoredWarnings = [
-  "unused-export-let",
-  "css-unused-selector",
-  "module-script-reactive-declaration",
-  "a11y-no-onchange",
-  "a11y-click-events-have-key-events"
-];
-var copyFonts = (dest) => viteStaticCopy({
-  targets: [
-    {
-      src: "../../node_modules/@fontsource/source-sans-pro",
-      dest
-    },
-    {
-      src: "../../node_modules/remixicon/fonts/*",
-      dest
-    }
-  ]
-});
-var vite_config_default = defineConfig(({ mode }) => {
-  const isProduction = mode === "production";
-  const env = loadEnv(mode, process.cwd());
-  const devOnlyPlugins = [
-    // Copy fonts to an additional path so that svelte's automatic
-    // prefixing of the base URL path can still resolve assets
-    copyFonts("builder/fonts")
-  ];
-  return {
-    test: {
-      setupFiles: ["./vitest.setup.js"],
-      globals: true,
-      environment: "jsdom"
-    },
-    server: {
-      fs: {
-        strict: false
-      },
-      hmr: {
-        protocol: env.VITE_HMR_PROTOCOL || "ws",
-        clientPort: env.VITE_HMR_CLIENT_PORT || 3e3,
-        path: env.VITE_HMR_PATH || "/"
-      },
-      port: 3e3
-    },
-    base: "/builder/",
-    build: {
-      minify: isProduction,
-      outDir: "../server/builder",
-      sourcemap: !isProduction
-    },
-    plugins: [
-      svelte({
-        hot: !isProduction,
-        emitCss: true,
-        onwarn: (warning, handler) => {
-          if (!ignoredWarnings.includes(warning.code)) {
-            handler(warning);
-          }
-        }
-      }),
-      replace({
-        preventAssignment: true,
-        "process.env.NODE_ENV": JSON.stringify(
-          isProduction ? "production" : "development"
-        ),
-        "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN)
-      }),
-      copyFonts("fonts"),
-      ...isProduction ? [] : devOnlyPlugins
-    ],
-    optimizeDeps: {
-      exclude: ["@roxi/routify", "fsevents"]
-    },
-    resolve: {
-      dedupe: ["@roxi/routify"],
-      alias: [
-        {
-          find: "assets",
-          replacement: path.resolve("./assets")
-        },
-        {
-          find: "components",
-          replacement: path.resolve("./src/components")
-        },
-        {
-          find: "pages",
-          replacement: path.resolve("./src/pages")
-        },
-        {
-          find: "templates",
-          replacement: path.resolve("./src/templates")
-        },
-        {
-          find: "stores",
-          replacement: path.resolve("./src/stores")
-        },
-        {
-          find: "dataBinding",
-          replacement: path.resolve("./src/dataBinding.js")
-        },
-        {
-          find: "api",
-          replacement: path.resolve("./src/api.js")
-        },
-        {
-          find: "constants",
-          replacement: path.resolve("./src/constants")
-        },
-        {
-          find: "analytics",
-          replacement: path.resolve("./src/analytics")
-        },
-        {
-          find: "actions",
-          replacement: path.resolve("./src/actions")
-        },
-        {
-          find: "helpers",
-          replacement: path.resolve("./src/helpers")
-        },
-        {
-          find: "@budibase/types",
-          replacement: path.resolve("../types/src")
-        },
-        {
-          find: "@budibase/shared-core",
-          replacement: path.resolve("../shared-core/src")
-        },
-        {
-          find: "@budibase/bbui",
-          replacement: path.resolve("../bbui/src")
-        }
-      ]
-    }
-  };
-});
-export {
-  vite_config_default as default
-};
-//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsidml0ZS5jb25maWcubWpzIl0sCiAgInNvdXJjZXNDb250ZW50IjogWyJjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfZGlybmFtZSA9IFwiL1VzZXJzL2FwdGsvZGV2L2J1ZGliYXNlL3BhY2thZ2VzL2J1aWxkZXJcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfZmlsZW5hbWUgPSBcIi9Vc2Vycy9hcHRrL2Rldi9idWRpYmFzZS9wYWNrYWdlcy9idWlsZGVyL3ZpdGUuY29uZmlnLm1qc1wiO2NvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9pbXBvcnRfbWV0YV91cmwgPSBcImZpbGU6Ly8vVXNlcnMvYXB0ay9kZXYvYnVkaWJhc2UvcGFja2FnZXMvYnVpbGRlci92aXRlLmNvbmZpZy5tanNcIjtpbXBvcnQgeyBzdmVsdGUgfSBmcm9tIFwiQHN2ZWx0ZWpzL3ZpdGUtcGx1Z2luLXN2ZWx0ZVwiXG5pbXBvcnQgcmVwbGFjZSBmcm9tIFwiQHJvbGx1cC9wbHVnaW4tcmVwbGFjZVwiXG5pbXBvcnQgeyBkZWZpbmVDb25maWcsIGxvYWRFbnYgfSBmcm9tIFwidml0ZVwiXG5pbXBvcnQgeyB2aXRlU3RhdGljQ29weSB9IGZyb20gXCJ2aXRlLXBsdWdpbi1zdGF0aWMtY29weVwiXG5pbXBvcnQgcGF0aCBmcm9tIFwicGF0aFwiXG5cbmNvbnN0IGlnbm9yZWRXYXJuaW5ncyA9IFtcbiAgXCJ1bnVzZWQtZXhwb3J0LWxldFwiLFxuICBcImNzcy11bnVzZWQtc2VsZWN0b3JcIixcbiAgXCJtb2R1bGUtc2NyaXB0LXJlYWN0aXZlLWRlY2xhcmF0aW9uXCIsXG4gIFwiYTExeS1uby1vbmNoYW5nZVwiLFxuICBcImExMXktY2xpY2stZXZlbnRzLWhhdmUta2V5LWV2ZW50c1wiLFxuXVxuXG5jb25zdCBjb3B5Rm9udHMgPSBkZXN0ID0+XG4gIHZpdGVTdGF0aWNDb3B5KHtcbiAgICB0YXJnZXRzOiBbXG4gICAgICB7XG4gICAgICAgIHNyYzogXCIuLi8uLi9ub2RlX21vZHVsZXMvQGZvbnRzb3VyY2Uvc291cmNlLXNhbnMtcHJvXCIsXG4gICAgICAgIGRlc3QsXG4gICAgICB9LFxuICAgICAge1xuICAgICAgICBzcmM6IFwiLi4vLi4vbm9kZV9tb2R1bGVzL3JlbWl4aWNvbi9mb250cy8qXCIsXG4gICAgICAgIGRlc3QsXG4gICAgICB9LFxuICAgIF0sXG4gIH0pXG5cbmV4cG9ydCBkZWZhdWx0IGRlZmluZUNvbmZpZygoeyBtb2RlIH0pID0+IHtcbiAgY29uc3QgaXNQcm9kdWN0aW9uID0gbW9kZSA9PT0gXCJwcm9kdWN0aW9uXCJcbiAgY29uc3QgZW52ID0gbG9hZEVudihtb2RlLCBwcm9jZXNzLmN3ZCgpKVxuXG4gIC8vIFBsdWdpbnMgdG8gb25seSBydW4gaW4gZGV2XG4gIGNvbnN0IGRldk9ubHlQbHVnaW5zID0gW1xuICAgIC8vIENvcHkgZm9udHMgdG8gYW4gYWRkaXRpb25hbCBwYXRoIHNvIHRoYXQgc3ZlbHRlJ3MgYXV0b21hdGljXG4gICAgLy8gcHJlZml4aW5nIG9mIHRoZSBiYXNlIFVSTCBwYXRoIGNhbiBzdGlsbCByZXNvbHZlIGFzc2V0c1xuICAgIGNvcHlGb250cyhcImJ1aWxkZXIvZm9udHNcIiksXG5dXG5cbiAgcmV0dXJuIHtcbiAgICB0ZXN0OiB7XG4gICAgICBzZXR1cEZpbGVzOiBbXCIuL3ZpdGVzdC5zZXR1cC5qc1wiXSxcbiAgICAgIGdsb2JhbHM6IHRydWUsXG4gICAgICBlbnZpcm9ubWVudDogXCJqc2RvbVwiLFxuICAgIH0sXG4gICAgc2VydmVyOiB7XG4gICAgICBmczoge1xuICAgICAgICBzdHJpY3Q6IGZhbHNlLFxuICAgICAgfSxcbiAgICAgIGhtcjoge1xuICAgICAgICBwcm90b2NvbDogZW52LlZJVEVfSE1SX1BST1RPQ09MIHx8IFwid3NcIixcbiAgICAgICAgY2xpZW50UG9ydDogZW52LlZJVEVfSE1SX0NMSUVOVF9QT1JUIHx8IDMwMDAsXG4gICAgICAgIHBhdGg6IGVudi5WSVRFX0hNUl9QQVRIIHx8IFwiL1wiLFxuICAgICAgfSxcbiAgICAgIHBvcnQ6IDMwMDAsXG4gICAgfSxcbiAgICBiYXNlOiBcIi9idWlsZGVyL1wiLFxuICAgIGJ1aWxkOiB7XG4gICAgICBtaW5pZnk6IGlzUHJvZHVjdGlvbixcbiAgICAgIG91dERpcjogXCIuLi9zZXJ2ZXIvYnVpbGRlclwiLFxuICAgICAgc291cmNlbWFwOiAhaXNQcm9kdWN0aW9uLFxuICAgIH0sXG4gICAgcGx1Z2luczogW1xuICAgICAgc3ZlbHRlKHtcbiAgICAgICAgaG90OiAhaXNQcm9kdWN0aW9uLFxuICAgICAgICBlbWl0Q3NzOiB0cnVlLFxuICAgICAgICBvbndhcm46ICh3YXJuaW5nLCBoYW5kbGVyKSA9PiB7XG4gICAgICAgICAgLy8gSWdub3JlIHNvbWUgd2FybmluZ3NcbiAgICAgICAgICBpZiAoIWlnbm9yZWRXYXJuaW5ncy5pbmNsdWRlcyh3YXJuaW5nLmNvZGUpKSB7XG4gICAgICAgICAgICBoYW5kbGVyKHdhcm5pbmcpXG4gICAgICAgICAgfVxuICAgICAgICB9LFxuICAgICAgfSksXG4gICAgICByZXBsYWNlKHtcbiAgICAgICAgcHJldmVudEFzc2lnbm1lbnQ6IHRydWUsXG4gICAgICAgIFwicHJvY2Vzcy5lbnYuTk9ERV9FTlZcIjogSlNPTi5zdHJpbmdpZnkoXG4gICAgICAgICAgaXNQcm9kdWN0aW9uID8gXCJwcm9kdWN0aW9uXCIgOiBcImRldmVsb3BtZW50XCJcbiAgICAgICAgKSxcbiAgICAgICAgXCJwcm9jZXNzLmVudi5QT1NUSE9HX1RPS0VOXCI6IEpTT04uc3RyaW5naWZ5KHByb2Nlc3MuZW52LlBPU1RIT0dfVE9LRU4pLFxuICAgICAgfSksXG4gICAgICBjb3B5Rm9udHMoXCJmb250c1wiKSxcbiAgICAgIC4uLihpc1Byb2R1Y3Rpb24gPyBbXSA6IGRldk9ubHlQbHVnaW5zKSxcbiAgICBdLFxuICAgIG9wdGltaXplRGVwczoge1xuICAgICAgZXhjbHVkZTogW1wiQHJveGkvcm91dGlmeVwiLCBcImZzZXZlbnRzXCJdLFxuICAgIH0sXG4gICAgcmVzb2x2ZToge1xuICAgICAgZGVkdXBlOiBbXCJAcm94aS9yb3V0aWZ5XCJdLFxuICAgICAgYWxpYXM6IFtcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYXNzZXRzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vYXNzZXRzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJjb21wb25lbnRzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2NvbXBvbmVudHNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcInBhZ2VzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL3BhZ2VzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJ0ZW1wbGF0ZXNcIixcbiAgICAgICAgICByZXBsYWNlbWVudDogcGF0aC5yZXNvbHZlKFwiLi9zcmMvdGVtcGxhdGVzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJzdG9yZXNcIixcbiAgICAgICAgICByZXBsYWNlbWVudDogcGF0aC5yZXNvbHZlKFwiLi9zcmMvc3RvcmVzXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJkYXRhQmluZGluZ1wiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuL3NyYy9kYXRhQmluZGluZy5qc1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYXBpXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2FwaS5qc1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiY29uc3RhbnRzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2NvbnN0YW50c1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYW5hbHl0aWNzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2FuYWx5dGljc1wiKSxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIGZpbmQ6IFwiYWN0aW9uc1wiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuL3NyYy9hY3Rpb25zXCIpLFxuICAgICAgICB9LFxuICAgICAgICB7XG4gICAgICAgICAgZmluZDogXCJoZWxwZXJzXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4vc3JjL2hlbHBlcnNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcIkBidWRpYmFzZS90eXBlc1wiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuLi90eXBlcy9zcmNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcIkBidWRpYmFzZS9zaGFyZWQtY29yZVwiLFxuICAgICAgICAgIHJlcGxhY2VtZW50OiBwYXRoLnJlc29sdmUoXCIuLi9zaGFyZWQtY29yZS9zcmNcIiksXG4gICAgICAgIH0sXG4gICAgICAgIHtcbiAgICAgICAgICBmaW5kOiBcIkBidWRpYmFzZS9iYnVpXCIsXG4gICAgICAgICAgcmVwbGFjZW1lbnQ6IHBhdGgucmVzb2x2ZShcIi4uL2JidWkvc3JjXCIpLFxuICAgICAgICB9LFxuICAgICAgXSxcbiAgICB9LFxuICB9XG59KVxuIl0sCiAgIm1hcHBpbmdzIjogIjtBQUErUyxTQUFTLGNBQWM7QUFDdFUsT0FBTyxhQUFhO0FBQ3BCLFNBQVMsY0FBYyxlQUFlO0FBQ3RDLFNBQVMsc0JBQXNCO0FBQy9CLE9BQU8sVUFBVTtBQUVqQixJQUFNLGtCQUFrQjtBQUFBLEVBQ3RCO0FBQUEsRUFDQTtBQUFBLEVBQ0E7QUFBQSxFQUNBO0FBQUEsRUFDQTtBQUNGO0FBRUEsSUFBTSxZQUFZLFVBQ2hCLGVBQWU7QUFBQSxFQUNiLFNBQVM7QUFBQSxJQUNQO0FBQUEsTUFDRSxLQUFLO0FBQUEsTUFDTDtBQUFBLElBQ0Y7QUFBQSxJQUNBO0FBQUEsTUFDRSxLQUFLO0FBQUEsTUFDTDtBQUFBLElBQ0Y7QUFBQSxFQUNGO0FBQ0YsQ0FBQztBQUVILElBQU8sc0JBQVEsYUFBYSxDQUFDLEVBQUUsS0FBSyxNQUFNO0FBQ3hDLFFBQU0sZUFBZSxTQUFTO0FBQzlCLFFBQU0sTUFBTSxRQUFRLE1BQU0sUUFBUSxJQUFJLENBQUM7QUFHdkMsUUFBTSxpQkFBaUI7QUFBQTtBQUFBO0FBQUEsSUFHckIsVUFBVSxlQUFlO0FBQUEsRUFDN0I7QUFFRSxTQUFPO0FBQUEsSUFDTCxNQUFNO0FBQUEsTUFDSixZQUFZLENBQUMsbUJBQW1CO0FBQUEsTUFDaEMsU0FBUztBQUFBLE1BQ1QsYUFBYTtBQUFBLElBQ2Y7QUFBQSxJQUNBLFFBQVE7QUFBQSxNQUNOLElBQUk7QUFBQSxRQUNGLFFBQVE7QUFBQSxNQUNWO0FBQUEsTUFDQSxLQUFLO0FBQUEsUUFDSCxVQUFVLElBQUkscUJBQXFCO0FBQUEsUUFDbkMsWUFBWSxJQUFJLHdCQUF3QjtBQUFBLFFBQ3hDLE1BQU0sSUFBSSxpQkFBaUI7QUFBQSxNQUM3QjtBQUFBLE1BQ0EsTUFBTTtBQUFBLElBQ1I7QUFBQSxJQUNBLE1BQU07QUFBQSxJQUNOLE9BQU87QUFBQSxNQUNMLFFBQVE7QUFBQSxNQUNSLFFBQVE7QUFBQSxNQUNSLFdBQVcsQ0FBQztBQUFBLElBQ2Q7QUFBQSxJQUNBLFNBQVM7QUFBQSxNQUNQLE9BQU87QUFBQSxRQUNMLEtBQUssQ0FBQztBQUFBLFFBQ04sU0FBUztBQUFBLFFBQ1QsUUFBUSxDQUFDLFNBQVMsWUFBWTtBQUU1QixjQUFJLENBQUMsZ0JBQWdCLFNBQVMsUUFBUSxJQUFJLEdBQUc7QUFDM0Msb0JBQVEsT0FBTztBQUFBLFVBQ2pCO0FBQUEsUUFDRjtBQUFBLE1BQ0YsQ0FBQztBQUFBLE1BQ0QsUUFBUTtBQUFBLFFBQ04sbUJBQW1CO0FBQUEsUUFDbkIsd0JBQXdCLEtBQUs7QUFBQSxVQUMzQixlQUFlLGVBQWU7QUFBQSxRQUNoQztBQUFBLFFBQ0EsNkJBQTZCLEtBQUssVUFBVSxRQUFRLElBQUksYUFBYTtBQUFBLE1BQ3ZFLENBQUM7QUFBQSxNQUNELFVBQVUsT0FBTztBQUFBLE1BQ2pCLEdBQUksZUFBZSxDQUFDLElBQUk7QUFBQSxJQUMxQjtBQUFBLElBQ0EsY0FBYztBQUFBLE1BQ1osU0FBUyxDQUFDLGlCQUFpQixVQUFVO0FBQUEsSUFDdkM7QUFBQSxJQUNBLFNBQVM7QUFBQSxNQUNQLFFBQVEsQ0FBQyxlQUFlO0FBQUEsTUFDeEIsT0FBTztBQUFBLFFBQ0w7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLFVBQVU7QUFBQSxRQUN0QztBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGtCQUFrQjtBQUFBLFFBQzlDO0FBQUEsUUFDQTtBQUFBLFVBQ0UsTUFBTTtBQUFBLFVBQ04sYUFBYSxLQUFLLFFBQVEsYUFBYTtBQUFBLFFBQ3pDO0FBQUEsUUFDQTtBQUFBLFVBQ0UsTUFBTTtBQUFBLFVBQ04sYUFBYSxLQUFLLFFBQVEsaUJBQWlCO0FBQUEsUUFDN0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxjQUFjO0FBQUEsUUFDMUM7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxzQkFBc0I7QUFBQSxRQUNsRDtBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGNBQWM7QUFBQSxRQUMxQztBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGlCQUFpQjtBQUFBLFFBQzdDO0FBQUEsUUFDQTtBQUFBLFVBQ0UsTUFBTTtBQUFBLFVBQ04sYUFBYSxLQUFLLFFBQVEsaUJBQWlCO0FBQUEsUUFDN0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxlQUFlO0FBQUEsUUFDM0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxlQUFlO0FBQUEsUUFDM0M7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxjQUFjO0FBQUEsUUFDMUM7QUFBQSxRQUNBO0FBQUEsVUFDRSxNQUFNO0FBQUEsVUFDTixhQUFhLEtBQUssUUFBUSxvQkFBb0I7QUFBQSxRQUNoRDtBQUFBLFFBQ0E7QUFBQSxVQUNFLE1BQU07QUFBQSxVQUNOLGFBQWEsS0FBSyxRQUFRLGFBQWE7QUFBQSxRQUN6QztBQUFBLE1BQ0Y7QUFBQSxJQUNGO0FBQUEsRUFDRjtBQUNGLENBQUM7IiwKICAibmFtZXMiOiBbXQp9Cg==

From 1cbe1b4a0b9bbc3c89e0e68e9e8bb1404db7f043 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 11 Oct 2024 16:47:21 +0100
Subject: [PATCH 248/455] Lint

---
 .../backend/DataTable/modals/CreateEditColumn.svelte           | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 79f80974ac5..ffd064fc944 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -49,13 +49,12 @@
   import OptionsEditor from "./OptionsEditor.svelte"
   import { isEnabled } from "helpers/featureFlags"
   import { getUserBindings } from "dataBinding"
-  import { makePropSafe as safe } from "@budibase/string-templates"
 
   export let field
 
   const dispatch = createEventDispatcher()
   const { dispatch: gridDispatch, rows } = getContext("grid")
-  const SafeID = `${safe("user")}.${safe("_id")}`
+  const SafeID = `${makePropSafe("user")}.${makePropSafe("_id")}`
   const SingleUserDefault = `{{ ${SafeID} }}`
   const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}`
 

From 02f8233a74d2e14f0950c31a38116c45ce2dc9c1 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Sun, 13 Oct 2024 20:09:25 +0100
Subject: [PATCH 249/455] lint and test

---
 packages/pro                                  |  2 +-
 .../rowProcessor/tests/utils.spec.ts          | 75 ++++++++++++++++++-
 .../types/src/documents/app/table/schema.ts   |  2 +-
 3 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/packages/pro b/packages/pro
index 1bce3082781..9348817b6f6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1bce30827813e135ee414141fdf51e45236b8f3c
+Subproject commit 9348817b6f6b3fae87547d9eee7ca533da10f13c
diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
index 49dd05910d2..41f9befc1a3 100644
--- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
@@ -1,11 +1,29 @@
-import { fixAutoColumnSubType } from "../utils"
+import { fixAutoColumnSubType, processAIColumns } from "../utils"
 import { AutoFieldDefaultNames } from "../../../constants"
 import {
+  AIOperationEnum,
   AutoFieldSubType,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   RelationshipType,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
+import { generator } from "@budibase/backend-core/tests"
+
+const buildPromptMock = jest.fn()
+
+jest.mock("@budibase/pro", () => ({
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => "response from LLM"),
+        buildPromptFromAIOperation: buildPromptMock,
+      }),
+    },
+  },
+}))
 
 describe("rowProcessor utility", () => {
   describe("fixAutoColumnSubType", () => {
@@ -60,4 +78,59 @@ describe("rowProcessor utility", () => {
       expect(fixAutoColumnSubType(schema)).toEqual(schema)
     })
   })
+
+  describe("processAIColumns", () => {
+    it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
+      const table: Table = {
+        _id: generator.guid(),
+        name: "AITestTable",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          product: {
+            type: FieldType.STRING,
+            name: "product",
+            constraints: {
+              presence: true,
+              type: "string",
+            },
+          },
+          aicol: {
+            type: FieldType.AI,
+            name: "aicol",
+            operation: AIOperationEnum.PROMPT,
+            prompt: "Translate '{{ product }}' into German",
+          },
+        },
+      }
+
+      const inputRows = [
+        {
+          product: "Car Battery",
+        },
+      ]
+
+      const result = await processAIColumns(table, inputRows, {
+        contextRows: inputRows,
+      })
+      expect(buildPromptMock).toHaveBeenCalledWith({
+        row: {
+          product: "Car Battery",
+        },
+        schema: {
+          name: "aicol",
+          operation: "PROMPT",
+          prompt: "Translate 'Car Battery' into German",
+          type: "ai",
+        },
+      })
+      expect(result).toEqual([
+        {
+          aicol: "response from LLM",
+          product: "Car Battery",
+        },
+      ])
+    })
+  })
 })
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index c62921a3ea2..bb3da3f8446 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -119,7 +119,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
 
 export interface AIFieldMetadata extends BaseFieldSchema {
   type: FieldType.AI
-  formula: string
+  // formula: string
   operation: AIOperationEnum
   columns?: string[]
   column?: string

From 0624a0b22052aa4aa62984842b6d5133e1bd73d7 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 14 Oct 2024 09:17:16 +0100
Subject: [PATCH 250/455] Removed unused function reference

---
 .../src/components/grid/cells/RelationshipCell.svelte            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index 6313681f364..92186f19b1a 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -198,7 +198,6 @@
 
   // Toggles whether a row is included in the relationship or not
   const toggleRow = async row => {
-    hideRelationshipFields()
     if (fieldValue?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
       // row to be the same position if possible

From 5798b42454edb868ab833abe6574e55dc606e4e5 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 14 Oct 2024 09:24:31 +0100
Subject: [PATCH 251/455] updating submodule references

---
 packages/account-portal | 2 +-
 packages/pro            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index 558a32dfd1f..4094517c07a 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 558a32dfd1f55bd894804a503e7e1090937df88c
+Subproject commit 4094517c07ad2b957895b801a7e1beaa7d894c4a
diff --git a/packages/pro b/packages/pro
index 9348817b6f6..fc4c7f49251 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 9348817b6f6b3fae87547d9eee7ca533da10f13c
+Subproject commit fc4c7f4925139af078480217965c3d6338dc0a7f

From cf8f7db36db80d6d4f4572949afda019e023cff9 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 14 Oct 2024 11:06:55 +0100
Subject: [PATCH 252/455] PR comments

---
 .../common/AIFieldConfiguration.svelte        |  2 +-
 .../table/[tableId]/[viewId]/index.svelte     |  3 ++-
 .../data/table/[tableId]/index.svelte         |  3 ++-
 .../src/components/grid/layout/Grid.svelte    |  2 ++
 .../grid/overlays/MenuOverlay.svelte          | 21 +++++++++++--------
 .../types/src/documents/app/table/schema.ts   |  1 -
 6 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte
index 0a8a2850c58..cb7ac77b755 100644
--- a/packages/builder/src/components/common/AIFieldConfiguration.svelte
+++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte
@@ -18,7 +18,7 @@
   export let schema
   export let aiField = {}
 
-  $: OperationField = OperationFields[aiField.operation] || null
+  $: OperationField = OperationFields[aiField.operation]
   $: schemaWithoutRelations = Object.keys(schema).filter(
     key => schema[key].type !== "link"
   )
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index eb5e7eb95d4..448fd47ba0c 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "stores/builder"
-  import { admin, themeStore } from "stores/portal"
+  import { admin, themeStore, licensing } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import { notifications } from "@budibase/bbui"
@@ -49,6 +49,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
+  licensing={$licensing}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 1ee96bf624a..3357f7ec981 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -7,7 +7,7 @@
     appStore,
     rowActions,
   } from "stores/builder"
-  import { themeStore, admin } from "stores/portal"
+  import { themeStore, admin, licensing } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
@@ -125,6 +125,7 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
+    licensing={$licensing}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 5f6de670df1..6cf4bb43e89 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -48,6 +48,7 @@
   export let darkMode = false
   export let isCloud = null
   export let rowConditions = null
+  export let licensing = null
 
   // Unique identifier for DOM nodes inside this instance
   const gridID = `grid-${Math.random().toString().slice(2)}`
@@ -104,6 +105,7 @@
     buttonsCollapsedText,
     darkMode,
     isCloud,
+    licensing,
     rowConditions,
   })
 
diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
index ed2846eb042..57a57518df7 100644
--- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
@@ -26,6 +26,7 @@
 
   $: style = makeStyle($menu)
   $: isNewRow = $focusedRowId === NewRowID
+  $: budibaseAIEnabled = $config.licensing?.budibaseAIEnabled || $config.licensing?.customAIConfigsEnabled
 
   const makeStyle = menu => {
     return `left:${menu.left}px; top:${menu.top}px;`
@@ -168,15 +169,17 @@
           >
             Delete row
           </MenuItem>
-          <MenuItem
-            icon="MagicWand"
-            disabled={isNewRow ||
-              !$focusedRow?._rev ||
-              !$hasBudibaseIdentifiers}
-            on:click={generateAIColumns}
-          >
-            Generate AI Columns
-          </MenuItem>
+          {#if budibaseAIEnabled}
+            <MenuItem
+              icon="MagicWand"
+              disabled={isNewRow ||
+                !$focusedRow?._rev ||
+                !$hasBudibaseIdentifiers}
+              on:click={generateAIColumns}
+            >
+              Generate AI Columns
+            </MenuItem>
+          {/if}
         {/if}
       </Menu>
     </GridPopover>
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index bb3da3f8446..c99ac2403c8 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -119,7 +119,6 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
 
 export interface AIFieldMetadata extends BaseFieldSchema {
   type: FieldType.AI
-  // formula: string
   operation: AIOperationEnum
   columns?: string[]
   column?: string

From 37e601254cb61403977df16a8f533ca4e3cdb9d4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 14 Oct 2024 12:20:53 +0100
Subject: [PATCH 253/455] Fix crash when many-type related column value is
 non-array

---
 .../frontend-core/src/utils/relatedColumns.js    | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index f625e83e952..6e7968f70c8 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -24,9 +24,9 @@ const columnTypeManyParser = {
       return parsed
     }
 
-    return value?.map(v => parseDate(v))
+    return value.map(v => parseDate(v))
   },
-  [FieldType.BOOLEAN]: value => value?.map(v => !!v),
+  [FieldType.BOOLEAN]: value => value.map(v => !!v),
   [FieldType.BB_REFERENCE_SINGLE]: value => [
     ...new Map(value.map(i => [i._id, i])).values(),
   ],
@@ -80,14 +80,10 @@ export function getRelatedTableValues(row, field, fromField) {
     result = row[field.related.field]?.[0]?.[field.related.subField]
   } else {
     const parser = columnTypeManyParser[field.type] || (value => value)
-
-    result = parser(
-      row[field.related.field]
-        ?.flatMap(r => r[field.related.subField])
-        ?.filter(i => i !== undefined && i !== null),
-      field
-    )
-
+    const value = row[field.related.field]
+      ?.flatMap(r => r[field.related.subField])
+      ?.filter(i => i !== undefined && i !== null)
+    result = parser(value || [], field)
     if (
       [
         FieldType.STRING,

From 4d460c2a46860da7ad7b42fc264d48465e1ec82f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 14 Oct 2024 12:58:15 +0100
Subject: [PATCH 254/455] Don't mutate views when transforming queryUI to make
 API happy

---
 packages/builder/src/stores/builder/viewsV2.js     | 14 ++++++++++----
 .../components/grid/stores/datasources/viewV2.js   |  9 ++++++---
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index b1db8165e2c..7c0a940c1b9 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -6,16 +6,22 @@ import { dataFilters } from "@budibase/shared-core"
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
   if (view?.query) {
-    view.queryUI = view.query
-    view.query = dataFilters.buildQuery(view.query)
+    return {
+      ...view,
+      queryUI: view.query,
+      query: dataFilters.buildQuery(view.query),
+    }
   }
   return view
 }
 
 function convertToSearchFilterGroup(view) {
   if (view?.queryUI) {
-    view.query = view.queryUI
-    delete view.queryUI
+    return {
+      ...view,
+      query: view.queryUI,
+      queryUI: undefined,
+    }
   }
   return view
 }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 84d5eb153ee..56a7479b44c 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -3,9 +3,12 @@ import { dataFilters } from "@budibase/shared-core"
 
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
-  if (view.query) {
-    view.queryUI = view.query
-    view.query = dataFilters.buildQuery(view.query)
+  if (view?.query) {
+    return {
+      ...view,
+      queryUI: view.query,
+      query: dataFilters.buildQuery(view.query),
+    }
   }
   return view
 }

From 5f59bf9f237648e840b6fbf5a3c6a56a7886f2ae Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 14 Oct 2024 13:05:38 +0100
Subject: [PATCH 255/455] Fix leftover reference to deleted code

---
 .../src/components/grid/cells/RelationshipCell.svelte            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index 6313681f364..92186f19b1a 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -198,7 +198,6 @@
 
   // Toggles whether a row is included in the relationship or not
   const toggleRow = async row => {
-    hideRelationshipFields()
     if (fieldValue?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
       // row to be the same position if possible

From 30215ea9be83be560981c7015f31b18d3b7fce97 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 14 Oct 2024 18:38:34 +0100
Subject: [PATCH 256/455] tests, pr comments

---
 packages/backend-core/src/constants/misc.ts   |  1 +
 .../tests/core/utilities/mocks/licenses.ts    |  8 ++
 .../grid/overlays/MenuOverlay.svelte          |  4 +-
 packages/pro                                  |  2 +-
 .../server/src/api/routes/tests/row.spec.ts   | 98 +++++++++++++++----
 .../src/api/routes/tests/search.spec.ts       | 88 ++++++++++++++++-
 .../server/src/tests/utilities/structures.ts  |  7 ++
 7 files changed, 187 insertions(+), 21 deletions(-)

diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts
index aee099e10a9..e2fd975e409 100644
--- a/packages/backend-core/src/constants/misc.ts
+++ b/packages/backend-core/src/constants/misc.ts
@@ -28,6 +28,7 @@ export enum Config {
   OIDC = "oidc",
   OIDC_LOGOS = "logos_oidc",
   SCIM = "scim",
+  AI = "AI",
 }
 
 export const MIN_VALID_DATE = new Date(-2147483647000)
diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
index bc9a3b635c0..5ba6fb36a13 100644
--- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
@@ -102,6 +102,14 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
+export const useBudibaseAI = () => {
+  return useFeature(Feature.BUDIBASE_AI)
+}
+
+export const useAICustomConfigs = () => {
+  return useFeature(Feature.AI_CUSTOM_CONFIGS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {
diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
index 57a57518df7..ba023cf2ca7 100644
--- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
@@ -26,7 +26,9 @@
 
   $: style = makeStyle($menu)
   $: isNewRow = $focusedRowId === NewRowID
-  $: budibaseAIEnabled = $config.licensing?.budibaseAIEnabled || $config.licensing?.customAIConfigsEnabled
+  $: budibaseAIEnabled =
+    $config.licensing?.budibaseAIEnabled ||
+    $config.licensing?.customAIConfigsEnabled
 
   const makeStyle = menu => {
     return `left:${menu.left}px; top:${menu.top}px;`
diff --git a/packages/pro b/packages/pro
index fc4c7f49251..a55a58fc962 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit fc4c7f4925139af078480217965c3d6338dc0a7f
+Subproject commit a55a58fc96291ae021e61ad369a2077992fddfcd
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 52220694601..843abd43c04 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1,10 +1,6 @@
 import * as setup from "./utilities"
 
-import {
-  DatabaseName,
-  getDatasource,
-  knexClient,
-} from "../../../integrations/tests/utils"
+import { DatabaseName, knexClient } from "../../../integrations/tests/utils"
 
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
@@ -12,36 +8,37 @@ import { outputProcessing } from "../../../utilities/rowProcessor"
 import {
   context,
   InternalTable,
+  setEnv as setCoreEnv,
   tenancy,
   withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
 } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AIOperationEnum,
   AttachmentFieldMetadata,
   AutoFieldSubType,
+  BBReferenceFieldSubType,
   Datasource,
   DateFieldMetadata,
   DeleteRow,
+  FeatureFlag,
   FieldSchema,
   FieldType,
-  BBReferenceFieldSubType,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
+  JsonFieldSubType,
   NumberFieldMetadata,
   QuotaUsageType,
+  RelationSchemaField,
   RelationshipType,
   Row,
+  RowExportFormat,
   SaveTableRequest,
   StaticQuotaName,
   Table,
+  TableSchema,
   TableSourceType,
   UpdatedRowEventEmitter,
-  TableSchema,
-  JsonFieldSubType,
-  RowExportFormat,
-  FeatureFlag,
-  RelationSchemaField,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import _, { merge } from "lodash"
@@ -51,6 +48,18 @@ import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => `Mock LLM Response`),
+        buildPromptFromAIOperation: jest.fn(),
+      }),
+    },
+  },
+}))
+
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 interface WaitOptions {
@@ -77,13 +86,13 @@ async function waitForEvent(
 }
 
 describe.each([
-  ["lucene", undefined],
+  // ["lucene", undefined],
   ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
   const isLucene = providerType === "lucene"
@@ -1986,6 +1995,7 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
+          [FieldType.AI]: undefined, // generated field
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2016,6 +2026,7 @@ describe.each([
             url: expect.any(String),
           }),
           [FieldType.FORMULA]: fullSchema[FieldType.FORMULA].formula,
+          [FieldType.AI]: fullSchema[FieldType.AI].prompt,
           [FieldType.AUTO]: expect.any(Number),
           [FieldType.JSON]: rowValues[FieldType.JSON],
           [FieldType.INTERNAL]: rowValues[FieldType.INTERNAL],
@@ -2880,6 +2891,57 @@ describe.each([
       )
     })
 
+  isSqs &&
+    describe("AI fields", () => {
+      let table: Table
+
+      beforeAll(async () => {
+        mocks.licenses.useBudibaseAI()
+        mocks.licenses.useAICustomConfigs()
+        table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              ai: {
+                name: "ai",
+                type: FieldType.AI,
+                operation: AIOperationEnum.PROMPT,
+                prompt: "Convert the following to German: '{{ product }}'",
+              },
+              product: {
+                name: "product",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+
+        await config.api.row.save(table._id!, {
+          product: generator.word(),
+        })
+      })
+
+      afterAll(() => {
+        jest.unmock("@budibase/pro")
+      })
+
+      it("should be able to save a row with an AI column", async () => {
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows.length).toBe(1)
+        expect(rows[0].ai).toEqual("Mock LLM Response")
+      })
+
+      it("should be able to update a row with an AI column", async () => {
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows.length).toBe(1)
+        await config.api.row.save(table._id!, {
+          product: generator.word(),
+          ...rows[0],
+        })
+        expect(rows.length).toBe(1)
+        expect(rows[0].ai).toEqual("Mock LLM Response")
+      })
+    })
+
   describe("Formula fields", () => {
     let table: Table
     let otherTable: Table
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 110899e2928..fb8bc084fde 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -17,6 +17,7 @@ import {
 
 import * as setup from "./utilities"
 import {
+  AIOperationEnum,
   AutoFieldSubType,
   BBReferenceFieldSubType,
   Datasource,
@@ -41,11 +42,23 @@ import tk from "timekeeper"
 import { encodeJSBinding } from "@budibase/string-templates"
 import { dataFilters } from "@budibase/shared-core"
 import { Knex } from "knex"
-import { generator, structures } from "@budibase/backend-core/tests"
+import { generator, structures, mocks } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 import { generateRowIdField } from "../../../integrations/utils"
 import { cloneDeep } from "lodash/fp"
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => `Mock LLM Response`),
+        buildPromptFromAIOperation: jest.fn(),
+      }),
+    },
+  },
+}))
+
 describe.each([
   ["in-memory", undefined],
   ["lucene", undefined],
@@ -1606,6 +1619,79 @@ describe.each([
         })
       })
 
+    isSqs &&
+      describe("AI Column", () => {
+        const UNEXISTING_AI_COLUMN = "Real LLM Response"
+
+        beforeAll(async () => {
+          mocks.licenses.useBudibaseAI()
+          mocks.licenses.useAICustomConfigs()
+
+          tableOrViewId = await createTableOrView({
+            product: { name: "product", type: FieldType.STRING },
+            ai: {
+              name: "AI",
+              type: FieldType.AI,
+              operation: AIOperationEnum.PROMPT,
+              prompt: "Translate '{{ product }}' into German",
+            },
+          })
+
+          await createRows([{ product: "Big Mac" }, { product: "McCrispy" }])
+        })
+
+        describe("equal", () => {
+          it("successfully finds rows based on AI column", async () => {
+            await expectQuery({
+              equal: { ai: "Mock LLM Response" },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+
+          it("fails to find nonexistent row", async () => {
+            await expectQuery({
+              equal: { ai: UNEXISTING_AI_COLUMN },
+            }).toFindNothing()
+          })
+        })
+
+        describe("notEqual", () => {
+          it("Returns nothing when searching notEqual on the mock AI response", async () => {
+            await expectQuery({
+              notEqual: { ai: "Mock LLM Response" },
+            }).toContainExactly([])
+          })
+
+          it("return all when requesting non-existing response", async () => {
+            await expectQuery({
+              notEqual: { ai: "Real LLM Response" },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+        })
+
+        describe("oneOf", () => {
+          it("successfully finds a row", async () => {
+            await expectQuery({
+              oneOf: { ai: ["Mock LLM Response", "Other LLM Response"] },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+
+          it("fails to find nonexistent row", async () => {
+            await expectQuery({
+              oneOf: { ai: ["Whopper"] },
+            }).toFindNothing()
+          })
+        })
+      })
+
     describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
       beforeAll(async () => {
         tableOrViewId = await createTableOrView({
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index 72cd31e383c..16803f19cd3 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -7,6 +7,7 @@ import {
   TRIGGER_DEFINITIONS,
 } from "../../automations"
 import {
+  AIOperationEnum,
   Automation,
   AutomationActionStepId,
   AutomationResults,
@@ -666,6 +667,12 @@ export function fullSchemaWithoutLinks({
         presence: allRequired,
       },
     },
+    [FieldType.AI]: {
+      name: "ai",
+      type: FieldType.AI,
+      operation: AIOperationEnum.PROMPT,
+      prompt: "Translate this into German :'{{ product }}'",
+    },
     [FieldType.BARCODEQR]: {
       name: "barcodeqr",
       type: FieldType.BARCODEQR,

From 95d8ad854c0e27621f0d5f9340281f27a253f076 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 14 Oct 2024 23:36:33 +0100
Subject: [PATCH 257/455] lint and types

---
 packages/server/src/api/routes/tests/datasource.spec.ts | 6 ------
 packages/server/src/api/routes/tests/viewV2.spec.ts     | 8 ++++----
 packages/server/src/utilities/rowProcessor/utils.ts     | 9 +++++----
 packages/types/src/documents/app/table/schema.ts        | 2 +-
 4 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts
index 768dfb32ded..75452531813 100644
--- a/packages/server/src/api/routes/tests/datasource.spec.ts
+++ b/packages/server/src/api/routes/tests/datasource.spec.ts
@@ -343,12 +343,6 @@ describe("/datasources", () => {
             type: FieldType.FORMULA,
             formula: "any formula",
           },
-          [FieldType.AI]: {
-            name: "ai",
-            type: FieldType.AI,
-            operation: "PROMPT",
-            prompt: "test prompt",
-          },
           [FieldType.BARCODEQR]: {
             name: "barcodeqr",
             type: FieldType.BARCODEQR,
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 7979fac5551..dfd4f50bd1c 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -206,7 +206,7 @@ describe.each([
               visible: false,
               icon: "ic",
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         }
 
         const createdView = await config.api.viewV2.create(newView)
@@ -250,7 +250,7 @@ describe.each([
               name: "Category",
               type: FieldType.STRING,
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         }
 
         await config.api.viewV2.create(newView, {
@@ -1044,7 +1044,7 @@ describe.each([
               visible: false,
               icon: "ic",
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         })
 
         expect(updatedView).toEqual({
@@ -1078,7 +1078,7 @@ describe.each([
                 name: "Category",
                 type: FieldType.STRING,
               },
-            } as Record<string, FieldSchema>,
+            } as ViewV2Schema,
           },
           {
             status: 200,
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 1df2ff2fec4..bf1fe6f4d3d 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -9,7 +9,7 @@ import {
   AutoFieldSubType,
   FieldType,
   OperationFieldTypeEnum,
-  AIFieldMetadata,
+  AIOperationEnum,
 } from "@budibase/types"
 import { OperationFields } from "@budibase/shared-core"
 import tracer from "dd-trace"
@@ -122,10 +122,11 @@ export async function processAIColumns<T extends Row | Row[]>(
 
           // Check if the type is bindable and pass through HBS if so
           const operationField =
-            OperationFields[(schema as AIFieldMetadata).operation]
+            OperationFields[schema.operation as AIOperationEnum]
           for (const key in schema) {
-            const fieldType = operationField[key]
+            const fieldType = operationField[key as keyof typeof operationField]
             if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
+              // @ts-ignore
               schema[key] = processStringSync(schema[key], contextRow)
             }
           }
@@ -134,7 +135,7 @@ export async function processAIColumns<T extends Row | Row[]>(
 
           return tracer.trace("processAIColumn", {}, async span => {
             span?.addTags({ table_id: table._id, column })
-            const llmResponse = await llm.run(prompt)
+            const llmResponse = await llm.run(prompt!)
             return {
               ...row,
               [column]: llmResponse,
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 1390e1ddbd3..45d0aa78d2a 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -122,7 +122,7 @@ export interface AIFieldMetadata extends BaseFieldSchema {
   operation: AIOperationEnum
   columns?: string[]
   column?: string
-  categories?: string
+  categories?: string[]
   prompt?: string
   language?: string
 }

From 9d5e60669d092d53efe97c5b51a8808cae5914c3 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 14 Oct 2024 23:49:46 +0100
Subject: [PATCH 258/455] Remove AI column from row export

---
 packages/pro                                     | 2 +-
 packages/server/src/api/routes/tests/row.spec.ts | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index a55a58fc962..69ad5e4f92b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit a55a58fc96291ae021e61ad369a2077992fddfcd
+Subproject commit 69ad5e4f92b4749cd1609c0bb7feb2f3326d239c
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index f6326dfac89..f1cd12a97a6 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2130,7 +2130,6 @@ describe.each([
             url: expect.any(String),
           }),
           [FieldType.FORMULA]: fullSchema[FieldType.FORMULA].formula,
-          [FieldType.AI]: fullSchema[FieldType.AI].prompt,
           [FieldType.AUTO]: expect.any(Number),
           [FieldType.JSON]: rowValues[FieldType.JSON],
           [FieldType.INTERNAL]: rowValues[FieldType.INTERNAL],

From b563acbc1c0fe4208a2cb254eea3e878426f9ca7 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 14 Oct 2024 23:58:41 +0100
Subject: [PATCH 259/455] remove AI field from assertion

---
 packages/server/src/api/routes/tests/row.spec.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index f1cd12a97a6..855863c5142 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2099,7 +2099,6 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
-          [FieldType.AI]: undefined, // generated field
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),

From c19c61ee9a4a85058c2824384c2e707589c94fd8 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 15 Oct 2024 00:52:48 +0100
Subject: [PATCH 260/455] fix csv test

---
 packages/server/src/api/routes/tests/row.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 855863c5142..5782a65ac5b 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2099,6 +2099,7 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
+          [FieldType.AI]: undefined, // generated field
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2202,6 +2203,7 @@ describe.each([
               expectedRowData["bb_reference_single"].sample,
               false
             ),
+            ai: null
           },
         ])
       })

From 0fe61f36bba0edd512a020a32d5c44adbe3d36a5 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 15 Oct 2024 00:57:01 +0100
Subject: [PATCH 261/455] lint

---
 packages/server/src/api/routes/tests/row.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 5782a65ac5b..66e931d0ab0 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2203,7 +2203,7 @@ describe.each([
               expectedRowData["bb_reference_single"].sample,
               false
             ),
-            ai: null
+            ai: null,
           },
         ])
       })

From 1d2d096ab0cfc41a5c72c02dc68ce5473d2ccad8 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 15 Oct 2024 10:05:03 +0100
Subject: [PATCH 262/455] PR comments

---
 .../table/[tableId]/[viewId]/index.svelte     |  2 +-
 .../data/table/[tableId]/index.svelte         |  3 ++-
 .../src/components/app/GridBlock.svelte       |  2 ++
 packages/client/src/stores/features.js        |  6 +++++
 .../src/components/FilterField.svelte         |  3 ++-
 .../src/components/grid/layout/Grid.svelte    |  4 ++--
 .../grid/overlays/MenuOverlay.svelte          | 16 ++++++-------
 .../src/components/grid/stores/columns.js     |  1 +
 .../src/components/grid/stores/rows.js        | 24 -------------------
 packages/pro                                  |  2 +-
 10 files changed, 24 insertions(+), 39 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 448fd47ba0c..9b0fe8beeae 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -49,7 +49,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
-  licensing={$licensing}
+  aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index 3357f7ec981..69a23f3a0e9 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -125,7 +125,8 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
-    licensing={$licensing}
+    aiEnabled={$licensing.budibaseAIEnabled ||
+      $licensing.customAIConfigsEnabled}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 543bade3827..c6b08e3869a 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -3,6 +3,7 @@
   // because it functions similarly to one
   import { getContext, onMount } from "svelte"
   import { get, derived, readable } from "svelte/store"
+  import { featuresStore } from "stores"
   import { Grid } from "@budibase/frontend-core"
 
   // table is actually any datasource, but called table for legacy compatibility
@@ -186,6 +187,7 @@
     {buttonsCollapsed}
     {buttonsCollapsedText}
     isCloud={$environmentStore.cloud}
+    aiEnabled={$featuresStore.aiEnabled}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>
diff --git a/packages/client/src/stores/features.js b/packages/client/src/stores/features.js
index 2bdc2bec1a9..86c16bf3446 100644
--- a/packages/client/src/stores/features.js
+++ b/packages/client/src/stores/features.js
@@ -2,6 +2,7 @@ import { derived } from "svelte/store"
 import { appStore } from "./app"
 import { authStore } from "./auth"
 import { Constants } from "@budibase/frontend-core"
+import { Feature } from "@budibase/types"
 
 const createFeaturesStore = () => {
   return derived([authStore, appStore], ([$authStore, $appStore]) => {
@@ -33,8 +34,13 @@ const createFeaturesStore = () => {
       }
     }
 
+    const license = getUserLicense()
+
     return {
       logoEnabled: isFreePlan(),
+      aiEnabled:
+        license?.features?.includes(Feature.AI_CUSTOM_CONFIGS) ||
+        license?.features?.includes(Feature.BUDIBASE_AI),
     }
   })
 }
diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte
index c763194d693..0fff7744468 100644
--- a/packages/frontend-core/src/components/FilterField.svelte
+++ b/packages/frontend-core/src/components/FilterField.svelte
@@ -35,6 +35,7 @@
   $: drawerValue = toDrawerValue(fieldValue)
   $: isJS = isJSBinding(fieldValue)
   $: fieldIsValid = isValid(fieldValue)
+  $: console.log(filter)
 
   const getFieldOptions = field => {
     const schema = schemaFields.find(x => x.name === field)
@@ -174,7 +175,7 @@
         />
       {:else}
         <div>
-          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
+          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)}
             <Input
               disabled={filter.noValue}
               value={readableValue}
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 6cf4bb43e89..fcaf9724c3d 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -48,7 +48,7 @@
   export let darkMode = false
   export let isCloud = null
   export let rowConditions = null
-  export let licensing = null
+  export let aiEnabled = false
 
   // Unique identifier for DOM nodes inside this instance
   const gridID = `grid-${Math.random().toString().slice(2)}`
@@ -105,7 +105,7 @@
     buttonsCollapsedText,
     darkMode,
     isCloud,
-    licensing,
+    aiEnabled,
     rowConditions,
   })
 
diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
index ba023cf2ca7..fab9f4b4802 100644
--- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
@@ -1,5 +1,6 @@
 <script>
   import { Menu, MenuItem, Helpers } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import { getContext } from "svelte"
   import { NewRowID } from "../lib/constants"
   import GridPopover from "./GridPopover.svelte"
@@ -26,9 +27,9 @@
 
   $: style = makeStyle($menu)
   $: isNewRow = $focusedRowId === NewRowID
-  $: budibaseAIEnabled =
-    $config.licensing?.budibaseAIEnabled ||
-    $config.licensing?.customAIConfigsEnabled
+  $: hasAIColumns = $visibleColumns.some(
+    col => col.schema.type === FieldType.AI
+  )
 
   const makeStyle = menu => {
     return `left:${menu.left}px; top:${menu.top}px;`
@@ -58,9 +59,8 @@
   }
 
   const generateAIColumns = async () => {
-    await rows.actions.saveRow({ rowId: $focusedRowId })
-    await rows.actions.refreshData()
     menu.actions.close()
+    await rows.actions.applyRowChanges({ rowId: $focusedRowId })
     $notifications.success("Generated AI columns")
   }
 </script>
@@ -171,12 +171,10 @@
           >
             Delete row
           </MenuItem>
-          {#if budibaseAIEnabled}
+          {#if $config.aiEnabled}
             <MenuItem
               icon="MagicWand"
-              disabled={isNewRow ||
-                !$focusedRow?._rev ||
-                !$hasBudibaseIdentifiers}
+              disabled={isNewRow || !hasAIColumns}
               on:click={generateAIColumns}
             >
               Generate AI Columns
diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 4823f2159c3..f23a17f14c2 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -109,6 +109,7 @@ export const createActions = context => {
       column.schema.autocolumn ||
       column.schema.disabled ||
       column.schema.type === "formula" ||
+      column.schema.type === "ai" ||
       column.schema.readonly
     )
   }
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 94fbdba5fac..4cfaa1cfd59 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -465,29 +465,6 @@ export const createActions = context => {
     return true
   }
 
-  const saveRow = async ({ rowId }) => {
-    const $rowLookupMap = get(rowLookupMap)
-    const row = $rowLookupMap[rowId]
-    if (row == null) {
-      return
-    }
-    let savedRow
-
-    // Save row
-    try {
-      const newRow = cleanRow(row)
-      savedRow = await datasource.actions.updateRow(newRow)
-
-      if (savedRow?.id) {
-        // Handle users table edge case
-        await refreshRow(savedRow.id)
-      }
-    } catch (error) {
-      handleValidationError(rowId, error)
-    }
-    return savedRow
-  }
-
   // Saves any pending changes to a row, as well as any additional changes
   // specified
   const applyRowChanges = async ({
@@ -708,7 +685,6 @@ export const createActions = context => {
       ...rows,
       actions: {
         addRow,
-        saveRow,
         duplicateRow,
         bulkDuplicate,
         updateValue,
diff --git a/packages/pro b/packages/pro
index 69ad5e4f92b..61391549614 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 69ad5e4f92b4749cd1609c0bb7feb2f3326d239c
+Subproject commit 61391549614b5ac153f267633d0aaea9b07f05c5

From 109c586b84907a7f5591b55c6115b93f89fac31e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 15 Oct 2024 10:11:46 +0100
Subject: [PATCH 263/455] remove log

---
 packages/frontend-core/src/components/FilterField.svelte | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte
index 0fff7744468..2f034ddf3ec 100644
--- a/packages/frontend-core/src/components/FilterField.svelte
+++ b/packages/frontend-core/src/components/FilterField.svelte
@@ -35,7 +35,6 @@
   $: drawerValue = toDrawerValue(fieldValue)
   $: isJS = isJSBinding(fieldValue)
   $: fieldIsValid = isValid(fieldValue)
-  $: console.log(filter)
 
   const getFieldOptions = field => {
     const schema = schemaFields.find(x => x.name === field)

From 0a1912e42c2119fe5e2eea122729bcfda5d92579 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 16 Oct 2024 11:56:52 +0100
Subject: [PATCH 264/455] Don't extend dataprovider queries when no extensions
 are in use

---
 packages/client/src/components/app/DataProvider.svelte | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index 7b5187ffb6e..054264a9e1f 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -126,6 +126,9 @@
   }
 
   const extendQuery = (defaultQuery, extensions) => {
+    if (!Object.keys(extensions).length) {
+      return
+    }
     const extended = {
       [LogicalOperator.AND]: {
         conditions: [

From 51c95058ee8b3096d13d479d8032d913fe4b1cc2 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 16 Oct 2024 12:04:19 +0100
Subject: [PATCH 265/455] Fix the drawerShow event not being broadcast,
 preventing component popovers from hiding properly when opening drawers

---
 .../design/settings/controls/FilterEditor/FilterEditor.svelte    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
index d6f1732e647..34c317e8651 100644
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
@@ -59,6 +59,7 @@
   bind:this={drawer}
   title="Filtering"
   on:drawerHide
+  on:drawerShow
   on:drawerShow={() => {
     // Reset to the currently available value.
     localFilters = Helpers.cloneDeep(value)

From 46b9877455e638d29071dd763a8e6947d738a8a1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 16 Oct 2024 15:15:23 +0100
Subject: [PATCH 266/455] Return default query

---
 packages/client/src/components/app/DataProvider.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index 054264a9e1f..e6629aa3f3b 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -127,7 +127,7 @@
 
   const extendQuery = (defaultQuery, extensions) => {
     if (!Object.keys(extensions).length) {
-      return
+      return defaultQuery
     }
     const extended = {
       [LogicalOperator.AND]: {

From 477bdf22e92373437e7945c30c45cbc7b352801d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 16 Oct 2024 15:18:37 +0100
Subject: [PATCH 267/455] Fixing some issues with finding roles.

---
 packages/backend-core/src/security/roles.ts   | 21 +++---
 packages/server/src/api/controllers/role.ts   | 10 ++-
 .../server/src/api/routes/tests/role.spec.ts  | 75 ++++++++++++++++---
 packages/server/src/middleware/currentapp.ts  | 12 ++-
 4 files changed, 91 insertions(+), 27 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 41a55c55bdb..03660b313d6 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -235,7 +235,7 @@ export function findRole(
   roleId: string,
   roles: RoleDoc[],
   opts?: { defaultPublic?: boolean }
-): RoleDoc {
+): RoleDoc | undefined {
   // built in roles mostly come from the in-code implementation,
   // but can be extended by a doc stored about them (e.g. permissions)
   let role: RoleDoc | undefined = getBuiltinRole(roleId)
@@ -249,13 +249,13 @@ export function findRole(
   if (!dbRole && !isBuiltin(roleId) && opts?.defaultPublic) {
     return cloneDeep(BUILTIN_ROLES.PUBLIC)
   }
-  if (!dbRole && (!role || Object.keys(role).length === 0)) {
-    throw new Error("Role could not be found")
-  }
+  // combine the roles
   role = Object.assign(role || {}, dbRole)
   // finalise the ID
-  role._id = getExternalRoleID(role._id!, role.version)
-  return role
+  if (role?._id) {
+    role._id = getExternalRoleID(role._id, role.version)
+  }
+  return Object.keys(role).length === 0 ? undefined : role
 }
 
 /**
@@ -268,7 +268,7 @@ export function findRole(
 export async function getRole(
   roleId: string,
   opts?: { defaultPublic?: boolean }
-): Promise<RoleDoc> {
+): Promise<RoleDoc | undefined> {
   const db = getAppDB()
   const roleList = []
   if (!isBuiltin(roleId)) {
@@ -305,7 +305,7 @@ async function getAllUserRoles(
 
   const roleIds = [userRoleId]
   const roles: RoleDoc[] = []
-  const iterateInherited = (role: RoleDoc) => {
+  const iterateInherited = (role: RoleDoc | undefined) => {
     if (!role || !role._id) {
       return
     }
@@ -335,7 +335,10 @@ async function getAllUserRoles(
   }
 
   // get all the inherited roles
-  iterateInherited(findRole(userRoleId, allRoles, opts))
+  const foundRole = findRole(userRoleId, allRoles, opts)
+  if (foundRole) {
+    iterateInherited(foundRole)
+  }
   const foundRoleIds: string[] = []
   return roles.filter(role => {
     if (role._id && !foundRoleIds.includes(role._id)) {
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index d8c9661c721..9cb7c134d6e 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -58,7 +58,12 @@ export async function fetch(ctx: UserCtx<void, FetchRolesResponse>) {
 }
 
 export async function find(ctx: UserCtx<void, FindRoleResponse>) {
-  ctx.body = await roles.getRole(ctx.params.roleId)
+  const role = await roles.getRole(ctx.params.roleId)
+  if (!role) {
+    ctx.throw(404, { message: "Role not found" })
+  } else {
+    ctx.body = role
+  }
 }
 
 export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
@@ -205,7 +210,8 @@ export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
   // If a custom role is provided in the header, filter out higher level roles
   const roleHeader = ctx.header?.[Header.PREVIEW_ROLE] as string
   if (roleHeader && !Object.keys(roles.BUILTIN_ROLE_IDS).includes(roleHeader)) {
-    const inherits = (await roles.getRole(roleHeader))?.inherits
+    const role = await roles.getRole(roleHeader)
+    const inherits = role?.inherits
     const orderedRoles = ctx.body.reverse()
     let filteredRoles = [roleHeader]
     for (let role of orderedRoles) {
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 396afeb2361..3a2ba3ce9d6 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -6,6 +6,8 @@ const { basicRole } = setup.structures
 const { BUILTIN_ROLE_IDS } = roles
 const { BuiltinPermissionID } = permissions
 
+const LOOP_ERROR = "Role inheritance contains a loop, this is not supported"
+
 describe("/roles", () => {
   let config = setup.getConfig()
 
@@ -31,6 +33,11 @@ describe("/roles", () => {
   })
 
   describe("update", () => {
+    beforeEach(async () => {
+      // Recreate the app
+      await config.init()
+    })
+
     it("updates a role", async () => {
       const role = basicRole()
       let res = await config.api.roles.save(role, {
@@ -49,22 +56,59 @@ describe("/roles", () => {
     })
 
     it("disallow loops", async () => {
-      let role1 = basicRole()
-      role1 = await config.api.roles.save(role1, {
-        status: 200,
-      })
-      let role2 = basicRole()
-      role2.inherits = [role1._id!]
-      role2 = await config.api.roles.save(role2, {
+      const role1 = await config.api.roles.save(basicRole(), {
         status: 200,
       })
-      role1.inherits = [role2._id!]
-      await config.api.roles.save(role1, {
-        status: 400,
-        body: {
-          message: "Role inheritance contains a loop, this is not supported",
+      const role2 = await config.api.roles.save(
+        {
+          ...basicRole(),
+          inherits: [role1._id!],
         },
+        {
+          status: 200,
+        }
+      )
+      await config.api.roles.save(
+        {
+          ...role1,
+          inherits: [role2._id!],
+        },
+        { status: 400, body: { message: LOOP_ERROR } }
+      )
+    })
+
+    it("disallow more complex loops", async () => {
+      let role1 = await config.api.roles.save({
+        ...basicRole(),
+        name: "role1",
+        inherits: [BUILTIN_ROLE_IDS.POWER],
+      })
+      let role2 = await config.api.roles.save({
+        ...basicRole(),
+        name: "role2",
+        inherits: [BUILTIN_ROLE_IDS.POWER, role1._id!],
+      })
+      let role3 = await config.api.roles.save({
+        ...basicRole(),
+        name: "role3",
+        inherits: [BUILTIN_ROLE_IDS.POWER, role1._id!, role2._id!],
       })
+      // go back to role1
+      role1 = await config.api.roles.save(
+        {
+          ...role1,
+          inherits: [BUILTIN_ROLE_IDS.POWER, role2._id!, role3._id!],
+        },
+        { status: 400, body: { message: LOOP_ERROR } }
+      )
+      // go back to role2
+      role2 = await config.api.roles.save(
+        {
+          ...role2,
+          inherits: [BUILTIN_ROLE_IDS.POWER, role1._id!, role3._id!],
+        },
+        { status: 400, body: { message: LOOP_ERROR } }
+      )
     })
   })
 
@@ -134,6 +178,13 @@ describe("/roles", () => {
   })
 
   describe("accessible", () => {
+    beforeAll(async () => {
+      // new app, reset roles
+      await config.init()
+      // create one custom role
+      await config.createRole()
+    })
+
     it("should be able to fetch accessible roles (with builder)", async () => {
       const res = await config.api.roles.accessible(config.defaultHeaders(), {
         status: 200,
diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts
index ad6f2afa181..d6163772980 100644
--- a/packages/server/src/middleware/currentapp.ts
+++ b/packages/server/src/middleware/currentapp.ts
@@ -59,11 +59,15 @@ export default async (ctx: UserCtx, next: any) => {
       // Ensure the role is valid by ensuring a definition exists
       try {
         if (roleHeader) {
-          await roles.getRole(roleHeader)
-          roleId = roleHeader
+          const role = await roles.getRole(roleHeader)
+          if (role) {
+            roleId = roleHeader
 
-          // Delete admin and builder flags so that the specified role is honoured
-          ctx.user = users.removePortalUserPermissions(ctx.user) as ContextUser
+            // Delete admin and builder flags so that the specified role is honoured
+            ctx.user = users.removePortalUserPermissions(
+              ctx.user
+            ) as ContextUser
+          }
         }
       } catch (error) {
         // Swallow error and do nothing

From f4428d60766d4b03427d3e8c29fa7c6d543e8186 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 16 Oct 2024 15:26:33 +0100
Subject: [PATCH 268/455] Bump pro module

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 61391549614..fc4c7f49251 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 61391549614b5ac153f267633d0aaea9b07f05c5
+Subproject commit fc4c7f4925139af078480217965c3d6338dc0a7f

From 38a57faee4de5278a0fba9432b57d3eadfb89eab Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 16 Oct 2024 15:37:12 +0100
Subject: [PATCH 269/455] Bump pro correctly

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index fc4c7f49251..1a749caba9c 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit fc4c7f4925139af078480217965c3d6338dc0a7f
+Subproject commit 1a749caba9c85aab2645e5d00db479eb53d3f80f

From edf6d95eec6a1740ae35a9cd581d781fe6df0acf Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 16 Oct 2024 15:46:03 +0100
Subject: [PATCH 270/455] Role save API returns role ID in correct format.

---
 packages/server/src/api/controllers/role.ts   |  5 ++++-
 .../server/src/api/routes/tests/role.spec.ts  | 22 +++++++++++++++----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 9cb7c134d6e..790325390b0 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -136,7 +136,10 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     role.version
   )
   role._rev = result.rev
-  ctx.body = role
+  ctx.body = {
+    ...role,
+    _id: roles.getExternalRoleID(role._id!, role.version),
+  }
 
   const devDb = context.getDevAppDB()
   const prodDb = context.getProdAppDB()
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 3a2ba3ce9d6..ef3a9191739 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -1,4 +1,9 @@
-import { roles, events, permissions } from "@budibase/backend-core"
+import {
+  roles,
+  events,
+  permissions,
+  db as dbCore,
+} from "@budibase/backend-core"
 import * as setup from "./utilities"
 import { PermissionLevel } from "@budibase/types"
 
@@ -28,7 +33,10 @@ describe("/roles", () => {
       expect(res._rev).toBeDefined()
       expect(events.role.updated).not.toHaveBeenCalled()
       expect(events.role.created).toHaveBeenCalledTimes(1)
-      expect(events.role.created).toHaveBeenCalledWith(res)
+      expect(events.role.created).toHaveBeenCalledWith({
+        ...res,
+        _id: dbCore.prefixRoleID(res._id!),
+      })
     })
   })
 
@@ -52,7 +60,10 @@ describe("/roles", () => {
       expect(res._rev).toBeDefined()
       expect(events.role.created).not.toHaveBeenCalled()
       expect(events.role.updated).toHaveBeenCalledTimes(1)
-      expect(events.role.updated).toHaveBeenCalledWith(res)
+      expect(events.role.updated).toHaveBeenCalledWith({
+        ...res,
+        _id: dbCore.prefixRoleID(res._id!),
+      })
     })
 
     it("disallow loops", async () => {
@@ -173,7 +184,10 @@ describe("/roles", () => {
         status: 404,
       })
       expect(events.role.deleted).toHaveBeenCalledTimes(1)
-      expect(events.role.deleted).toHaveBeenCalledWith(customRole)
+      expect(events.role.deleted).toHaveBeenCalledWith({
+        ...customRole,
+        _id: dbCore.prefixRoleID(customRole._id!),
+      })
     })
   })
 

From 02d4c2d607f6d98a6a7b0886c198c31f50aafbde Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 16 Oct 2024 17:26:48 +0100
Subject: [PATCH 271/455] fixing rbac

---
 packages/backend-core/src/security/roles.ts   | 29 ++++++++++++++-
 packages/server/src/api/controllers/role.ts   | 36 ++++++++++++-------
 .../server/src/api/routes/tests/role.spec.ts  |  4 +--
 packages/shared-core/src/helpers/roles.ts     |  2 +-
 4 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 03660b313d6..fdea09944ae 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -63,6 +63,12 @@ export class Role implements RoleDoc {
   }
 
   addInheritance(inherits?: string | string[]) {
+    // make sure IDs are correct format
+    if (inherits && typeof inherits === "string") {
+      inherits = prefixRoleIDNoBuiltin(inherits)
+    } else if (inherits && Array.isArray(inherits)) {
+      inherits = inherits.map(inherit => prefixRoleIDNoBuiltin(inherit))
+    }
     if (inherits) {
       this.inherits = inherits
     }
@@ -128,7 +134,15 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } {
 }
 
 export function isBuiltin(role: string) {
-  return getBuiltinRole(role) !== undefined
+  return Object.values(BUILTIN_ROLE_IDS).includes(role)
+}
+
+export function prefixRoleIDNoBuiltin(roleId: string) {
+  if (isBuiltin(roleId)) {
+    return roleId
+  } else {
+    return prefixRoleID(roleId)
+  }
 }
 
 export function getBuiltinRole(roleId: string): Role | undefined {
@@ -536,3 +550,16 @@ export function getExternalRoleID(roleId: string, version?: string) {
   }
   return roleId
 }
+
+export function getExternalRoleIDs(
+  roleIds: string | string[] | undefined,
+  version?: string
+) {
+  if (!roleIds) {
+    return roleIds
+  } else if (typeof roleIds === "string") {
+    return getExternalRoleID(roleIds, version)
+  } else {
+    return roleIds.map(roleId => getExternalRoleID(roleId, version))
+  }
+}
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 790325390b0..34455d2c525 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -28,6 +28,18 @@ const UpdateRolesOptions = {
   REMOVED: "removed",
 }
 
+function externalRole(role: Role): Role {
+  let _id: string | undefined
+  if (role._id) {
+    _id = roles.getExternalRoleID(role._id)
+  }
+  return {
+    ...role,
+    _id,
+    inherits: roles.getExternalRoleIDs(role.inherits, role.version),
+  }
+}
+
 async function updateRolesOnUserTable(
   db: Database,
   roleId: string,
@@ -54,7 +66,7 @@ async function updateRolesOnUserTable(
 }
 
 export async function fetch(ctx: UserCtx<void, FetchRolesResponse>) {
-  ctx.body = await roles.getAllRoles()
+  ctx.body = (await roles.getAllRoles()).map(role => externalRole(role))
 }
 
 export async function find(ctx: UserCtx<void, FindRoleResponse>) {
@@ -62,7 +74,7 @@ export async function find(ctx: UserCtx<void, FindRoleResponse>) {
   if (!role) {
     ctx.throw(404, { message: "Role not found" })
   } else {
-    ctx.body = role
+    ctx.body = externalRole(role)
   }
 }
 
@@ -136,10 +148,7 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     role.version
   )
   role._rev = result.rev
-  ctx.body = {
-    ...role,
-    _id: roles.getExternalRoleID(role._id!, role.version),
-  }
+  ctx.body = externalRole(role)
 
   const devDb = context.getDevAppDB()
   const prodDb = context.getProdAppDB()
@@ -199,15 +208,14 @@ export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
   if (!roleId) {
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
   }
+  let roleIds: string[] = []
   if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
     const appId = context.getAppId()
-    if (!appId) {
-      ctx.body = []
-    } else {
-      ctx.body = await roles.getAllRoleIds(appId)
+    if (appId) {
+      roleIds = await roles.getAllRoleIds(appId)
     }
   } else {
-    ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
+    roleIds = await roles.getUserRoleIdHierarchy(roleId!)
   }
 
   // If a custom role is provided in the header, filter out higher level roles
@@ -215,7 +223,7 @@ export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
   if (roleHeader && !Object.keys(roles.BUILTIN_ROLE_IDS).includes(roleHeader)) {
     const role = await roles.getRole(roleHeader)
     const inherits = role?.inherits
-    const orderedRoles = ctx.body.reverse()
+    const orderedRoles = roleIds.reverse()
     let filteredRoles = [roleHeader]
     for (let role of orderedRoles) {
       filteredRoles = [role, ...filteredRoles]
@@ -227,6 +235,8 @@ export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
       }
     }
     filteredRoles.pop()
-    ctx.body = [roleHeader, ...filteredRoles]
+    roleIds = [roleHeader, ...filteredRoles]
   }
+
+  ctx.body = roleIds.map(roleId => roles.getExternalRoleID(roleId))
 }
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index ef3a9191739..6ed5dfd30ff 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -105,7 +105,7 @@ describe("/roles", () => {
         inherits: [BUILTIN_ROLE_IDS.POWER, role1._id!, role2._id!],
       })
       // go back to role1
-      role1 = await config.api.roles.save(
+      await config.api.roles.save(
         {
           ...role1,
           inherits: [BUILTIN_ROLE_IDS.POWER, role2._id!, role3._id!],
@@ -113,7 +113,7 @@ describe("/roles", () => {
         { status: 400, body: { message: LOOP_ERROR } }
       )
       // go back to role2
-      role2 = await config.api.roles.save(
+      await config.api.roles.save(
         {
           ...role2,
           inherits: [BUILTIN_ROLE_IDS.POWER, role1._id!, role3._id!],
diff --git a/packages/shared-core/src/helpers/roles.ts b/packages/shared-core/src/helpers/roles.ts
index 27f28cff824..24b65501ceb 100644
--- a/packages/shared-core/src/helpers/roles.ts
+++ b/packages/shared-core/src/helpers/roles.ts
@@ -1,4 +1,4 @@
-import { Role } from "@budibase/types"
+import { Role, SEPARATOR, DocumentType } from "@budibase/types"
 
 // Function to detect loops in roles
 export function checkForRoleInheritanceLoops(roles: Role[]): boolean {

From 31c0ed69f1ce23e8dbf9cc796bc95f1124124278 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 16 Oct 2024 18:28:40 +0100
Subject: [PATCH 272/455] wip

---
 .../src/api/routes/tests/viewV2.spec.ts       |   3 +-
 packages/server/src/sdk/app/views/internal.ts |  15 +++
 packages/shared-core/src/filters.ts           |  33 +++---
 packages/shared-core/src/utils.ts             | 109 ++++++++----------
 4 files changed, 86 insertions(+), 74 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 7979fac5551..7c866e19fe3 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -144,7 +144,7 @@ describe.each([
         })
       })
 
-      it("can persist views with all fields", async () => {
+      it.only("can persist views with all fields", async () => {
         const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
           name: generator.name(),
           tableId: table._id!,
@@ -178,6 +178,7 @@ describe.each([
               visible: true,
             },
           },
+          queryUI: {},
           id: expect.any(String),
           version: 2,
         })
diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts
index 96b41bffe8d..30e0c18d057 100644
--- a/packages/server/src/sdk/app/views/internal.ts
+++ b/packages/server/src/sdk/app/views/internal.ts
@@ -4,6 +4,19 @@ import { context, HTTPError } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
+import { utils as sharedUtils, dataFilters } from "@budibase/shared-core"
+
+function ensureQueryUISet(view: ViewV2) {
+  if (!view.queryUI && view.query) {
+    view.queryUI = sharedUtils.processSearchFilters(view.query)
+  }
+}
+
+function ensureQuerySet(view: ViewV2) {
+  if (!view.query && view.queryUI) {
+    view.query = dataFilters.buildQuery(view.queryUI)
+  }
+}
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -13,6 +26,7 @@ export async function get(viewId: string): Promise<ViewV2> {
   if (!found) {
     throw new Error("No view found")
   }
+  ensureQueryUISet(found)
   return found
 }
 
@@ -24,6 +38,7 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   if (!found) {
     throw new Error("No view found")
   }
+  ensureQueryUISet(found)
   return await enrichSchema(found, table.schema)
 }
 
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 40c70a8a231..64958dd4f7d 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -572,29 +572,34 @@ export const buildQueryLegacy = (
  *
  * @returns {SearchFilters}
  */
-
-export const buildQuery = (
+export function buildQuery(filter: undefined): undefined
+export function buildQuery(
+  filter: SearchFilterGroup | LegacyFilter[]
+): SearchFilters
+export function buildQuery(
   filter?: SearchFilterGroup | LegacyFilter[]
-): SearchFilters | undefined => {
-  const parsedFilter: SearchFilterGroup | undefined =
-    processSearchFilters(filter)
-
-  if (!parsedFilter) {
+): SearchFilters | undefined {
+  if (!filter) {
     return
   }
 
-  const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
-    {
-      [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
-      [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
-    }
+  let parsedFilter: SearchFilterGroup
+  if (Array.isArray(filter)) {
+    parsedFilter = processSearchFilters(filter)
+  } else {
+    parsedFilter = filter
+  }
+
+  const operatorMap = {
+    [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
+    [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
+  }
 
   const globalOnEmpty = parsedFilter.onEmptyFilter
     ? parsedFilter.onEmptyFilter
     : null
 
-  const globalOperator: LogicalOperator =
-    operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
+  const globalOperator = operatorMap[parsedFilter.logicalOperator]
 
   return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index bead5961f0e..7f19371d794 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -137,12 +137,8 @@ export function isSupportedUserSearch(query: SearchFilters) {
  * @param {LegacyFilter[] | SearchFilterGroup} filters
  */
 export const processSearchFilters = (
-  filters: LegacyFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup | undefined => {
-  if (!filters) {
-    return
-  }
-
+  filters: LegacyFilter[]
+): SearchFilterGroup => {
   // Base search config.
   const defaultCfg: SearchFilterGroup = {
     logicalOperator: FilterGroupLogicalOperator.ALL,
@@ -160,65 +156,60 @@ export const processSearchFilters = (
     "formulaType",
   ]
 
-  if (Array.isArray(filters)) {
-    let baseGroup: SearchFilterGroup = {
-      filters: [],
-      logicalOperator: FilterGroupLogicalOperator.ALL,
-    }
-
-    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
-      // Sort the properties for easier debugging
-      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
-        .sort((a, b) => {
-          return a.localeCompare(b)
-        })
-        .filter(key => filter[key])
-
-      if (filterPropertyKeys.length == 1) {
-        const key = filterPropertyKeys[0],
-          value = filter[key]
-        // Global
-        if (key === "onEmptyFilter") {
-          // unset otherwise
-          acc.onEmptyFilter = value
-        } else if (key === "operator" && value === "allOr") {
-          // Group 1 logical operator
-          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
-        }
+  let baseGroup: SearchFilterGroup = {
+    filters: [],
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+  }
 
-        return acc
+  return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+    // Sort the properties for easier debugging
+    const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
+      .sort((a, b) => {
+        return a.localeCompare(b)
+      })
+      .filter(key => filter[key])
+
+    if (filterPropertyKeys.length == 1) {
+      const key = filterPropertyKeys[0],
+        value = filter[key]
+      // Global
+      if (key === "onEmptyFilter") {
+        // unset otherwise
+        acc.onEmptyFilter = value
+      } else if (key === "operator" && value === "allOr") {
+        // Group 1 logical operator
+        baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
       }
 
-      const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
-        (acc: AllowedFilters, key) => {
-          const value = filter[key]
-          if (filterAllowedKeys.includes(key)) {
-            if (key === "field") {
-              acc.push([key, removeKeyNumbering(value)])
-            } else {
-              acc.push([key, value])
-            }
+      return acc
+    }
+
+    const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
+      (acc: AllowedFilters, key) => {
+        const value = filter[key]
+        if (filterAllowedKeys.includes(key)) {
+          if (key === "field") {
+            acc.push([key, removeKeyNumbering(value)])
+          } else {
+            acc.push([key, value])
           }
-          return acc
-        },
-        []
-      )
+        }
+        return acc
+      },
+      []
+    )
 
-      const migratedFilter: LegacyFilter = Object.fromEntries(
-        allowedFilterSettings
-      ) as LegacyFilter
+    const migratedFilter: LegacyFilter = Object.fromEntries(
+      allowedFilterSettings
+    ) as LegacyFilter
 
-      baseGroup.filters!.push(migratedFilter)
+    baseGroup.filters!.push(migratedFilter)
 
-      if (!acc.groups || !acc.groups.length) {
-        // init the base group
-        acc.groups = [baseGroup]
-      }
+    if (!acc.groups || !acc.groups.length) {
+      // init the base group
+      acc.groups = [baseGroup]
+    }
 
-      return acc
-    }, defaultCfg)
-  } else if (!filters?.groups) {
-    return
-  }
-  return filters
+    return acc
+  }, defaultCfg)
 }

From cb41861d13342d6ffcda69d2d4058dcf08857084 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 17 Oct 2024 11:54:34 +0100
Subject: [PATCH 273/455] Making progress toward testing buildCondition and
 friends.

---
 .../src/api/routes/tests/viewV2.spec.ts       |  52 +++++++--
 packages/server/src/sdk/app/views/external.ts |   7 ++
 packages/server/src/sdk/app/views/internal.ts |  18 +---
 packages/server/src/sdk/app/views/utils.ts    |  30 ++++++
 packages/shared-core/src/filters.ts           | 101 +++++++++---------
 packages/shared-core/src/utils.ts             |   6 +-
 packages/types/src/api/web/searchFilter.ts    |  11 +-
 packages/types/src/sdk/search.ts              |  14 ++-
 8 files changed, 156 insertions(+), 83 deletions(-)
 create mode 100644 packages/server/src/sdk/app/views/utils.ts

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 7c866e19fe3..b81c7314859 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -27,6 +27,8 @@ import {
   ViewV2Schema,
   ViewV2Type,
   JsonTypes,
+  FilterGroupLogicalOperator,
+  EmptyFilterOption,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -145,17 +147,26 @@ describe.each([
       })
 
       it.only("can persist views with all fields", async () => {
-        const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
+        const newView: Required<Omit<CreateViewRequest, "query" | "type">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
-          query: [
-            {
-              operator: BasicOperator.EQUAL,
-              field: "field",
-              value: "value",
-            },
-          ],
+          queryUI: {
+            logicalOperator: FilterGroupLogicalOperator.ALL,
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+            groups: [
+              {
+                logicalOperator: FilterGroupLogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "field",
+                    value: "value",
+                  },
+                ],
+              },
+            ],
+          },
           sort: {
             field: "fieldToSort",
             order: SortOrder.DESCENDING,
@@ -170,7 +181,7 @@ describe.each([
         }
         const res = await config.api.viewV2.create(newView)
 
-        expect(res).toEqual({
+        const expected: ViewV2 = {
           ...newView,
           schema: {
             id: { visible: true },
@@ -178,10 +189,29 @@ describe.each([
               visible: true,
             },
           },
-          queryUI: {},
+          query: {
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+            $and: {
+              conditions: [
+                {
+                  $and: {
+                    conditions: [
+                      {
+                        equal: {
+                          field: "value",
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+            },
+          },
           id: expect.any(String),
           version: 2,
-        })
+        }
+
+        expect(res).toEqual(expected)
       })
 
       it("persist only UI schema overrides", async () => {
diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts
index b69ac0b9eb4..d5251122c9c 100644
--- a/packages/server/src/sdk/app/views/external.ts
+++ b/packages/server/src/sdk/app/views/external.ts
@@ -5,6 +5,7 @@ import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
 import { breakExternalTableId } from "../../../integrations/utils"
+import { ensureQuerySet, ensureQueryUISet } from "./utils"
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -18,6 +19,7 @@ export async function get(viewId: string): Promise<ViewV2> {
   if (!found) {
     throw new Error("No view found")
   }
+  ensureQueryUISet(found)
   return found
 }
 
@@ -33,6 +35,7 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   if (!found) {
     throw new Error("No view found")
   }
+  ensureQueryUISet(found)
   return await enrichSchema(found, table.schema)
 }
 
@@ -46,6 +49,8 @@ export async function create(
     version: 2,
   }
 
+  ensureQuerySet(view)
+
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -74,6 +79,8 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`Cannot update view type after creation`, 400)
   }
 
+  ensureQuerySet(view)
+
   delete views[existingView.name]
   views[view.name] = view
   await db.put(ds)
diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts
index 30e0c18d057..33b68759d72 100644
--- a/packages/server/src/sdk/app/views/internal.ts
+++ b/packages/server/src/sdk/app/views/internal.ts
@@ -4,19 +4,7 @@ import { context, HTTPError } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
-import { utils as sharedUtils, dataFilters } from "@budibase/shared-core"
-
-function ensureQueryUISet(view: ViewV2) {
-  if (!view.queryUI && view.query) {
-    view.queryUI = sharedUtils.processSearchFilters(view.query)
-  }
-}
-
-function ensureQuerySet(view: ViewV2) {
-  if (!view.query && view.queryUI) {
-    view.query = dataFilters.buildQuery(view.queryUI)
-  }
-}
+import { ensureQuerySet, ensureQueryUISet } from "./utils"
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -52,6 +40,8 @@ export async function create(
     version: 2,
   }
 
+  ensureQuerySet(view)
+
   const db = context.getAppDB()
 
   const table = await sdk.tables.getTable(tableId)
@@ -78,6 +68,8 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`Cannot update view type after creation`, 400)
   }
 
+  ensureQuerySet(view)
+
   delete table.views[existingView.name]
   table.views[view.name] = view
   await db.put(table)
diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
new file mode 100644
index 00000000000..45b091e0463
--- /dev/null
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -0,0 +1,30 @@
+import { ViewV2 } from "@budibase/types"
+import { utils, dataFilters } from "@budibase/shared-core"
+
+export function ensureQueryUISet(view: ViewV2) {
+  if (!view.queryUI && view.query) {
+    if (!Array.isArray(view.query)) {
+      // In practice this should not happen. `view.query`, at the time this code
+      // goes into the codebase, only contains LegacyFilter[] in production.
+      // We're changing it in the change that this comment is part of to also
+      // include SearchFilters objects. These are created when we receive an
+      // update to a ViewV2 that contains a queryUI and not a query field. We
+      // can convert SearchFilterGroup (the type of queryUI) to SearchFilters,
+      // but not LegacyFilter[], they are incompatible due to SearchFilterGroup
+      // and SearchFilters being recursive types.
+      //
+      // So despite the type saying that `view.query` is a LegacyFilter[] |
+      // SearchFilters, it will never be a SearchFilters when a `view.queryUI`
+      // is specified, making it "safe" to throw an error here.
+      throw new Error("view is missing queryUI field")
+    }
+
+    view.queryUI = utils.processSearchFilters(view.query)
+  }
+}
+
+export function ensureQuerySet(view: ViewV2) {
+  if (!view.query && view.queryUI) {
+    view.query = dataFilters.buildQuery(view.queryUI)
+  }
+}
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 64958dd4f7d..3b9253af6e6 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -21,6 +21,9 @@ import {
   isLogicalSearchOperator,
   SearchFilterGroup,
   FilterGroupLogicalOperator,
+  isBasicSearchOperator,
+  isArraySearchOperator,
+  isRangeSearchOperator,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -307,36 +310,23 @@ export class ColumnSplitter {
  * Builds a JSON query from the filter a SearchFilter definition
  * @param filter the builder filter structure
  */
-
 const buildCondition = (expression: LegacyFilter) => {
-  // Filter body
-  let query: SearchFilters = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-    oneOf: {},
-    containsAny: {},
-  }
-  let { operator, field, type, value, externalType, onEmptyFilter } = expression
+  const query: SearchFilters = {}
+  const { operator, field, type, externalType, onEmptyFilter } = expression
+  let { value } = expression
 
   if (!operator || !field) {
     return
   }
 
-  const queryOperator = operator as SearchFilterOperator
   const isHbs =
     typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
-  // Parse all values into correct types
+
   if (operator === "allOr") {
     query.allOr = true
     return
   }
+
   if (onEmptyFilter) {
     query.onEmptyFilter = onEmptyFilter
     return
@@ -344,15 +334,15 @@ const buildCondition = (expression: LegacyFilter) => {
 
   // Default the value for noValue fields to ensure they are correctly added
   // to the final query
-  if (queryOperator === "empty" || queryOperator === "notEmpty") {
+  if (operator === "empty" || operator === "notEmpty") {
     value = null
   }
 
   if (
     type === "datetime" &&
     !isHbs &&
-    queryOperator !== "empty" &&
-    queryOperator !== "notEmpty"
+    operator !== "empty" &&
+    operator !== "notEmpty"
   ) {
     // Ensure date value is a valid date and parse into correct format
     if (!value) {
@@ -365,7 +355,7 @@ const buildCondition = (expression: LegacyFilter) => {
     }
   }
   if (type === "number" && typeof value === "string" && !isHbs) {
-    if (queryOperator === "oneOf") {
+    if (operator === "oneOf") {
       value = value.split(",").map(item => parseFloat(item))
     } else {
       value = parseFloat(value)
@@ -383,50 +373,55 @@ const buildCondition = (expression: LegacyFilter) => {
   ) {
     value = value.split(",")
   }
-  if (operator.toLocaleString().startsWith("range") && query.range) {
-    const minint =
-      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
-        ?.min || Number.MIN_SAFE_INTEGER
-    const maxint =
-      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
-        ?.max || Number.MAX_SAFE_INTEGER
-    if (!query.range[field]) {
-      query.range[field] = {
-        low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-        high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-      }
+
+  if (isRangeSearchOperator(operator)) {
+    const key = externalType as keyof typeof SqlNumberTypeRangeMap
+    const limits = SqlNumberTypeRangeMap[key] || {
+      min: Number.MIN_SAFE_INTEGER,
+      max: Number.MAX_SAFE_INTEGER,
     }
-    if (operator === "rangeLow" && value != null && value !== "") {
-      query.range[field] = {
-        ...query.range[field],
-        low: value,
-      }
-    } else if (operator === "rangeHigh" && value != null && value !== "") {
-      query.range[field] = {
-        ...query.range[field],
-        high: value,
-      }
+
+    query[operator] ??= {}
+    query[operator][field] = {
+      low: type === "number" ? limits.min : "0000-00-00T00:00:00.000Z",
+      high: type === "number" ? limits.max : "9999-00-00T00:00:00.000Z",
     }
-  } else if (isLogicalSearchOperator(queryOperator)) {
+  } else if (operator === "rangeHigh" && value != null && value !== "") {
+    query.range ??= {}
+    query.range[field] = {
+      ...query.range[field],
+      high: value,
+    }
+  } else if (operator === "rangeLow" && value != null && value !== "") {
+    query.range ??= {}
+    query.range[field] = {
+      ...query.range[field],
+      low: value,
+    }
+  } else if (isLogicalSearchOperator(operator)) {
     // TODO
-  } else if (query[queryOperator] && operator !== "onEmptyFilter") {
+  } else if (
+    isBasicSearchOperator(operator) ||
+    isArraySearchOperator(operator) ||
+    isRangeSearchOperator(operator)
+  ) {
     if (type === "boolean") {
       // Transform boolean filters to cope with null.
       // "equals false" needs to be "not equals true"
       // "not equals false" needs to be "equals true"
-      if (queryOperator === "equal" && value === false) {
+      if (operator === "equal" && value === false) {
         query.notEqual = query.notEqual || {}
         query.notEqual[field] = true
-      } else if (queryOperator === "notEqual" && value === false) {
+      } else if (operator === "notEqual" && value === false) {
         query.equal = query.equal || {}
         query.equal[field] = true
       } else {
-        query[queryOperator] ??= {}
-        query[queryOperator]![field] = value
+        query[operator] ??= {}
+        query[operator][field] = value
       }
     } else {
-      query[queryOperator] ??= {}
-      query[queryOperator]![field] = value
+      query[operator] ??= {}
+      query[operator][field] = value
     }
   }
 
@@ -604,7 +599,7 @@ export function buildQuery(
   return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
-      conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
+      conditions: parsedFilter.groups?.map(group => {
         return {
           [operatorMap[group.logicalOperator]]: {
             conditions: group.filters
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 7f19371d794..98f7c9b2154 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -6,6 +6,8 @@ import {
   BasicOperator,
   ArrayOperator,
   isLogicalSearchOperator,
+  EmptyFilterOption,
+  SearchFilterChild,
 } from "@budibase/types"
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
@@ -142,6 +144,7 @@ export const processSearchFilters = (
   // Base search config.
   const defaultCfg: SearchFilterGroup = {
     logicalOperator: FilterGroupLogicalOperator.ALL,
+    onEmptyFilter: EmptyFilterOption.RETURN_ALL,
     groups: [],
   }
 
@@ -156,8 +159,7 @@ export const processSearchFilters = (
     "formulaType",
   ]
 
-  let baseGroup: SearchFilterGroup = {
-    filters: [],
+  let baseGroup: SearchFilterChild = {
     logicalOperator: FilterGroupLogicalOperator.ALL,
   }
 
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index 23c599027e5..305f2bab007 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -14,10 +14,15 @@ export type LegacyFilter = {
   externalType?: string
 }
 
+export type SearchFilterChild = {
+  logicalOperator: FilterGroupLogicalOperator
+  groups?: SearchFilterChild[]
+  filters?: LegacyFilter[]
+}
+
 // this is a type purely used by the UI
 export type SearchFilterGroup = {
   logicalOperator: FilterGroupLogicalOperator
-  onEmptyFilter?: EmptyFilterOption
-  groups?: SearchFilterGroup[]
-  filters?: LegacyFilter[]
+  onEmptyFilter: EmptyFilterOption
+  groups: SearchFilterChild[]
 }
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index d41bb0fb99a..d64e87d4347 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -32,7 +32,19 @@ export enum LogicalOperator {
 export function isLogicalSearchOperator(
   value: string
 ): value is LogicalOperator {
-  return value === LogicalOperator.AND || value === LogicalOperator.OR
+  return Object.values(LogicalOperator).includes(value as LogicalOperator)
+}
+
+export function isBasicSearchOperator(value: string): value is BasicOperator {
+  return Object.values(BasicOperator).includes(value as BasicOperator)
+}
+
+export function isArraySearchOperator(value: string): value is ArrayOperator {
+  return Object.values(ArrayOperator).includes(value as ArrayOperator)
+}
+
+export function isRangeSearchOperator(value: string): value is RangeOperator {
+  return Object.values(RangeOperator).includes(value as RangeOperator)
 }
 
 export type SearchFilterOperator =

From cbd84ae249452c4bf7caba7aa0fea8524a70f798 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 17 Oct 2024 13:38:08 +0100
Subject: [PATCH 274/455] Handle role errors and reset state as required

---
 .../components/backend/RoleEditor/RoleFlow.svelte    | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 5507e5ee870..152946fa96e 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Heading, Helpers } from "@budibase/bbui"
+  import { Heading, Helpers, notifications } from "@budibase/bbui"
   import { writable, derived } from "svelte/store"
   import {
     SvelteFlow,
@@ -118,7 +118,13 @@
     if (metadata) {
       flow.updateNodeData(roleId, metadata)
     }
-    await roles.save(nodeToRole({ node, edges: $edges }))
+    try {
+      await roles.save(nodeToRole({ node, edges: $edges }))
+      layoutAndFit()
+    } catch (error) {
+      notifications.error(error?.message || error || "Failed to update role")
+      handleExternalRoleChanges($roles)
+    }
   }
 
   const deleteRole = async roleId => {
@@ -133,12 +139,10 @@
   const deleteEdge = async edgeId => {
     const edge = $edges.find(edge => edge.id === edgeId)
     edges.set($edges.filter(edge => edge.id !== edgeId))
-    layoutAndFit()
     await updateRole(edge.target)
   }
 
   const onConnect = async connection => {
-    layoutAndFit()
     await updateRole(connection.target)
   }
 

From d50e1cf31a40f2f0197dee75465469058cd25bce Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 17 Oct 2024 13:42:23 +0100
Subject: [PATCH 275/455]  Fix detection of custom roles

---
 .../src/components/backend/RoleEditor/constants.js  |  1 +
 .../src/components/backend/RoleEditor/utils.js      | 13 ++++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js
index dacd37d745e..6f188e2141e 100644
--- a/packages/builder/src/components/backend/RoleEditor/constants.js
+++ b/packages/builder/src/components/backend/RoleEditor/constants.js
@@ -6,3 +6,4 @@ export const NodeWidth = GridResolution * 12
 export const NodeHSpacing = GridResolution * 6
 export const NodeVSpacing = GridResolution * 2
 export const MinHeight = GridResolution * 10
+export const EmptyStateID = "empty"
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index 1cbe6a17af7..0142b7b36ff 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -6,6 +6,7 @@ import {
   NodeHSpacing,
   NodeVSpacing,
   MinHeight,
+  EmptyStateID,
 } from "./constants"
 import { getNodesBounds, Position } from "@xyflow/svelte"
 import { Roles } from "constants/backend"
@@ -50,7 +51,7 @@ export const getAdminPosition = bounds => ({
 
 // Filters out invalid nodes and edges
 const preProcessLayout = ({ nodes, edges }) => {
-  const ignoredIds = [Roles.PUBLIC, Roles.BASIC, Roles.ADMIN, "empty"]
+  const ignoredIds = [Roles.PUBLIC, Roles.BASIC, Roles.ADMIN, EmptyStateID]
   const targetlessIds = [Roles.POWER]
   return {
     nodes: nodes.filter(node => {
@@ -139,7 +140,7 @@ const postProcessLayout = ({ nodes, edges }) => {
   // Add empty state node if required
   if (!nodes.some(node => node.data.interactive)) {
     nodes.push({
-      id: "empty",
+      id: EmptyStateID,
       type: "empty",
       position: {
         x: bounds.x + bounds.width / 2 - NodeWidth / 2,
@@ -167,7 +168,13 @@ export const autoLayout = ({ nodes, edges }) => {
 
 // Converts a role doc into a node structure
 export const roleToNode = role => {
-  const custom = !role._id.match(/[A-Z]+/)
+  const custom = ![
+    Roles.PUBLIC,
+    Roles.BASIC,
+    Roles.POWER,
+    Roles.ADMIN,
+    Roles.BUILDER,
+  ].includes(role._id)
   const interactive = custom || role._id === Roles.POWER
   return {
     id: role._id,

From ac46e27cea62780fc4bafde62064818f001a4f66 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 17 Oct 2024 13:48:20 +0100
Subject: [PATCH 276/455] Lint

---
 packages/shared-core/src/helpers/roles.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/shared-core/src/helpers/roles.ts b/packages/shared-core/src/helpers/roles.ts
index 24b65501ceb..27f28cff824 100644
--- a/packages/shared-core/src/helpers/roles.ts
+++ b/packages/shared-core/src/helpers/roles.ts
@@ -1,4 +1,4 @@
-import { Role, SEPARATOR, DocumentType } from "@budibase/types"
+import { Role } from "@budibase/types"
 
 // Function to detect loops in roles
 export function checkForRoleInheritanceLoops(roles: Role[]): boolean {

From 17e946845e49d9823ad5903e904085bb92b6d60a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 17 Oct 2024 17:16:54 +0100
Subject: [PATCH 277/455] Get existing tests passing.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 2150 +++++++++--------
 packages/server/src/sdk/app/views/utils.ts    |    9 +-
 .../server/src/tests/utilities/api/viewV2.ts  |    4 +-
 packages/shared-core/src/filters.ts           |   88 +-
 4 files changed, 1149 insertions(+), 1102 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index b81c7314859..71506da0fa9 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -146,7 +146,7 @@ describe.each([
         })
       })
 
-      it.only("can persist views with all fields", async () => {
+      it("can persist views with all fields", async () => {
         const newView: Required<Omit<CreateViewRequest, "query" | "type">> = {
           name: generator.name(),
           tableId: table._id!,
@@ -2332,548 +2332,391 @@ describe.each([
       })
     })
 
-    describe("search", () => {
-      it("returns empty rows from view when no schema is passed", async () => {
-        const rows = await Promise.all(
-          Array.from({ length: 10 }, () => config.api.row.save(table._id!, {}))
-        )
-        const response = await config.api.viewV2.search(view.id)
-        expect(response.rows).toHaveLength(10)
-        expect(response).toEqual({
-          rows: expect.arrayContaining(
-            rows.map(r => ({
-              _viewId: view.id,
-              tableId: table._id,
-              id: r.id,
-              _id: r._id,
-              _rev: r._rev,
-              ...(isInternal
-                ? {
-                    type: "row",
-                    updatedAt: expect.any(String),
-                    createdAt: expect.any(String),
-                  }
-                : {}),
-            }))
-          ),
-          ...(isInternal
-            ? {}
-            : {
-                hasNextPage: false,
-              }),
+    !isLucene &&
+      describe("search", () => {
+        it("returns empty rows from view when no schema is passed", async () => {
+          const rows = await Promise.all(
+            Array.from({ length: 10 }, () =>
+              config.api.row.save(table._id!, {})
+            )
+          )
+          const response = await config.api.viewV2.search(view.id)
+          expect(response.rows).toHaveLength(10)
+          expect(response).toEqual({
+            rows: expect.arrayContaining(
+              rows.map(r => ({
+                _viewId: view.id,
+                tableId: table._id,
+                id: r.id,
+                _id: r._id,
+                _rev: r._rev,
+                ...(isInternal
+                  ? {
+                      type: "row",
+                      updatedAt: expect.any(String),
+                      createdAt: expect.any(String),
+                    }
+                  : {}),
+              }))
+            ),
+            ...(isInternal
+              ? {}
+              : {
+                  hasNextPage: false,
+                }),
+          })
         })
-      })
 
-      it("searching respects the view filters", async () => {
-        await config.api.row.save(table._id!, {
-          one: "foo",
-          two: "bar",
-        })
-        const two = await config.api.row.save(table._id!, {
-          one: "foo2",
-          two: "bar2",
-        })
+        it("searching respects the view filters", async () => {
+          await config.api.row.save(table._id!, {
+            one: "foo",
+            two: "bar",
+          })
+          const two = await config.api.row.save(table._id!, {
+            one: "foo2",
+            two: "bar2",
+          })
 
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          query: [
-            {
-              operator: BasicOperator.EQUAL,
-              field: "two",
-              value: "bar2",
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            queryUI: {
+              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              groups: [
+                {
+                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  filters: [
+                    {
+                      operator: BasicOperator.EQUAL,
+                      field: "two",
+                      value: "bar2",
+                    },
+                  ],
+                },
+              ],
             },
-          ],
-          schema: {
-            id: { visible: true },
-            one: { visible: false },
-            two: { visible: true },
-          },
-        })
-
-        const response = await config.api.viewV2.search(view.id)
-        expect(response.rows).toHaveLength(1)
-        expect(response).toEqual({
-          rows: expect.arrayContaining([
-            {
-              _viewId: view.id,
-              tableId: table._id,
-              id: two.id,
-              two: two.two,
-              _id: two._id,
-              _rev: two._rev,
-              ...(isInternal
-                ? {
-                    type: "row",
-                    createdAt: expect.any(String),
-                    updatedAt: expect.any(String),
-                  }
-                : {}),
+            schema: {
+              id: { visible: true },
+              one: { visible: false },
+              two: { visible: true },
             },
-          ]),
-          ...(isInternal
-            ? {}
-            : {
-                hasNextPage: false,
-              }),
-        })
-      })
-
-      it("views filters are respected even if the column is hidden", async () => {
-        await config.api.row.save(table._id!, {
-          one: "foo",
-          two: "bar",
-        })
-        const two = await config.api.row.save(table._id!, {
-          one: "foo2",
-          two: "bar2",
-        })
+          })
 
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          query: [
-            {
-              operator: BasicOperator.EQUAL,
-              field: "two",
-              value: "bar2",
-            },
-          ],
-          schema: {
-            id: { visible: true },
-            one: { visible: false },
-            two: { visible: false },
-          },
+          const response = await config.api.viewV2.search(view.id)
+          expect(response.rows).toHaveLength(1)
+          expect(response).toEqual({
+            rows: expect.arrayContaining([
+              {
+                _viewId: view.id,
+                tableId: table._id,
+                id: two.id,
+                two: two.two,
+                _id: two._id,
+                _rev: two._rev,
+                ...(isInternal
+                  ? {
+                      type: "row",
+                      createdAt: expect.any(String),
+                      updatedAt: expect.any(String),
+                    }
+                  : {}),
+              },
+            ]),
+            ...(isInternal
+              ? {}
+              : {
+                  hasNextPage: false,
+                }),
+          })
         })
 
-        const response = await config.api.viewV2.search(view.id)
-        expect(response.rows).toHaveLength(1)
-        expect(response.rows).toEqual([
-          expect.objectContaining({ _id: two._id }),
-        ])
-      })
+        it("views filters are respected even if the column is hidden", async () => {
+          await config.api.row.save(table._id!, {
+            one: "foo",
+            two: "bar",
+          })
+          const two = await config.api.row.save(table._id!, {
+            one: "foo2",
+            two: "bar2",
+          })
 
-      it("views without data can be returned", async () => {
-        const response = await config.api.viewV2.search(view.id)
-        expect(response.rows).toHaveLength(0)
-      })
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            queryUI: {
+              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              groups: [
+                {
+                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  filters: [
+                    {
+                      operator: BasicOperator.EQUAL,
+                      field: "two",
+                      value: "bar2",
+                    },
+                  ],
+                },
+              ],
+            },
+            schema: {
+              id: { visible: true },
+              one: { visible: false },
+              two: { visible: false },
+            },
+          })
 
-      it("respects the limit parameter", async () => {
-        await Promise.all(
-          Array.from({ length: 10 }, () => config.api.row.save(table._id!, {}))
-        )
-        const limit = generator.integer({ min: 1, max: 8 })
-        const response = await config.api.viewV2.search(view.id, {
-          limit,
-          query: {},
+          const response = await config.api.viewV2.search(view.id)
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows).toEqual([
+            expect.objectContaining({ _id: two._id }),
+          ])
         })
-        expect(response.rows).toHaveLength(limit)
-      })
 
-      it("can handle pagination", async () => {
-        await Promise.all(
-          Array.from({ length: 10 }, () => config.api.row.save(table._id!, {}))
-        )
-        const rows = (await config.api.viewV2.search(view.id)).rows
-
-        const page1 = await config.api.viewV2.search(view.id, {
-          paginate: true,
-          limit: 4,
-          query: {},
-          countRows: true,
-        })
-        expect(page1).toEqual({
-          rows: expect.arrayContaining(rows.slice(0, 4)),
-          hasNextPage: true,
-          bookmark: expect.anything(),
-          totalRows: 10,
-        })
-
-        const page2 = await config.api.viewV2.search(view.id, {
-          paginate: true,
-          limit: 4,
-          bookmark: page1.bookmark,
-          query: {},
-          countRows: true,
-        })
-        expect(page2).toEqual({
-          rows: expect.arrayContaining(rows.slice(4, 8)),
-          hasNextPage: true,
-          bookmark: expect.anything(),
-          totalRows: 10,
-        })
-
-        const page3 = await config.api.viewV2.search(view.id, {
-          paginate: true,
-          limit: 4,
-          bookmark: page2.bookmark,
-          query: {},
-          countRows: true,
-        })
-        const expectation: SearchResponse<Row> = {
-          rows: expect.arrayContaining(rows.slice(8)),
-          hasNextPage: false,
-          totalRows: 10,
-        }
-        if (isLucene) {
-          expectation.bookmark = expect.anything()
-        }
-        expect(page3).toEqual(expectation)
-      })
-
-      const sortTestOptions: [
-        {
-          field: string
-          order?: SortOrder
-          type?: SortType
-        },
-        string[]
-      ][] = [
-        [
-          {
-            field: "name",
-            order: SortOrder.ASCENDING,
-            type: SortType.STRING,
-          },
-          ["Alice", "Bob", "Charly", "Danny"],
-        ],
-        [
-          {
-            field: "name",
-          },
-          ["Alice", "Bob", "Charly", "Danny"],
-        ],
-        [
-          {
-            field: "name",
-            order: SortOrder.DESCENDING,
-          },
-          ["Danny", "Charly", "Bob", "Alice"],
-        ],
-        [
-          {
-            field: "name",
-            order: SortOrder.DESCENDING,
-            type: SortType.STRING,
-          },
-          ["Danny", "Charly", "Bob", "Alice"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.ASCENDING,
-            type: SortType.NUMBER,
-          },
-          ["Danny", "Alice", "Charly", "Bob"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.ASCENDING,
-          },
-          ["Danny", "Alice", "Charly", "Bob"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.DESCENDING,
-          },
-          ["Bob", "Charly", "Alice", "Danny"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.DESCENDING,
-            type: SortType.NUMBER,
-          },
-          ["Bob", "Charly", "Alice", "Danny"],
-        ],
-      ]
-
-      describe("sorting", () => {
-        let table: Table
-        const viewSchema = {
-          id: { visible: true },
-          age: { visible: true },
-          name: { visible: true },
-        }
-
-        beforeAll(async () => {
-          table = await config.api.table.save(
-            saveTableRequest({
-              type: "table",
-              schema: {
-                name: {
-                  type: FieldType.STRING,
-                  name: "name",
-                },
-                surname: {
-                  type: FieldType.STRING,
-                  name: "surname",
-                },
-                age: {
-                  type: FieldType.NUMBER,
-                  name: "age",
-                },
-                address: {
-                  type: FieldType.STRING,
-                  name: "address",
-                },
-                jobTitle: {
-                  type: FieldType.STRING,
-                  name: "jobTitle",
-                },
-              },
-            })
-          )
+        it("views without data can be returned", async () => {
+          const response = await config.api.viewV2.search(view.id)
+          expect(response.rows).toHaveLength(0)
+        })
 
-          const users = [
-            { name: "Alice", age: 25 },
-            { name: "Bob", age: 30 },
-            { name: "Charly", age: 27 },
-            { name: "Danny", age: 15 },
-          ]
+        it("respects the limit parameter", async () => {
           await Promise.all(
-            users.map(u =>
-              config.api.row.save(table._id!, {
-                tableId: table._id,
-                ...u,
-              })
+            Array.from({ length: 10 }, () =>
+              config.api.row.save(table._id!, {})
             )
           )
+          const limit = generator.integer({ min: 1, max: 8 })
+          const response = await config.api.viewV2.search(view.id, {
+            limit,
+            query: {},
+          })
+          expect(response.rows).toHaveLength(limit)
         })
 
-        it.each(sortTestOptions)(
-          "allow sorting (%s)",
-          async (sortParams, expected) => {
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              sort: sortParams,
-              schema: viewSchema,
-            })
-
-            const response = await config.api.viewV2.search(view.id)
-
-            expect(response.rows).toHaveLength(4)
-            expect(response.rows).toEqual(
-              expected.map(name => expect.objectContaining({ name }))
+        it("can handle pagination", async () => {
+          await Promise.all(
+            Array.from({ length: 10 }, () =>
+              config.api.row.save(table._id!, {})
             )
-          }
-        )
+          )
+          const rows = (await config.api.viewV2.search(view.id)).rows
 
-        it.each(sortTestOptions)(
-          "allow override the default view sorting (%s)",
-          async (sortParams, expected) => {
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              sort: {
-                field: "name",
-                order: SortOrder.ASCENDING,
-                type: SortType.STRING,
-              },
-              schema: viewSchema,
-            })
+          const page1 = await config.api.viewV2.search(view.id, {
+            paginate: true,
+            limit: 4,
+            query: {},
+            countRows: true,
+          })
+          expect(page1).toEqual({
+            rows: expect.arrayContaining(rows.slice(0, 4)),
+            hasNextPage: true,
+            bookmark: expect.anything(),
+            totalRows: 10,
+          })
 
-            const response = await config.api.viewV2.search(view.id, {
-              sort: sortParams.field,
-              sortOrder: sortParams.order,
-              sortType: sortParams.type,
-              query: {},
-            })
+          const page2 = await config.api.viewV2.search(view.id, {
+            paginate: true,
+            limit: 4,
+            bookmark: page1.bookmark,
+            query: {},
+            countRows: true,
+          })
+          expect(page2).toEqual({
+            rows: expect.arrayContaining(rows.slice(4, 8)),
+            hasNextPage: true,
+            bookmark: expect.anything(),
+            totalRows: 10,
+          })
 
-            expect(response.rows).toHaveLength(4)
-            expect(response.rows).toEqual(
-              expected.map(name => expect.objectContaining({ name }))
-            )
+          const page3 = await config.api.viewV2.search(view.id, {
+            paginate: true,
+            limit: 4,
+            bookmark: page2.bookmark,
+            query: {},
+            countRows: true,
+          })
+          const expectation: SearchResponse<Row> = {
+            rows: expect.arrayContaining(rows.slice(8)),
+            hasNextPage: false,
+            totalRows: 10,
           }
-        )
-      })
-
-      it("can query on top of the view filters", async () => {
-        await config.api.row.save(table._id!, {
-          one: "foo",
-          two: "bar",
-        })
-        await config.api.row.save(table._id!, {
-          one: "foo2",
-          two: "bar2",
-        })
-        const three = await config.api.row.save(table._id!, {
-          one: "foo3",
-          two: "bar3",
+          if (isLucene) {
+            expectation.bookmark = expect.anything()
+          }
+          expect(page3).toEqual(expectation)
         })
 
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          query: [
+        const sortTestOptions: [
+          {
+            field: string
+            order?: SortOrder
+            type?: SortType
+          },
+          string[]
+        ][] = [
+          [
             {
-              operator: BasicOperator.NOT_EQUAL,
-              field: "one",
-              value: "foo2",
+              field: "name",
+              order: SortOrder.ASCENDING,
+              type: SortType.STRING,
             },
+            ["Alice", "Bob", "Charly", "Danny"],
           ],
-          schema: {
-            id: { visible: true },
-            one: { visible: true },
-            two: { visible: true },
-          },
-        })
-
-        const response = await config.api.viewV2.search(view.id, {
-          query: {
-            [BasicOperator.EQUAL]: {
-              two: "bar3",
+          [
+            {
+              field: "name",
             },
-            [BasicOperator.NOT_EMPTY]: {
-              two: null,
+            ["Alice", "Bob", "Charly", "Danny"],
+          ],
+          [
+            {
+              field: "name",
+              order: SortOrder.DESCENDING,
             },
-          },
-        })
-        expect(response.rows).toHaveLength(1)
-        expect(response.rows).toEqual(
-          expect.arrayContaining([expect.objectContaining({ _id: three._id })])
-        )
-      })
-
-      it("can query on top of the view filters (using or filters)", async () => {
-        const one = await config.api.row.save(table._id!, {
-          one: "foo",
-          two: "bar",
-        })
-        await config.api.row.save(table._id!, {
-          one: "foo2",
-          two: "bar2",
-        })
-        const three = await config.api.row.save(table._id!, {
-          one: "foo3",
-          two: "bar3",
-        })
-
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          query: [
+            ["Danny", "Charly", "Bob", "Alice"],
+          ],
+          [
             {
-              operator: BasicOperator.NOT_EQUAL,
-              field: "two",
-              value: "bar2",
+              field: "name",
+              order: SortOrder.DESCENDING,
+              type: SortType.STRING,
             },
+            ["Danny", "Charly", "Bob", "Alice"],
           ],
-          schema: {
-            id: { visible: true },
-            one: { visible: false },
-            two: { visible: true },
-          },
-        })
-
-        const response = await config.api.viewV2.search(view.id, {
-          query: {
-            allOr: true,
-            [BasicOperator.NOT_EQUAL]: {
-              two: "bar",
+          [
+            {
+              field: "age",
+              order: SortOrder.ASCENDING,
+              type: SortType.NUMBER,
             },
-            [BasicOperator.NOT_EMPTY]: {
-              two: null,
+            ["Danny", "Alice", "Charly", "Bob"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.ASCENDING,
             },
-          },
-        })
-        expect(response.rows).toHaveLength(2)
-        expect(response.rows).toEqual(
-          expect.arrayContaining([
-            expect.objectContaining({ _id: one._id }),
-            expect.objectContaining({ _id: three._id }),
-          ])
-        )
-      })
+            ["Danny", "Alice", "Charly", "Bob"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.DESCENDING,
+            },
+            ["Bob", "Charly", "Alice", "Danny"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.DESCENDING,
+              type: SortType.NUMBER,
+            },
+            ["Bob", "Charly", "Alice", "Danny"],
+          ],
+        ]
 
-      isLucene &&
-        it.each([true, false])(
-          "in lucene, cannot override a view filter",
-          async allOr => {
-            await config.api.row.save(table._id!, {
-              one: "foo",
-              two: "bar",
-            })
-            const two = await config.api.row.save(table._id!, {
-              one: "foo2",
-              two: "bar2",
-            })
+        describe("sorting", () => {
+          let table: Table
+          const viewSchema = {
+            id: { visible: true },
+            age: { visible: true },
+            name: { visible: true },
+          }
 
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              query: [
-                {
-                  operator: BasicOperator.EQUAL,
-                  field: "two",
-                  value: "bar2",
+          beforeAll(async () => {
+            table = await config.api.table.save(
+              saveTableRequest({
+                type: "table",
+                schema: {
+                  name: {
+                    type: FieldType.STRING,
+                    name: "name",
+                  },
+                  surname: {
+                    type: FieldType.STRING,
+                    name: "surname",
+                  },
+                  age: {
+                    type: FieldType.NUMBER,
+                    name: "age",
+                  },
+                  address: {
+                    type: FieldType.STRING,
+                    name: "address",
+                  },
+                  jobTitle: {
+                    type: FieldType.STRING,
+                    name: "jobTitle",
+                  },
                 },
-              ],
-              schema: {
-                id: { visible: true },
-                one: { visible: false },
-                two: { visible: true },
-              },
-            })
+              })
+            )
 
-            const response = await config.api.viewV2.search(view.id, {
-              query: {
-                allOr,
-                equal: {
-                  two: "bar",
-                },
-              },
-            })
-            expect(response.rows).toHaveLength(1)
-            expect(response.rows).toEqual([
-              expect.objectContaining({ _id: two._id }),
-            ])
-          }
-        )
+            const users = [
+              { name: "Alice", age: 25 },
+              { name: "Bob", age: 30 },
+              { name: "Charly", age: 27 },
+              { name: "Danny", age: 15 },
+            ]
+            await Promise.all(
+              users.map(u =>
+                config.api.row.save(table._id!, {
+                  tableId: table._id,
+                  ...u,
+                })
+              )
+            )
+          })
 
-      !isLucene &&
-        it.each([true, false])(
-          "can filter a view without a view filter",
-          async allOr => {
-            const one = await config.api.row.save(table._id!, {
-              one: "foo",
-              two: "bar",
-            })
-            await config.api.row.save(table._id!, {
-              one: "foo2",
-              two: "bar2",
-            })
+          it.each(sortTestOptions)(
+            "allow sorting (%s)",
+            async (sortParams, expected) => {
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                sort: sortParams,
+                schema: viewSchema,
+              })
 
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              schema: {
-                id: { visible: true },
-                one: { visible: false },
-                two: { visible: true },
-              },
-            })
+              const response = await config.api.viewV2.search(view.id)
 
-            const response = await config.api.viewV2.search(view.id, {
-              query: {
-                allOr,
-                equal: {
-                  two: "bar",
+              expect(response.rows).toHaveLength(4)
+              expect(response.rows).toEqual(
+                expected.map(name => expect.objectContaining({ name }))
+              )
+            }
+          )
+
+          it.each(sortTestOptions)(
+            "allow override the default view sorting (%s)",
+            async (sortParams, expected) => {
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                sort: {
+                  field: "name",
+                  order: SortOrder.ASCENDING,
+                  type: SortType.STRING,
                 },
-              },
-            })
-            expect(response.rows).toHaveLength(1)
-            expect(response.rows).toEqual([
-              expect.objectContaining({ _id: one._id }),
-            ])
-          }
-        )
+                schema: viewSchema,
+              })
 
-      !isLucene &&
-        it.each([true, false])("cannot bypass a view filter", async allOr => {
+              const response = await config.api.viewV2.search(view.id, {
+                sort: sortParams.field,
+                sortOrder: sortParams.order,
+                sortType: sortParams.type,
+                query: {},
+              })
+
+              expect(response.rows).toHaveLength(4)
+              expect(response.rows).toEqual(
+                expected.map(name => expect.objectContaining({ name }))
+              )
+            }
+          )
+        })
+
+        it("can query on top of the view filters", async () => {
           await config.api.row.save(table._id!, {
             one: "foo",
             two: "bar",
@@ -2882,746 +2725,939 @@ describe.each([
             one: "foo2",
             two: "bar2",
           })
+          const three = await config.api.row.save(table._id!, {
+            one: "foo3",
+            two: "bar3",
+          })
 
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),
-            query: [
-              {
-                operator: BasicOperator.EQUAL,
-                field: "two",
-                value: "bar2",
-              },
-            ],
+            queryUI: {
+              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              groups: [
+                {
+                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  filters: [
+                    {
+                      operator: BasicOperator.NOT_EQUAL,
+                      field: "one",
+                      value: "foo2",
+                    },
+                  ],
+                },
+              ],
+            },
             schema: {
               id: { visible: true },
-              one: { visible: false },
+              one: { visible: true },
               two: { visible: true },
             },
           })
 
           const response = await config.api.viewV2.search(view.id, {
             query: {
-              allOr,
-              equal: {
-                two: "bar",
+              [BasicOperator.EQUAL]: {
+                two: "bar3",
+              },
+              [BasicOperator.NOT_EMPTY]: {
+                two: null,
               },
             },
           })
-          expect(response.rows).toHaveLength(0)
-        })
-
-      describe("foreign relationship columns", () => {
-        let envCleanup: () => void
-        beforeAll(() => {
-          envCleanup = features.testutils.setFeatureFlags("*", {
-            ENRICHED_RELATIONSHIPS: true,
-          })
-        })
-
-        afterAll(() => {
-          envCleanup?.()
-        })
-
-        const createMainTable = async (
-          links: {
-            name: string
-            tableId: string
-            fk: string
-          }[]
-        ) => {
-          const table = await config.api.table.save(
-            saveTableRequest({
-              schema: { title: { name: "title", type: FieldType.STRING } },
-            })
-          )
-          await config.api.table.save({
-            ...table,
-            schema: {
-              ...table.schema,
-              ...links.reduce<TableSchema>((acc, c) => {
-                acc[c.name] = {
-                  name: c.name,
-                  relationshipType: RelationshipType.ONE_TO_MANY,
-                  type: FieldType.LINK,
-                  tableId: c.tableId,
-                  fieldName: c.fk,
-                  constraints: { type: "array" },
-                }
-                return acc
-              }, {}),
-            },
-          })
-          return table
-        }
-        const createAuxTable = (schema: TableSchema) =>
-          config.api.table.save(
-            saveTableRequest({
-              primaryDisplay: "name",
-              schema: {
-                ...schema,
-                name: { name: "name", type: FieldType.STRING },
-              },
-            })
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows).toEqual(
+            expect.arrayContaining([
+              expect.objectContaining({ _id: three._id }),
+            ])
           )
+        })
 
-        it("returns squashed fields respecting the view config", async () => {
-          const auxTable = await createAuxTable({
-            age: { name: "age", type: FieldType.NUMBER },
-          })
-          const auxRow = await config.api.row.save(auxTable._id!, {
-            name: generator.name(),
-            age: generator.age(),
+        it("can query on top of the view filters (using or filters)", async () => {
+          const one = await config.api.row.save(table._id!, {
+            one: "foo",
+            two: "bar",
           })
-
-          const table = await createMainTable([
-            { name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
-          ])
           await config.api.row.save(table._id!, {
-            title: generator.word(),
-            aux: [auxRow],
+            one: "foo2",
+            two: "bar2",
+          })
+          const three = await config.api.row.save(table._id!, {
+            one: "foo3",
+            two: "bar3",
           })
 
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),
-            schema: {
-              title: { visible: true },
-              aux: {
-                visible: true,
-                columns: {
-                  name: { visible: false, readonly: false },
-                  age: { visible: true, readonly: true },
+            queryUI: {
+              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              groups: [
+                {
+                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  filters: [
+                    {
+                      operator: BasicOperator.NOT_EQUAL,
+                      field: "one",
+                      value: "foo2",
+                    },
+                  ],
                 },
+              ],
+            },
+            schema: {
+              id: { visible: true },
+              one: { visible: false },
+              two: { visible: true },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {
+              allOr: true,
+              [BasicOperator.NOT_EQUAL]: {
+                two: "bar",
+              },
+              [BasicOperator.NOT_EMPTY]: {
+                two: null,
               },
             },
           })
+          expect(response.rows).toHaveLength(2)
+          expect(response.rows).toEqual(
+            expect.arrayContaining([
+              expect.objectContaining({ _id: one._id }),
+              expect.objectContaining({ _id: three._id }),
+            ])
+          )
+        })
 
-          const response = await config.api.viewV2.search(view.id)
-          expect(response.rows).toEqual([
-            expect.objectContaining({
-              aux: [
-                {
-                  _id: auxRow._id,
-                  primaryDisplay: auxRow.name,
-                  age: auxRow.age,
+        !isLucene &&
+          it.each([true, false])(
+            "can filter a view without a view filter",
+            async allOr => {
+              const one = await config.api.row.save(table._id!, {
+                one: "foo",
+                two: "bar",
+              })
+              await config.api.row.save(table._id!, {
+                one: "foo2",
+                two: "bar2",
+              })
+
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                schema: {
+                  id: { visible: true },
+                  one: { visible: false },
+                  two: { visible: true },
                 },
-              ],
-            }),
-          ])
-        })
+              })
 
-        it("enriches squashed fields", async () => {
-          const auxTable = await createAuxTable({
-            user: {
-              name: "user",
-              type: FieldType.BB_REFERENCE_SINGLE,
-              subtype: BBReferenceFieldSubType.USER,
-              constraints: { presence: true },
-            },
-          })
-          const table = await createMainTable([
-            { name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
-          ])
+              const response = await config.api.viewV2.search(view.id, {
+                query: {
+                  allOr,
+                  equal: {
+                    two: "bar",
+                  },
+                },
+              })
+              expect(response.rows).toHaveLength(1)
+              expect(response.rows).toEqual([
+                expect.objectContaining({ _id: one._id }),
+              ])
+            }
+          )
 
-          const user = config.getUser()
-          const auxRow = await config.api.row.save(auxTable._id!, {
-            name: generator.name(),
-            user: user._id,
-          })
-          await config.api.row.save(table._id!, {
-            title: generator.word(),
-            aux: [auxRow],
-          })
+        !isLucene &&
+          it.each([true, false])("cannot bypass a view filter", async allOr => {
+            await config.api.row.save(table._id!, {
+              one: "foo",
+              two: "bar",
+            })
+            await config.api.row.save(table._id!, {
+              one: "foo2",
+              two: "bar2",
+            })
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              queryUI: {
+                onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+                logicalOperator: FilterGroupLogicalOperator.ALL,
+                groups: [
+                  {
+                    logicalOperator: FilterGroupLogicalOperator.ALL,
+                    filters: [
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "two",
+                        value: "bar2",
+                      },
+                    ],
+                  },
+                ],
+              },
+              schema: {
+                id: { visible: true },
+                one: { visible: false },
+                two: { visible: true },
+              },
+            })
 
-          const view = await config.api.viewV2.create({
-            tableId: table._id!,
-            name: generator.guid(),
-            schema: {
-              title: { visible: true },
-              aux: {
-                visible: true,
-                columns: {
-                  name: { visible: true, readonly: true },
-                  user: { visible: true, readonly: true },
+            const response = await config.api.viewV2.search(view.id, {
+              query: {
+                allOr,
+                equal: {
+                  two: "bar",
                 },
               },
-            },
+            })
+            expect(response.rows).toHaveLength(0)
           })
 
-          const response = await config.api.viewV2.search(view.id)
-
-          expect(response.rows).toEqual([
-            expect.objectContaining({
-              aux: [
-                {
-                  _id: auxRow._id,
-                  primaryDisplay: auxRow.name,
-                  name: auxRow.name,
-                  user: {
-                    _id: user._id,
-                    email: user.email,
-                    firstName: user.firstName,
-                    lastName: user.lastName,
-                    primaryDisplay: user.email,
-                  },
-                },
-              ],
-            }),
-          ])
-        })
-      })
+        describe("foreign relationship columns", () => {
+          let envCleanup: () => void
+          beforeAll(() => {
+            envCleanup = features.testutils.setFeatureFlags("*", {
+              ENRICHED_RELATIONSHIPS: true,
+            })
+          })
 
-      !isLucene &&
-        describe("calculations", () => {
-          let table: Table
-          let rows: Row[]
+          afterAll(() => {
+            envCleanup?.()
+          })
 
-          beforeAll(async () => {
-            table = await config.api.table.save(
+          const createMainTable = async (
+            links: {
+              name: string
+              tableId: string
+              fk: string
+            }[]
+          ) => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: { title: { name: "title", type: FieldType.STRING } },
+              })
+            )
+            await config.api.table.save({
+              ...table,
+              schema: {
+                ...table.schema,
+                ...links.reduce<TableSchema>((acc, c) => {
+                  acc[c.name] = {
+                    name: c.name,
+                    relationshipType: RelationshipType.ONE_TO_MANY,
+                    type: FieldType.LINK,
+                    tableId: c.tableId,
+                    fieldName: c.fk,
+                    constraints: { type: "array" },
+                  }
+                  return acc
+                }, {}),
+              },
+            })
+            return table
+          }
+          const createAuxTable = (schema: TableSchema) =>
+            config.api.table.save(
               saveTableRequest({
+                primaryDisplay: "name",
                 schema: {
-                  quantity: {
-                    type: FieldType.NUMBER,
-                    name: "quantity",
-                  },
-                  price: {
-                    type: FieldType.NUMBER,
-                    name: "price",
-                  },
+                  ...schema,
+                  name: { name: "name", type: FieldType.STRING },
                 },
               })
             )
 
-            rows = await Promise.all(
-              Array.from({ length: 10 }, () =>
-                config.api.row.save(table._id!, {
-                  quantity: generator.natural({ min: 1, max: 10 }),
-                  price: generator.natural({ min: 1, max: 10 }),
-                })
-              )
-            )
-          })
+          it("returns squashed fields respecting the view config", async () => {
+            const auxTable = await createAuxTable({
+              age: { name: "age", type: FieldType.NUMBER },
+            })
+            const auxRow = await config.api.row.save(auxTable._id!, {
+              name: generator.name(),
+              age: generator.age(),
+            })
+
+            const table = await createMainTable([
+              { name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
+            ])
+            await config.api.row.save(table._id!, {
+              title: generator.word(),
+              aux: [auxRow],
+            })
 
-          it("should be able to search by calculations", async () => {
             const view = await config.api.viewV2.create({
               tableId: table._id!,
-              type: ViewV2Type.CALCULATION,
               name: generator.guid(),
               schema: {
-                "Quantity Sum": {
+                title: { visible: true },
+                aux: {
                   visible: true,
-                  calculationType: CalculationType.SUM,
-                  field: "quantity",
+                  columns: {
+                    name: { visible: false, readonly: false },
+                    age: { visible: true, readonly: true },
+                  },
                 },
               },
             })
 
-            const response = await config.api.viewV2.search(view.id, {
-              query: {},
-            })
+            const response = await config.api.viewV2.search(view.id)
+            expect(response.rows).toEqual([
+              expect.objectContaining({
+                aux: [
+                  {
+                    _id: auxRow._id,
+                    primaryDisplay: auxRow.name,
+                    age: auxRow.age,
+                  },
+                ],
+              }),
+            ])
+          })
 
-            expect(response.rows).toHaveLength(1)
-            expect(response.rows).toEqual(
-              expect.arrayContaining([
-                expect.objectContaining({
-                  "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
-                }),
-              ])
-            )
+          it("enriches squashed fields", async () => {
+            const auxTable = await createAuxTable({
+              user: {
+                name: "user",
+                type: FieldType.BB_REFERENCE_SINGLE,
+                subtype: BBReferenceFieldSubType.USER,
+                constraints: { presence: true },
+              },
+            })
+            const table = await createMainTable([
+              { name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
+            ])
 
-            // Calculation views do not return rows that can be linked back to
-            // the source table, and so should not have an _id field.
-            for (const row of response.rows) {
-              expect("_id" in row).toBe(false)
-            }
-          })
+            const user = config.getUser()
+            const auxRow = await config.api.row.save(auxTable._id!, {
+              name: generator.name(),
+              user: user._id,
+            })
+            await config.api.row.save(table._id!, {
+              title: generator.word(),
+              aux: [auxRow],
+            })
 
-          it("should be able to group by a basic field", async () => {
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
-              type: ViewV2Type.CALCULATION,
               schema: {
-                quantity: {
-                  visible: true,
-                  field: "quantity",
-                },
-                "Total Price": {
+                title: { visible: true },
+                aux: {
                   visible: true,
-                  calculationType: CalculationType.SUM,
-                  field: "price",
+                  columns: {
+                    name: { visible: true, readonly: true },
+                    user: { visible: true, readonly: true },
+                  },
                 },
               },
             })
 
-            const response = await config.api.viewV2.search(view.id, {
-              query: {},
-            })
-
-            const priceByQuantity: Record<number, number> = {}
-            for (const row of rows) {
-              priceByQuantity[row.quantity] ??= 0
-              priceByQuantity[row.quantity] += row.price
-            }
+            const response = await config.api.viewV2.search(view.id)
 
-            for (const row of response.rows) {
-              expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity])
-            }
+            expect(response.rows).toEqual([
+              expect.objectContaining({
+                aux: [
+                  {
+                    _id: auxRow._id,
+                    primaryDisplay: auxRow.name,
+                    name: auxRow.name,
+                    user: {
+                      _id: user._id,
+                      email: user.email,
+                      firstName: user.firstName,
+                      lastName: user.lastName,
+                      primaryDisplay: user.email,
+                    },
+                  },
+                ],
+              }),
+            ])
           })
+        })
 
-          it.each([
-            CalculationType.COUNT,
-            CalculationType.SUM,
-            CalculationType.AVG,
-            CalculationType.MIN,
-            CalculationType.MAX,
-          ])("should be able to calculate $type", async type => {
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              type: ViewV2Type.CALCULATION,
-              schema: {
-                aggregate: {
-                  visible: true,
-                  calculationType: type,
-                  field: "price",
-                },
-              },
+        !isLucene &&
+          describe("calculations", () => {
+            let table: Table
+            let rows: Row[]
+
+            beforeAll(async () => {
+              table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    quantity: {
+                      type: FieldType.NUMBER,
+                      name: "quantity",
+                    },
+                    price: {
+                      type: FieldType.NUMBER,
+                      name: "price",
+                    },
+                  },
+                })
+              )
+
+              rows = await Promise.all(
+                Array.from({ length: 10 }, () =>
+                  config.api.row.save(table._id!, {
+                    quantity: generator.natural({ min: 1, max: 10 }),
+                    price: generator.natural({ min: 1, max: 10 }),
+                  })
+                )
+              )
             })
 
-            const response = await config.api.viewV2.search(view.id, {
-              query: {},
+            it("should be able to search by calculations", async () => {
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                type: ViewV2Type.CALCULATION,
+                name: generator.guid(),
+                schema: {
+                  "Quantity Sum": {
+                    visible: true,
+                    calculationType: CalculationType.SUM,
+                    field: "quantity",
+                  },
+                },
+              })
+
+              const response = await config.api.viewV2.search(view.id, {
+                query: {},
+              })
+
+              expect(response.rows).toHaveLength(1)
+              expect(response.rows).toEqual(
+                expect.arrayContaining([
+                  expect.objectContaining({
+                    "Quantity Sum": rows.reduce(
+                      (acc, r) => acc + r.quantity,
+                      0
+                    ),
+                  }),
+                ])
+              )
+
+              // Calculation views do not return rows that can be linked back to
+              // the source table, and so should not have an _id field.
+              for (const row of response.rows) {
+                expect("_id" in row).toBe(false)
+              }
             })
 
-            function calculate(
-              type: CalculationType,
-              numbers: number[]
-            ): number {
-              switch (type) {
-                case CalculationType.COUNT:
-                  return numbers.length
-                case CalculationType.SUM:
-                  return numbers.reduce((a, b) => a + b, 0)
-                case CalculationType.AVG:
-                  return numbers.reduce((a, b) => a + b, 0) / numbers.length
-                case CalculationType.MIN:
-                  return Math.min(...numbers)
-                case CalculationType.MAX:
-                  return Math.max(...numbers)
+            it("should be able to group by a basic field", async () => {
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  quantity: {
+                    visible: true,
+                    field: "quantity",
+                  },
+                  "Total Price": {
+                    visible: true,
+                    calculationType: CalculationType.SUM,
+                    field: "price",
+                  },
+                },
+              })
+
+              const response = await config.api.viewV2.search(view.id, {
+                query: {},
+              })
+
+              const priceByQuantity: Record<number, number> = {}
+              for (const row of rows) {
+                priceByQuantity[row.quantity] ??= 0
+                priceByQuantity[row.quantity] += row.price
               }
-            }
 
-            const prices = rows.map(row => row.price)
-            const expected = calculate(type, prices)
-            const actual = response.rows[0].aggregate
-
-            if (type === CalculationType.AVG) {
-              // The average calculation can introduce floating point rounding
-              // errors, so we need to compare to within a small margin of
-              // error.
-              expect(actual).toBeCloseTo(expected)
-            } else {
-              expect(actual).toEqual(expected)
-            }
-          })
+              for (const row of response.rows) {
+                expect(row["Total Price"]).toEqual(
+                  priceByQuantity[row.quantity]
+                )
+              }
+            })
 
-          it("should be able to do a COUNT(DISTINCT)", async () => {
-            const table = await config.api.table.save(
-              saveTableRequest({
+            it.each([
+              CalculationType.COUNT,
+              CalculationType.SUM,
+              CalculationType.AVG,
+              CalculationType.MIN,
+              CalculationType.MAX,
+            ])("should be able to calculate $type", async type => {
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
                 schema: {
-                  name: {
-                    name: "name",
-                    type: FieldType.STRING,
+                  aggregate: {
+                    visible: true,
+                    calculationType: type,
+                    field: "price",
                   },
                 },
               })
-            )
 
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              type: ViewV2Type.CALCULATION,
-              schema: {
-                count: {
-                  visible: true,
-                  calculationType: CalculationType.COUNT,
-                  distinct: true,
-                  field: "name",
+              const response = await config.api.viewV2.search(view.id, {
+                query: {},
+              })
+
+              function calculate(
+                type: CalculationType,
+                numbers: number[]
+              ): number {
+                switch (type) {
+                  case CalculationType.COUNT:
+                    return numbers.length
+                  case CalculationType.SUM:
+                    return numbers.reduce((a, b) => a + b, 0)
+                  case CalculationType.AVG:
+                    return numbers.reduce((a, b) => a + b, 0) / numbers.length
+                  case CalculationType.MIN:
+                    return Math.min(...numbers)
+                  case CalculationType.MAX:
+                    return Math.max(...numbers)
+                }
+              }
+
+              const prices = rows.map(row => row.price)
+              const expected = calculate(type, prices)
+              const actual = response.rows[0].aggregate
+
+              if (type === CalculationType.AVG) {
+                // The average calculation can introduce floating point rounding
+                // errors, so we need to compare to within a small margin of
+                // error.
+                expect(actual).toBeCloseTo(expected)
+              } else {
+                expect(actual).toEqual(expected)
+              }
+            })
+
+            it("should be able to do a COUNT(DISTINCT)", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    name: {
+                      name: "name",
+                      type: FieldType.STRING,
+                    },
+                  },
+                })
+              )
+
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  count: {
+                    visible: true,
+                    calculationType: CalculationType.COUNT,
+                    distinct: true,
+                    field: "name",
+                  },
                 },
-              },
+              })
+
+              await config.api.row.bulkImport(table._id!, {
+                rows: [
+                  {
+                    name: "John",
+                  },
+                  {
+                    name: "John",
+                  },
+                  {
+                    name: "Sue",
+                  },
+                ],
+              })
+
+              const { rows } = await config.api.row.search(view.id)
+              expect(rows).toHaveLength(1)
+              expect(rows[0].count).toEqual(2)
             })
 
-            await config.api.row.bulkImport(table._id!, {
-              rows: [
+            it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => {
+              await config.api.viewV2.create(
                 {
-                  name: "John",
+                  tableId: table._id!,
+                  name: generator.guid(),
+                  type: ViewV2Type.CALCULATION,
+                  schema: {
+                    count: {
+                      visible: true,
+                      calculationType: CalculationType.COUNT,
+                      distinct: true,
+                      field: "does not exist oh no",
+                    },
+                  },
                 },
                 {
-                  name: "John",
+                  status: 400,
+                  body: {
+                    message:
+                      'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema',
+                  },
+                }
+              )
+            })
+
+            it("should be able to filter rows on the view itself", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    quantity: {
+                      type: FieldType.NUMBER,
+                      name: "quantity",
+                    },
+                    price: {
+                      type: FieldType.NUMBER,
+                      name: "price",
+                    },
+                  },
+                })
+              )
+
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                queryUI: {
+                  onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  groups: [
+                    {
+                      logicalOperator: FilterGroupLogicalOperator.ALL,
+                      filters: [
+                        {
+                          operator: BasicOperator.EQUAL,
+                          field: "quantity",
+                          value: 1,
+                        },
+                      ],
+                    },
+                  ],
                 },
-                {
-                  name: "Sue",
+                schema: {
+                  sum: {
+                    visible: true,
+                    calculationType: CalculationType.SUM,
+                    field: "price",
+                  },
                 },
-              ],
+              })
+
+              await config.api.row.bulkImport(table._id!, {
+                rows: [
+                  {
+                    quantity: 1,
+                    price: 1,
+                  },
+                  {
+                    quantity: 1,
+                    price: 2,
+                  },
+                  {
+                    quantity: 2,
+                    price: 10,
+                  },
+                ],
+              })
+
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {},
+              })
+              expect(rows).toHaveLength(1)
+              expect(rows[0].sum).toEqual(3)
             })
 
-            const { rows } = await config.api.row.search(view.id)
-            expect(rows).toHaveLength(1)
-            expect(rows[0].count).toEqual(2)
-          })
+            it("should be able to filter on group by fields", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    quantity: {
+                      type: FieldType.NUMBER,
+                      name: "quantity",
+                    },
+                    price: {
+                      type: FieldType.NUMBER,
+                      name: "price",
+                    },
+                  },
+                })
+              )
 
-          it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => {
-            await config.api.viewV2.create(
-              {
+              const view = await config.api.viewV2.create({
                 tableId: table._id!,
                 name: generator.guid(),
                 type: ViewV2Type.CALCULATION,
                 schema: {
-                  count: {
+                  quantity: { visible: true },
+                  sum: {
                     visible: true,
-                    calculationType: CalculationType.COUNT,
-                    distinct: true,
-                    field: "does not exist oh no",
+                    calculationType: CalculationType.SUM,
+                    field: "price",
                   },
                 },
-              },
-              {
-                status: 400,
-                body: {
-                  message:
-                    'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema',
-                },
-              }
-            )
-          })
+              })
 
-          it("should be able to filter rows on the view itself", async () => {
-            const table = await config.api.table.save(
-              saveTableRequest({
-                schema: {
-                  quantity: {
-                    type: FieldType.NUMBER,
-                    name: "quantity",
+              await config.api.row.bulkImport(table._id!, {
+                rows: [
+                  {
+                    quantity: 1,
+                    price: 1,
                   },
-                  price: {
-                    type: FieldType.NUMBER,
-                    name: "price",
+                  {
+                    quantity: 1,
+                    price: 2,
                   },
-                },
+                  {
+                    quantity: 2,
+                    price: 10,
+                  },
+                ],
               })
-            )
 
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              type: ViewV2Type.CALCULATION,
-              query: {
-                equal: {
-                  quantity: 1,
-                },
-              },
-              schema: {
-                sum: {
-                  visible: true,
-                  calculationType: CalculationType.SUM,
-                  field: "price",
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {
+                  equal: {
+                    quantity: 1,
+                  },
                 },
-              },
-            })
+              })
 
-            await config.api.row.bulkImport(table._id!, {
-              rows: [
-                {
-                  quantity: 1,
-                  price: 1,
-                },
-                {
-                  quantity: 1,
-                  price: 2,
-                },
-                {
-                  quantity: 2,
-                  price: 10,
-                },
-              ],
+              expect(rows).toHaveLength(1)
+              expect(rows[0].sum).toEqual(3)
             })
 
-            const { rows } = await config.api.viewV2.search(view.id, {
-              query: {},
-            })
-            expect(rows).toHaveLength(1)
-            expect(rows[0].sum).toEqual(3)
-          })
+            it("should be able to sort by group by field", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    quantity: {
+                      type: FieldType.NUMBER,
+                      name: "quantity",
+                    },
+                    price: {
+                      type: FieldType.NUMBER,
+                      name: "price",
+                    },
+                  },
+                })
+              )
 
-          it("should be able to filter on group by fields", async () => {
-            const table = await config.api.table.save(
-              saveTableRequest({
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
                 schema: {
-                  quantity: {
-                    type: FieldType.NUMBER,
-                    name: "quantity",
-                  },
-                  price: {
-                    type: FieldType.NUMBER,
-                    name: "price",
+                  quantity: { visible: true },
+                  sum: {
+                    visible: true,
+                    calculationType: CalculationType.SUM,
+                    field: "price",
                   },
                 },
               })
-            )
 
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              type: ViewV2Type.CALCULATION,
-              schema: {
-                quantity: { visible: true },
-                sum: {
-                  visible: true,
-                  calculationType: CalculationType.SUM,
-                  field: "price",
-                },
-              },
-            })
+              await config.api.row.bulkImport(table._id!, {
+                rows: [
+                  {
+                    quantity: 1,
+                    price: 1,
+                  },
+                  {
+                    quantity: 1,
+                    price: 2,
+                  },
+                  {
+                    quantity: 2,
+                    price: 10,
+                  },
+                ],
+              })
 
-            await config.api.row.bulkImport(table._id!, {
-              rows: [
-                {
-                  quantity: 1,
-                  price: 1,
-                },
-                {
-                  quantity: 1,
-                  price: 2,
-                },
-                {
-                  quantity: 2,
-                  price: 10,
-                },
-              ],
-            })
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {},
+                sort: "quantity",
+                sortOrder: SortOrder.DESCENDING,
+              })
 
-            const { rows } = await config.api.viewV2.search(view.id, {
-              query: {
-                equal: {
-                  quantity: 1,
-                },
-              },
+              expect(rows).toEqual([
+                expect.objectContaining({ quantity: 2, sum: 10 }),
+                expect.objectContaining({ quantity: 1, sum: 3 }),
+              ])
             })
 
-            expect(rows).toHaveLength(1)
-            expect(rows[0].sum).toEqual(3)
-          })
+            it("should be able to sort by a calculation", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    quantity: {
+                      type: FieldType.NUMBER,
+                      name: "quantity",
+                    },
+                    price: {
+                      type: FieldType.NUMBER,
+                      name: "price",
+                    },
+                  },
+                })
+              )
 
-          it("should be able to sort by group by field", async () => {
-            const table = await config.api.table.save(
-              saveTableRequest({
-                schema: {
-                  quantity: {
-                    type: FieldType.NUMBER,
-                    name: "quantity",
+              await config.api.row.bulkImport(table._id!, {
+                rows: [
+                  {
+                    quantity: 1,
+                    price: 1,
                   },
-                  price: {
-                    type: FieldType.NUMBER,
-                    name: "price",
+                  {
+                    quantity: 1,
+                    price: 2,
                   },
-                },
+                  {
+                    quantity: 2,
+                    price: 10,
+                  },
+                ],
               })
-            )
 
-            const view = await config.api.viewV2.create({
-              tableId: table._id!,
-              name: generator.guid(),
-              type: ViewV2Type.CALCULATION,
-              schema: {
-                quantity: { visible: true },
-                sum: {
-                  visible: true,
-                  calculationType: CalculationType.SUM,
-                  field: "price",
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  quantity: { visible: true },
+                  sum: {
+                    visible: true,
+                    calculationType: CalculationType.SUM,
+                    field: "price",
+                  },
                 },
-              },
-            })
+              })
 
-            await config.api.row.bulkImport(table._id!, {
-              rows: [
-                {
-                  quantity: 1,
-                  price: 1,
-                },
-                {
-                  quantity: 1,
-                  price: 2,
-                },
-                {
-                  quantity: 2,
-                  price: 10,
-                },
-              ],
-            })
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {},
+                sort: "sum",
+                sortOrder: SortOrder.DESCENDING,
+              })
 
-            const { rows } = await config.api.viewV2.search(view.id, {
-              query: {},
-              sort: "quantity",
-              sortOrder: SortOrder.DESCENDING,
+              expect(rows).toEqual([
+                expect.objectContaining({ quantity: 2, sum: 10 }),
+                expect.objectContaining({ quantity: 1, sum: 3 }),
+              ])
             })
-
-            expect(rows).toEqual([
-              expect.objectContaining({ quantity: 2, sum: 10 }),
-              expect.objectContaining({ quantity: 1, sum: 3 }),
-            ])
           })
 
-          it("should be able to sort by a calculation", async () => {
+        !isLucene &&
+          it("should not need required fields to be present", async () => {
             const table = await config.api.table.save(
               saveTableRequest({
                 schema: {
-                  quantity: {
-                    type: FieldType.NUMBER,
-                    name: "quantity",
+                  name: {
+                    name: "name",
+                    type: FieldType.STRING,
+                    constraints: {
+                      presence: true,
+                    },
                   },
-                  price: {
+                  age: {
+                    name: "age",
                     type: FieldType.NUMBER,
-                    name: "price",
                   },
                 },
               })
             )
 
-            await config.api.row.bulkImport(table._id!, {
-              rows: [
-                {
-                  quantity: 1,
-                  price: 1,
-                },
-                {
-                  quantity: 1,
-                  price: 2,
-                },
-                {
-                  quantity: 2,
-                  price: 10,
-                },
-              ],
-            })
+            await Promise.all([
+              config.api.row.save(table._id!, { name: "Steve", age: 30 }),
+              config.api.row.save(table._id!, { name: "Jane", age: 31 }),
+            ])
 
             const view = await config.api.viewV2.create({
               tableId: table._id!,
               name: generator.guid(),
               type: ViewV2Type.CALCULATION,
               schema: {
-                quantity: { visible: true },
                 sum: {
                   visible: true,
                   calculationType: CalculationType.SUM,
-                  field: "price",
+                  field: "age",
                 },
               },
             })
 
-            const { rows } = await config.api.viewV2.search(view.id, {
+            const response = await config.api.viewV2.search(view.id, {
               query: {},
-              sort: "sum",
-              sortOrder: SortOrder.DESCENDING,
             })
 
-            expect(rows).toEqual([
-              expect.objectContaining({ quantity: 2, sum: 10 }),
-              expect.objectContaining({ quantity: 1, sum: 3 }),
-            ])
+            expect(response.rows).toHaveLength(1)
+            expect(response.rows[0].sum).toEqual(61)
           })
-        })
 
-      !isLucene &&
-        it("should not need required fields to be present", async () => {
+        it("should be able to filter on a single user field in both the view query and search query", async () => {
           const table = await config.api.table.save(
             saveTableRequest({
               schema: {
-                name: {
-                  name: "name",
-                  type: FieldType.STRING,
-                  constraints: {
-                    presence: true,
-                  },
-                },
-                age: {
-                  name: "age",
-                  type: FieldType.NUMBER,
+                user: {
+                  name: "user",
+                  type: FieldType.BB_REFERENCE_SINGLE,
+                  subtype: BBReferenceFieldSubType.USER,
                 },
               },
             })
           )
 
-          await Promise.all([
-            config.api.row.save(table._id!, { name: "Steve", age: 30 }),
-            config.api.row.save(table._id!, { name: "Jane", age: 31 }),
-          ])
+          await config.api.row.save(table._id!, {
+            user: config.getUser()._id,
+          })
 
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),
-            type: ViewV2Type.CALCULATION,
+            queryUI: {
+              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              groups: [
+                {
+                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  filters: [
+                    {
+                      operator: BasicOperator.EQUAL,
+                      field: "user",
+                      value: "{{ [user].[_id] }}",
+                    },
+                  ],
+                },
+              ],
+            },
             schema: {
-              sum: {
+              user: {
                 visible: true,
-                calculationType: CalculationType.SUM,
-                field: "age",
               },
             },
           })
 
-          const response = await config.api.viewV2.search(view.id, {
-            query: {},
-          })
-
-          expect(response.rows).toHaveLength(1)
-          expect(response.rows[0].sum).toEqual(61)
-        })
-
-      it("should be able to filter on a single user field in both the view query and search query", async () => {
-        const table = await config.api.table.save(
-          saveTableRequest({
-            schema: {
-              user: {
-                name: "user",
-                type: FieldType.BB_REFERENCE_SINGLE,
-                subtype: BBReferenceFieldSubType.USER,
+          const { rows } = await config.api.viewV2.search(view.id, {
+            query: {
+              equal: {
+                user: "{{ [user].[_id] }}",
               },
             },
           })
-        )
-
-        await config.api.row.save(table._id!, {
-          user: config.getUser()._id,
-        })
-
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          query: {
-            equal: {
-              user: "{{ [user].[_id] }}",
-            },
-          },
-          schema: {
-            user: {
-              visible: true,
-            },
-          },
-        })
 
-        const { rows } = await config.api.viewV2.search(view.id, {
-          query: {
-            equal: {
-              user: "{{ [user].[_id] }}",
-            },
-          },
+          expect(rows).toHaveLength(1)
+          expect(rows[0].user._id).toEqual(config.getUser()._id)
         })
-
-        expect(rows).toHaveLength(1)
-        expect(rows[0].user._id).toEqual(config.getUser()._id)
       })
-    })
 
     describe("permissions", () => {
       beforeEach(async () => {
diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
index 45b091e0463..1468328921e 100644
--- a/packages/server/src/sdk/app/views/utils.ts
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -1,8 +1,13 @@
 import { ViewV2 } from "@budibase/types"
 import { utils, dataFilters } from "@budibase/shared-core"
+import { isPlainObject } from "lodash"
+
+function isEmptyObject(obj: any) {
+  return obj && isPlainObject(obj) && Object.keys(obj).length === 0
+}
 
 export function ensureQueryUISet(view: ViewV2) {
-  if (!view.queryUI && view.query) {
+  if (!view.queryUI && view.query && !isEmptyObject(view.query)) {
     if (!Array.isArray(view.query)) {
       // In practice this should not happen. `view.query`, at the time this code
       // goes into the codebase, only contains LegacyFilter[] in production.
@@ -24,7 +29,7 @@ export function ensureQueryUISet(view: ViewV2) {
 }
 
 export function ensureQuerySet(view: ViewV2) {
-  if (!view.query && view.queryUI) {
+  if (!view.query && view.queryUI && !isEmptyObject(view.queryUI)) {
     view.query = dataFilters.buildQuery(view.queryUI)
   }
 }
diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts
index 9741240f27b..ba07dbe5f04 100644
--- a/packages/server/src/tests/utilities/api/viewV2.ts
+++ b/packages/server/src/tests/utilities/api/viewV2.ts
@@ -10,7 +10,9 @@ import { Expectations, TestAPI } from "./base"
 
 export class ViewV2API extends TestAPI {
   create = async (
-    view: CreateViewRequest,
+    // The frontend changed in v3 from sending query to sending only queryUI,
+    // making sure tests reflect that.
+    view: Omit<CreateViewRequest, "query">,
     expectations?: Expectations
   ): Promise<ViewV2> => {
     const exp: Expectations = {
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 3b9253af6e6..721cc7565c5 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -319,9 +319,6 @@ const buildCondition = (expression: LegacyFilter) => {
     return
   }
 
-  const isHbs =
-    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
-
   if (operator === "allOr") {
     query.allOr = true
     return
@@ -338,40 +335,45 @@ const buildCondition = (expression: LegacyFilter) => {
     value = null
   }
 
-  if (
-    type === "datetime" &&
-    !isHbs &&
-    operator !== "empty" &&
-    operator !== "notEmpty"
-  ) {
-    // Ensure date value is a valid date and parse into correct format
-    if (!value) {
-      return
-    }
-    try {
-      value = new Date(value).toISOString()
-    } catch (error) {
-      return
-    }
-  }
-  if (type === "number" && typeof value === "string" && !isHbs) {
-    if (operator === "oneOf") {
-      value = value.split(",").map(item => parseFloat(item))
-    } else {
-      value = parseFloat(value)
-    }
-  }
-  if (type === "boolean") {
-    value = `${value}`?.toLowerCase() === "true"
-  }
-  if (
-    ["contains", "notContains", "containsAny"].includes(
-      operator.toLocaleString()
-    ) &&
-    type === "array" &&
-    typeof value === "string"
-  ) {
-    value = value.split(",")
+  const isHbs =
+    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+
+  // Parsing value depending on what the type is.
+  switch (type) {
+    case FieldType.DATETIME:
+      if (!isHbs && operator !== "empty" && operator !== "notEmpty") {
+        if (!value) {
+          return
+        }
+        try {
+          value = new Date(value).toISOString()
+        } catch (error) {
+          return
+        }
+      }
+      break
+    case FieldType.NUMBER:
+      if (typeof value === "string" && !isHbs) {
+        if (operator === "oneOf") {
+          value = value.split(",").map(parseFloat)
+        } else {
+          value = parseFloat(value)
+        }
+      }
+      break
+    case FieldType.BOOLEAN:
+      value = `${value}`.toLowerCase() === "true"
+      break
+    case FieldType.ARRAY:
+      if (
+        ["contains", "notContains", "containsAny"].includes(
+          operator.toLocaleString()
+        ) &&
+        typeof value === "string"
+      ) {
+        value = value.split(",")
+      }
+      break
   }
 
   if (isRangeSearchOperator(operator)) {
@@ -398,17 +400,17 @@ const buildCondition = (expression: LegacyFilter) => {
       ...query.range[field],
       low: value,
     }
-  } else if (isLogicalSearchOperator(operator)) {
-    // TODO
   } else if (
     isBasicSearchOperator(operator) ||
     isArraySearchOperator(operator) ||
     isRangeSearchOperator(operator)
   ) {
     if (type === "boolean") {
-      // Transform boolean filters to cope with null.
-      // "equals false" needs to be "not equals true"
-      // "not equals false" needs to be "equals true"
+      // TODO(samwho): I suspect this boolean transformation isn't needed anymore,
+      // write some tests to confirm.
+
+      // Transform boolean filters to cope with null.  "equals false" needs to
+      // be "not equals true" "not equals false" needs to be "equals true"
       if (operator === "equal" && value === false) {
         query.notEqual = query.notEqual || {}
         query.notEqual[field] = true
@@ -423,6 +425,8 @@ const buildCondition = (expression: LegacyFilter) => {
       query[operator] ??= {}
       query[operator][field] = value
     }
+  } else {
+    throw new Error(`Unsupported operator: ${operator}`)
   }
 
   return query

From 714029b9a036b16f02d3c4c4b411bc8c6f814fd2 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 17 Oct 2024 18:06:45 +0100
Subject: [PATCH 278/455] Making more progress on testing view searching.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 148 ++++++++++++++++++
 packages/server/src/sdk/app/rows/search.ts    |   6 -
 packages/shared-core/src/filters.ts           |   4 +-
 3 files changed, 151 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 71506da0fa9..c2626155c92 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -29,12 +29,18 @@ import {
   JsonTypes,
   FilterGroupLogicalOperator,
   EmptyFilterOption,
+  JsonFieldSubType,
+  SearchFilterGroup,
+  LegacyFilter,
+  SearchViewRowRequest,
+  SearchFilterChild,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { db, roles, features } from "@budibase/backend-core"
+import { single } from "validate.js"
 
 describe.each([
   ["lucene", undefined],
@@ -3657,6 +3663,148 @@ describe.each([
           expect(rows).toHaveLength(1)
           expect(rows[0].user._id).toEqual(config.getUser()._id)
         })
+
+        describe("search operators", () => {
+          let table: Table
+          beforeEach(async () => {
+            table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  string: { name: "string", type: FieldType.STRING },
+                  longform: { name: "longform", type: FieldType.LONGFORM },
+                  options: {
+                    name: "options",
+                    type: FieldType.OPTIONS,
+                    constraints: { inclusion: ["a", "b", "c"] },
+                  },
+                  array: {
+                    name: "array",
+                    type: FieldType.ARRAY,
+                    constraints: {
+                      type: JsonFieldSubType.ARRAY,
+                      inclusion: ["a", "b", "c"],
+                    },
+                  },
+                  number: { name: "number", type: FieldType.NUMBER },
+                  bigint: { name: "bigint", type: FieldType.BIGINT },
+                  datetime: { name: "datetime", type: FieldType.DATETIME },
+                  timeOnly: {
+                    name: "timeOnly",
+                    type: FieldType.DATETIME,
+                    timeOnly: true,
+                  },
+                  boolean: { name: "boolean", type: FieldType.BOOLEAN },
+                  user: {
+                    name: "user",
+                    type: FieldType.BB_REFERENCE_SINGLE,
+                    subtype: BBReferenceFieldSubType.USER,
+                  },
+                  users: {
+                    name: "users",
+                    type: FieldType.BB_REFERENCE,
+                    subtype: BBReferenceFieldSubType.USER,
+                  },
+                },
+              })
+            )
+          })
+
+          interface TestCase {
+            name: string
+            query: SearchFilterGroup
+            insert: Row[]
+            expected: Row[]
+            searchOpts?: SearchViewRowRequest
+          }
+
+          function defaultQuery(
+            query: Partial<SearchFilterGroup>
+          ): SearchFilterGroup {
+            return {
+              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              groups: [],
+              ...query,
+            }
+          }
+
+          function defaultGroup(
+            group: Partial<SearchFilterChild>
+          ): SearchFilterChild {
+            return {
+              logicalOperator: FilterGroupLogicalOperator.ALL,
+              filters: [],
+              ...group,
+            }
+          }
+
+          function simpleQuery(...filters: LegacyFilter[]): SearchFilterGroup {
+            return defaultQuery({ groups: [defaultGroup({ filters })] })
+          }
+
+          const testCases: TestCase[] = [
+            {
+              name: "empty query return all",
+              insert: [{ string: "foo" }],
+              query: defaultQuery({
+                onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+              }),
+              expected: [{ string: "foo" }],
+            },
+            {
+              name: "empty query return none",
+              insert: [{ string: "foo" }],
+              query: defaultQuery({
+                onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+              }),
+              expected: [],
+            },
+            {
+              name: "simple string search",
+              insert: [{ string: "foo" }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "string",
+                value: "foo",
+              }),
+              expected: [{ string: "foo" }],
+            },
+            {
+              name: "non matching string search",
+              insert: [{ string: "foo" }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "string",
+                value: "bar",
+              }),
+              expected: [],
+            },
+          ]
+
+          it.only.each(testCases)(
+            "$name",
+            async ({ query, insert, expected, searchOpts }) => {
+              await config.api.row.bulkImport(table._id!, { rows: insert })
+
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                queryUI: query,
+                schema: {
+                  string: { visible: true },
+                },
+              })
+
+              const { rows } = await config.api.viewV2.search(
+                view.id,
+                searchOpts
+              )
+              expect(rows).toEqual(
+                expected.map(r => expect.objectContaining(r))
+              )
+            }
+          )
+        })
       })
 
     describe("permissions", () => {
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 7ac3bb8ead0..6abfe0c6818 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -89,9 +89,6 @@ export async function search(
     options = searchInputMapping(table, options)
 
     if (options.viewId) {
-      // Delete extraneous search params that cannot be overridden
-      delete options.query.onEmptyFilter
-
       const view = source as ViewV2
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
@@ -99,7 +96,6 @@ export async function search(
       let viewQuery = await enrichSearchContext(view.query || {}, context)
       viewQuery = dataFilters.buildQueryLegacy(viewQuery) || {}
       viewQuery = checkFilters(table, viewQuery)
-      delete viewQuery?.onEmptyFilter
 
       const sqsEnabled = await features.flags.isEnabled("SQS")
       const supportsLogicalOperators =
@@ -112,8 +108,6 @@ export async function search(
           ? view.query
           : []
 
-        delete options.query.onEmptyFilter
-
         // Extract existing fields
         const existingFields =
           queryFilters
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 721cc7565c5..756a8147101 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -600,7 +600,7 @@ export function buildQuery(
 
   const globalOperator = operatorMap[parsedFilter.logicalOperator]
 
-  return {
+  const ret = {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
       conditions: parsedFilter.groups?.map(group => {
@@ -614,6 +614,8 @@ export function buildQuery(
       }),
     },
   }
+
+  return ret
 }
 
 // The frontend can send single values for array fields sometimes, so to handle

From 1109e27b04f18a1a9e0b7da00b8fadb10dfaad25 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 18 Oct 2024 09:56:35 +0100
Subject: [PATCH 279/455] Display numbers using the locale format in tables

---
 .../frontend-core/src/components/grid/cells/TextCell.svelte | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/TextCell.svelte b/packages/frontend-core/src/components/grid/cells/TextCell.svelte
index 0cf0ab2004c..1f6415598f4 100644
--- a/packages/frontend-core/src/components/grid/cells/TextCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/TextCell.svelte
@@ -52,7 +52,11 @@
 {:else}
   <div class="text-cell" class:number={type === "number"}>
     <div class="value">
-      {value ?? ""}
+      {#if type === "number"}
+        {(value ?? "").toLocaleString()}
+      {:else}
+        {value ?? ""}
+      {/if}
     </div>
   </div>
 {/if}

From b7979f4719ad029a66258caed81fbccf0b5c6422 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 18 Oct 2024 10:32:28 +0100
Subject: [PATCH 280/455] Refactor SearchFilterGroup type to be less verbose
 and more clearly named.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 73 +++++++------------
 packages/server/src/sdk/app/views/utils.ts    |  4 +-
 packages/shared-core/src/filters.ts           | 52 ++++++-------
 packages/shared-core/src/utils.ts             | 24 +++---
 packages/types/src/api/web/searchFilter.ts    | 20 ++---
 packages/types/src/documents/app/view.ts      |  4 +-
 packages/types/src/sdk/search.ts              |  2 +-
 7 files changed, 72 insertions(+), 107 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index c2626155c92..fa1f1690f66 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -27,20 +27,18 @@ import {
   ViewV2Schema,
   ViewV2Type,
   JsonTypes,
-  FilterGroupLogicalOperator,
+  UILogicalOperator,
   EmptyFilterOption,
   JsonFieldSubType,
-  SearchFilterGroup,
+  UISearchFilter,
   LegacyFilter,
   SearchViewRowRequest,
-  SearchFilterChild,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { db, roles, features } from "@budibase/backend-core"
-import { single } from "validate.js"
 
 describe.each([
   ["lucene", undefined],
@@ -158,11 +156,11 @@ describe.each([
           tableId: table._id!,
           primaryDisplay: "id",
           queryUI: {
-            logicalOperator: FilterGroupLogicalOperator.ALL,
+            logicalOperator: UILogicalOperator.ALL,
             onEmptyFilter: EmptyFilterOption.RETURN_ALL,
             groups: [
               {
-                logicalOperator: FilterGroupLogicalOperator.ALL,
+                logicalOperator: UILogicalOperator.ALL,
                 filters: [
                   {
                     operator: BasicOperator.EQUAL,
@@ -2388,10 +2386,10 @@ describe.each([
             name: generator.guid(),
             queryUI: {
               onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: FilterGroupLogicalOperator.ALL,
+              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.EQUAL,
@@ -2452,10 +2450,10 @@ describe.each([
             name: generator.guid(),
             queryUI: {
               onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: FilterGroupLogicalOperator.ALL,
+              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.EQUAL,
@@ -2741,10 +2739,10 @@ describe.each([
             name: generator.guid(),
             queryUI: {
               onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: FilterGroupLogicalOperator.ALL,
+              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.NOT_EQUAL,
@@ -2799,10 +2797,10 @@ describe.each([
             name: generator.guid(),
             queryUI: {
               onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: FilterGroupLogicalOperator.ALL,
+              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.NOT_EQUAL,
@@ -2894,10 +2892,10 @@ describe.each([
               name: generator.guid(),
               queryUI: {
                 onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-                logicalOperator: FilterGroupLogicalOperator.ALL,
+                logicalOperator: UILogicalOperator.ALL,
                 groups: [
                   {
-                    logicalOperator: FilterGroupLogicalOperator.ALL,
+                    logicalOperator: UILogicalOperator.ALL,
                     filters: [
                       {
                         operator: BasicOperator.EQUAL,
@@ -3338,10 +3336,10 @@ describe.each([
                 type: ViewV2Type.CALCULATION,
                 queryUI: {
                   onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  logicalOperator: UILogicalOperator.ALL,
                   groups: [
                     {
-                      logicalOperator: FilterGroupLogicalOperator.ALL,
+                      logicalOperator: UILogicalOperator.ALL,
                       filters: [
                         {
                           operator: BasicOperator.EQUAL,
@@ -3631,10 +3629,10 @@ describe.each([
             name: generator.guid(),
             queryUI: {
               onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: FilterGroupLogicalOperator.ALL,
+              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: FilterGroupLogicalOperator.ALL,
+                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.EQUAL,
@@ -3711,52 +3709,31 @@ describe.each([
 
           interface TestCase {
             name: string
-            query: SearchFilterGroup
+            query: UISearchFilter
             insert: Row[]
             expected: Row[]
             searchOpts?: SearchViewRowRequest
           }
 
-          function defaultQuery(
-            query: Partial<SearchFilterGroup>
-          ): SearchFilterGroup {
-            return {
-              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: FilterGroupLogicalOperator.ALL,
-              groups: [],
-              ...query,
-            }
-          }
-
-          function defaultGroup(
-            group: Partial<SearchFilterChild>
-          ): SearchFilterChild {
-            return {
-              logicalOperator: FilterGroupLogicalOperator.ALL,
-              filters: [],
-              ...group,
-            }
-          }
-
-          function simpleQuery(...filters: LegacyFilter[]): SearchFilterGroup {
-            return defaultQuery({ groups: [defaultGroup({ filters })] })
+          function simpleQuery(...filters: LegacyFilter[]): UISearchFilter {
+            return { groups: [{ filters }] }
           }
 
           const testCases: TestCase[] = [
             {
               name: "empty query return all",
               insert: [{ string: "foo" }],
-              query: defaultQuery({
+              query: {
                 onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              }),
+              },
               expected: [{ string: "foo" }],
             },
             {
               name: "empty query return none",
               insert: [{ string: "foo" }],
-              query: defaultQuery({
+              query: {
                 onEmptyFilter: EmptyFilterOption.RETURN_NONE,
-              }),
+              },
               expected: [],
             },
             {
diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
index 1468328921e..a110acf0722 100644
--- a/packages/server/src/sdk/app/views/utils.ts
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -14,8 +14,8 @@ export function ensureQueryUISet(view: ViewV2) {
       // We're changing it in the change that this comment is part of to also
       // include SearchFilters objects. These are created when we receive an
       // update to a ViewV2 that contains a queryUI and not a query field. We
-      // can convert SearchFilterGroup (the type of queryUI) to SearchFilters,
-      // but not LegacyFilter[], they are incompatible due to SearchFilterGroup
+      // can convert UISearchFilter (the type of queryUI) to SearchFilters,
+      // but not LegacyFilter[], they are incompatible due to UISearchFilter
       // and SearchFilters being recursive types.
       //
       // So despite the type saying that `view.query` is a LegacyFilter[] |
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 756a8147101..932d5a8ca02 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -19,8 +19,8 @@ import {
   RangeOperator,
   LogicalOperator,
   isLogicalSearchOperator,
-  SearchFilterGroup,
-  FilterGroupLogicalOperator,
+  UISearchFilter,
+  UILogicalOperator,
   isBasicSearchOperator,
   isArraySearchOperator,
   isRangeSearchOperator,
@@ -561,7 +561,7 @@ export const buildQueryLegacy = (
 }
 
 /**
- * Converts a **SearchFilterGroup** filter definition into a grouped
+ * Converts a **UISearchFilter** filter definition into a grouped
  * search query of type **SearchFilters**
  *
  * Legacy support remains for the old **SearchFilter[]** format.
@@ -573,49 +573,41 @@ export const buildQueryLegacy = (
  */
 export function buildQuery(filter: undefined): undefined
 export function buildQuery(
-  filter: SearchFilterGroup | LegacyFilter[]
+  filter: UISearchFilter | LegacyFilter[]
 ): SearchFilters
 export function buildQuery(
-  filter?: SearchFilterGroup | LegacyFilter[]
+  filter?: UISearchFilter | LegacyFilter[]
 ): SearchFilters | undefined {
   if (!filter) {
     return
   }
 
-  let parsedFilter: SearchFilterGroup
+  let parsedFilter: UISearchFilter
   if (Array.isArray(filter)) {
     parsedFilter = processSearchFilters(filter)
   } else {
     parsedFilter = filter
   }
 
-  const operatorMap = {
-    [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
-    [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
-  }
+  const operator = logicalOperatorFromUI(
+    parsedFilter.logicalOperator || UILogicalOperator.ALL
+  )
+  const groups = parsedFilter.groups || []
+  const conditions: SearchFilters[] = groups.map(group => {
+    const filters = group.filters || []
+    return { [operator]: { conditions: filters.map(x => buildCondition(x)) } }
+  })
 
-  const globalOnEmpty = parsedFilter.onEmptyFilter
-    ? parsedFilter.onEmptyFilter
-    : null
-
-  const globalOperator = operatorMap[parsedFilter.logicalOperator]
-
-  const ret = {
-    ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
-    [globalOperator]: {
-      conditions: parsedFilter.groups?.map(group => {
-        return {
-          [operatorMap[group.logicalOperator]]: {
-            conditions: group.filters
-              ?.map(x => buildCondition(x))
-              .filter(filter => filter),
-          },
-        }
-      }),
-    },
+  return {
+    onEmptyFilter: parsedFilter.onEmptyFilter,
+    [operator]: { conditions },
   }
+}
 
-  return ret
+function logicalOperatorFromUI(operator: UILogicalOperator): LogicalOperator {
+  return operator === UILogicalOperator.ALL
+    ? LogicalOperator.AND
+    : LogicalOperator.OR
 }
 
 // The frontend can send single values for array fields sometimes, so to handle
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 98f7c9b2154..4f3c4f3deb5 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -1,13 +1,13 @@
 import {
   LegacyFilter,
-  SearchFilterGroup,
-  FilterGroupLogicalOperator,
+  UISearchFilter,
+  UILogicalOperator,
   SearchFilters,
   BasicOperator,
   ArrayOperator,
   isLogicalSearchOperator,
   EmptyFilterOption,
-  SearchFilterChild,
+  SearchFilterGroup,
 } from "@budibase/types"
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
@@ -132,18 +132,18 @@ export function isSupportedUserSearch(query: SearchFilters) {
 
 /**
  * Processes the filter config. Filters are migrated from
- * SearchFilter[] to SearchFilterGroup
+ * SearchFilter[] to UISearchFilter
  *
  * If filters is not an array, the migration is skipped
  *
- * @param {LegacyFilter[] | SearchFilterGroup} filters
+ * @param {LegacyFilter[] | UISearchFilter} filters
  */
 export const processSearchFilters = (
   filters: LegacyFilter[]
-): SearchFilterGroup => {
+): UISearchFilter => {
   // Base search config.
-  const defaultCfg: SearchFilterGroup = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
+  const defaultCfg: UISearchFilter = {
+    logicalOperator: UILogicalOperator.ALL,
     onEmptyFilter: EmptyFilterOption.RETURN_ALL,
     groups: [],
   }
@@ -159,11 +159,11 @@ export const processSearchFilters = (
     "formulaType",
   ]
 
-  let baseGroup: SearchFilterChild = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
+  let baseGroup: SearchFilterGroup = {
+    logicalOperator: UILogicalOperator.ALL,
   }
 
-  return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+  return filters.reduce((acc: UISearchFilter, filter: LegacyFilter) => {
     // Sort the properties for easier debugging
     const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
       .sort((a, b) => {
@@ -180,7 +180,7 @@ export const processSearchFilters = (
         acc.onEmptyFilter = value
       } else if (key === "operator" && value === "allOr") {
         // Group 1 logical operator
-        baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
+        baseGroup.logicalOperator = UILogicalOperator.ANY
       }
 
       return acc
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index 305f2bab007..ef35217f9d1 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -1,9 +1,5 @@
 import { FieldType } from "../../documents"
-import {
-  EmptyFilterOption,
-  FilterGroupLogicalOperator,
-  SearchFilters,
-} from "../../sdk"
+import { EmptyFilterOption, UILogicalOperator, SearchFilters } from "../../sdk"
 
 export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
@@ -14,15 +10,15 @@ export type LegacyFilter = {
   externalType?: string
 }
 
-export type SearchFilterChild = {
-  logicalOperator: FilterGroupLogicalOperator
-  groups?: SearchFilterChild[]
+export type SearchFilterGroup = {
+  logicalOperator?: UILogicalOperator
+  groups?: SearchFilterGroup[]
   filters?: LegacyFilter[]
 }
 
 // this is a type purely used by the UI
-export type SearchFilterGroup = {
-  logicalOperator: FilterGroupLogicalOperator
-  onEmptyFilter: EmptyFilterOption
-  groups: SearchFilterChild[]
+export type UISearchFilter = {
+  logicalOperator?: UILogicalOperator
+  onEmptyFilter?: EmptyFilterOption
+  groups?: SearchFilterGroup[]
 }
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 1b2372da857..3ae120fccac 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -1,4 +1,4 @@
-import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
+import { LegacyFilter, UISearchFilter, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
 import { DBView, SearchFilters } from "../../sdk"
@@ -92,7 +92,7 @@ export interface ViewV2 {
   tableId: string
   query?: LegacyFilter[] | SearchFilters
   // duplicate to store UI information about filters
-  queryUI?: SearchFilterGroup
+  queryUI?: UISearchFilter
   sort?: {
     field: string
     order?: SortOrder
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index d64e87d4347..a2d4b4760fd 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -203,7 +203,7 @@ export enum EmptyFilterOption {
   RETURN_NONE = "none",
 }
 
-export enum FilterGroupLogicalOperator {
+export enum UILogicalOperator {
   ALL = "all",
   ANY = "any",
 }

From 1509d7d650928bba8ff92e4207afbb9f1894259a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 18 Oct 2024 10:44:36 +0100
Subject: [PATCH 281/455] Use single number formatter for more performant
 locale string conversions

---
 .../frontend-core/src/components/grid/cells/TextCell.svelte | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/TextCell.svelte b/packages/frontend-core/src/components/grid/cells/TextCell.svelte
index 1f6415598f4..b6b67560683 100644
--- a/packages/frontend-core/src/components/grid/cells/TextCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/TextCell.svelte
@@ -1,3 +1,7 @@
+<script context="module">
+  const NumberFormatter = Intl.NumberFormat()
+</script>
+
 <script>
   import { onMount } from "svelte"
 
@@ -53,7 +57,7 @@
   <div class="text-cell" class:number={type === "number"}>
     <div class="value">
       {#if type === "number"}
-        {(value ?? "").toLocaleString()}
+        {NumberFormatter.format(value ?? "")}
       {:else}
         {value ?? ""}
       {/if}

From a3371974027175231980ccf6af8d43f7dc6f670e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 18 Oct 2024 11:01:15 +0100
Subject: [PATCH 282/455] Improve number formatting to handle additional edge
 cases

---
 .../components/grid/cells/NumberCell.svelte   | 24 ++++++++++++++++++-
 .../src/components/grid/cells/TextCell.svelte | 12 +++-------
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/NumberCell.svelte b/packages/frontend-core/src/components/grid/cells/NumberCell.svelte
index 2acec7d11e1..c8ae96ef21a 100644
--- a/packages/frontend-core/src/components/grid/cells/NumberCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/NumberCell.svelte
@@ -1,3 +1,7 @@
+<script context="module">
+  const NumberFormatter = Intl.NumberFormat()
+</script>
+
 <script>
   import TextCell from "./TextCell.svelte"
 
@@ -9,6 +13,24 @@
     const newValue = isNaN(float) ? null : float
     onChange(newValue)
   }
+
+  const formatNumber = value => {
+    const type = typeof value
+    if (type !== "string" && type !== "number") {
+      return ""
+    }
+    if (type === "string" && !value.trim().length) {
+      return ""
+    }
+    const res = NumberFormatter.format(value)
+    return res === "NaN" ? value : res
+  }
 </script>
 
-<TextCell {...$$props} onChange={numberOnChange} bind:api type="number" />
+<TextCell
+  {...$$props}
+  onChange={numberOnChange}
+  bind:api
+  type="number"
+  format={formatNumber}
+/>
diff --git a/packages/frontend-core/src/components/grid/cells/TextCell.svelte b/packages/frontend-core/src/components/grid/cells/TextCell.svelte
index b6b67560683..9275bca3c6b 100644
--- a/packages/frontend-core/src/components/grid/cells/TextCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/TextCell.svelte
@@ -1,7 +1,3 @@
-<script context="module">
-  const NumberFormatter = Intl.NumberFormat()
-</script>
-
 <script>
   import { onMount } from "svelte"
 
@@ -11,11 +7,13 @@
   export let type = "text"
   export let readonly = false
   export let api
+  export let format = null
 
   let input
   let active = false
 
   $: editable = focused && !readonly
+  $: displayValue = format?.(value) ?? value ?? ""
 
   const handleChange = e => {
     onChange(e.target.value)
@@ -56,11 +54,7 @@
 {:else}
   <div class="text-cell" class:number={type === "number"}>
     <div class="value">
-      {#if type === "number"}
-        {NumberFormatter.format(value ?? "")}
-      {:else}
-        {value ?? ""}
-      {/if}
+      {displayValue}
     </div>
   </div>
 {/if}

From 3981cecc30208ca7d34fa31c62bc97cf5a946a89 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 18 Oct 2024 11:32:58 +0100
Subject: [PATCH 283/455] Remove buildQueryLegacy

---
 packages/server/src/sdk/app/rows/search.ts |  10 +-
 packages/shared-core/src/filters.ts        | 128 ---------------------
 packages/types/src/api/web/searchFilter.ts |  11 +-
 3 files changed, 18 insertions(+), 131 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 6abfe0c6818..caf85a78581 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -5,6 +5,7 @@ import {
   Row,
   RowSearchParams,
   SearchFilterKey,
+  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -90,11 +91,16 @@ export async function search(
 
     if (options.viewId) {
       const view = source as ViewV2
+
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = await enrichSearchContext(view.query || {}, context)
-      viewQuery = dataFilters.buildQueryLegacy(viewQuery) || {}
+      let viewQuery = (await enrichSearchContext(view.query || {}, context)) as
+        | SearchFilters
+        | LegacyFilter[]
+      if (Array.isArray(viewQuery)) {
+        viewQuery = dataFilters.buildQuery(viewQuery)
+      }
       viewQuery = checkFilters(table, viewQuery)
 
       const sqsEnabled = await features.flags.isEnabled("SQS")
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 932d5a8ca02..eda1139c5ed 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -432,134 +432,6 @@ const buildCondition = (expression: LegacyFilter) => {
   return query
 }
 
-export const buildQueryLegacy = (
-  filter?: LegacyFilter[] | SearchFilters
-): SearchFilters | undefined => {
-  // this is of type SearchFilters or is undefined
-  if (!Array.isArray(filter)) {
-    return filter
-  }
-
-  let query: SearchFilters = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-    oneOf: {},
-    containsAny: {},
-  }
-
-  if (!Array.isArray(filter)) {
-    return query
-  }
-
-  filter.forEach(expression => {
-    let { operator, field, type, value, externalType, onEmptyFilter } =
-      expression
-    const queryOperator = operator as SearchFilterOperator
-    const isHbs =
-      typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
-    // Parse all values into correct types
-    if (operator === "allOr") {
-      query.allOr = true
-      return
-    }
-    if (onEmptyFilter) {
-      query.onEmptyFilter = onEmptyFilter
-      return
-    }
-    if (
-      type === "datetime" &&
-      !isHbs &&
-      queryOperator !== "empty" &&
-      queryOperator !== "notEmpty"
-    ) {
-      // Ensure date value is a valid date and parse into correct format
-      if (!value) {
-        return
-      }
-      try {
-        value = new Date(value).toISOString()
-      } catch (error) {
-        return
-      }
-    }
-    if (type === "number" && typeof value === "string" && !isHbs) {
-      if (queryOperator === "oneOf") {
-        value = value.split(",").map(item => parseFloat(item))
-      } else {
-        value = parseFloat(value)
-      }
-    }
-    if (type === "boolean") {
-      value = `${value}`?.toLowerCase() === "true"
-    }
-    if (
-      ["contains", "notContains", "containsAny"].includes(
-        operator.toLocaleString()
-      ) &&
-      type === "array" &&
-      typeof value === "string"
-    ) {
-      value = value.split(",")
-    }
-    if (operator.toLocaleString().startsWith("range") && query.range) {
-      const minint =
-        SqlNumberTypeRangeMap[
-          externalType as keyof typeof SqlNumberTypeRangeMap
-        ]?.min || Number.MIN_SAFE_INTEGER
-      const maxint =
-        SqlNumberTypeRangeMap[
-          externalType as keyof typeof SqlNumberTypeRangeMap
-        ]?.max || Number.MAX_SAFE_INTEGER
-      if (!query.range[field]) {
-        query.range[field] = {
-          low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-          high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-        }
-      }
-      if (operator === "rangeLow" && value != null && value !== "") {
-        query.range[field] = {
-          ...query.range[field],
-          low: value,
-        }
-      } else if (operator === "rangeHigh" && value != null && value !== "") {
-        query.range[field] = {
-          ...query.range[field],
-          high: value,
-        }
-      }
-    } else if (isLogicalSearchOperator(queryOperator)) {
-      // ignore
-    } else if (query[queryOperator] && operator !== "onEmptyFilter") {
-      if (type === "boolean") {
-        // Transform boolean filters to cope with null.
-        // "equals false" needs to be "not equals true"
-        // "not equals false" needs to be "equals true"
-        if (queryOperator === "equal" && value === false) {
-          query.notEqual = query.notEqual || {}
-          query.notEqual[field] = true
-        } else if (queryOperator === "notEqual" && value === false) {
-          query.equal = query.equal || {}
-          query.equal[field] = true
-        } else {
-          query[queryOperator] ??= {}
-          query[queryOperator]![field] = value
-        }
-      } else {
-        query[queryOperator] ??= {}
-        query[queryOperator]![field] = value
-      }
-    }
-  })
-  return query
-}
-
 /**
  * Converts a **UISearchFilter** filter definition into a grouped
  * search query of type **SearchFilters**
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index ef35217f9d1..428d5d364af 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -1,6 +1,8 @@
 import { FieldType } from "../../documents"
 import { EmptyFilterOption, UILogicalOperator, SearchFilters } from "../../sdk"
 
+// Prior to v2, this is the type the frontend sent us when filters were
+// involved. We convert this to a SearchFilters before use with the search SDK.
 export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
   onEmptyFilter?: EmptyFilterOption
@@ -16,7 +18,14 @@ export type SearchFilterGroup = {
   filters?: LegacyFilter[]
 }
 
-// this is a type purely used by the UI
+// As of v3, this is the format that the frontend always sends when search
+// filters are involved. We convert this to SearchFilters before use with the
+// search SDK.
+//
+// The reason we migrated was that we started to support "logical operators" in
+// tests and SearchFilters because a recursive data structure. LegacyFilter[]
+// wasn't able to support these sorts of recursive structures, so we changed the
+// format.
 export type UISearchFilter = {
   logicalOperator?: UILogicalOperator
   onEmptyFilter?: EmptyFilterOption

From 3d1b78623e6e2582e7105321a089e51a47e5292d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 18 Oct 2024 11:56:00 +0100
Subject: [PATCH 284/455] Remove frontend role prefixing hacks and fix 409s on
 role deletion

---
 packages/builder/src/stores/builder/roles.js | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/packages/builder/src/stores/builder/roles.js b/packages/builder/src/stores/builder/roles.js
index 994f87a8d3c..fd3581f1d47 100644
--- a/packages/builder/src/stores/builder/roles.js
+++ b/packages/builder/src/stores/builder/roles.js
@@ -47,28 +47,14 @@ export function createRolesStore() {
         roleId: role?._id,
         roleRev: role?._rev,
       })
-      store.update(state => state.filter(existing => existing._id !== role._id))
+      await actions.fetch()
     },
     save: async role => {
       const savedRole = await API.saveRole(role)
       await actions.fetch()
-
-      // When saving a role we get back an _id prefixed by role_, but the API does not want this
-      // in future requests
-      return {
-        ...savedRole,
-        _id: savedRole._id.replace("role_", ""),
-      }
+      return savedRole
     },
     replace: (roleId, role) => {
-      // Remove role_ prefix
-      if (roleId?.startsWith("role_")) {
-        roleId = roleId.replace("role_", "")
-      }
-      if (role?._id.startsWith("role_")) {
-        role._id = role._id.replace("role_", "")
-      }
-
       // Handles external updates of roles
       if (!roleId) {
         return

From 73999e4f3e5ea15729517d9b2c67f4fa7f204ead Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 18 Oct 2024 12:01:37 +0100
Subject: [PATCH 285/455] Remove old code

---
 packages/frontend-core/src/components/grid/stores/columns.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 86a0783c3f3..f23a17f14c2 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -160,7 +160,6 @@ export const initialise = context => {
             readonly: fieldSchema.readonly,
             order: fieldSchema.order ?? oldColumn?.order,
             conditions: fieldSchema.conditions,
-            enrichValue: fieldSchema.enrichValue,
             related: fieldSchema.related,
           }
           // Override a few properties for primary display

From 0fce17c3c06198505f8cddcf4c8f94d1511cacbb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 18 Oct 2024 14:50:06 +0100
Subject: [PATCH 286/455] Fix allOr for everything except SQS.

---
 .../src/api/routes/tests/viewV2.spec.ts       |  20 +++
 packages/server/src/sdk/app/rows/search.ts    |   9 +-
 packages/shared-core/src/filters.ts           |  83 ++++++++-----
 packages/shared-core/src/utils.ts             | 115 ++++--------------
 packages/types/src/api/web/searchFilter.ts    |  38 ++++--
 5 files changed, 135 insertions(+), 130 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index fa1f1690f66..6ba6fb8cb8e 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -3756,6 +3756,26 @@ describe.each([
               }),
               expected: [],
             },
+            {
+              name: "allOr",
+              insert: [{ string: "foo" }, { string: "bar" }],
+              query: simpleQuery(
+                {
+                  operator: BasicOperator.EQUAL,
+                  field: "string",
+                  value: "foo",
+                },
+                {
+                  operator: BasicOperator.EQUAL,
+                  field: "string",
+                  value: "bar",
+                },
+                {
+                  operator: "allOr",
+                }
+              ),
+              expected: [{ string: "foo" }, { string: "bar" }],
+            },
           ]
 
           it.only.each(testCases)(
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index caf85a78581..ee16058416d 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -114,11 +114,12 @@ export async function search(
           ? view.query
           : []
 
+        const { filters } = dataFilters.splitFiltersArray(queryFilters)
+
         // Extract existing fields
-        const existingFields =
-          queryFilters
-            ?.filter(filter => filter.field)
-            .map(filter => db.removeKeyNumbering(filter.field)) || []
+        const existingFields = filters.map(filter =>
+          db.removeKeyNumbering(filter.field)
+        )
 
         // Carry over filters for unused fields
         Object.keys(options.query).forEach(key => {
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index eda1139c5ed..2d01c243638 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -24,6 +24,7 @@ import {
   isBasicSearchOperator,
   isArraySearchOperator,
   isRangeSearchOperator,
+  SearchFilter,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -310,24 +311,17 @@ export class ColumnSplitter {
  * Builds a JSON query from the filter a SearchFilter definition
  * @param filter the builder filter structure
  */
-const buildCondition = (expression: LegacyFilter) => {
-  const query: SearchFilters = {}
-  const { operator, field, type, externalType, onEmptyFilter } = expression
-  let { value } = expression
-
-  if (!operator || !field) {
-    return
-  }
 
-  if (operator === "allOr") {
-    query.allOr = true
+function buildCondition(filter: undefined): undefined
+function buildCondition(filter: SearchFilter): SearchFilters
+function buildCondition(filter?: SearchFilter): SearchFilters | undefined {
+  if (!filter) {
     return
   }
 
-  if (onEmptyFilter) {
-    query.onEmptyFilter = onEmptyFilter
-    return
-  }
+  const query: SearchFilters = {}
+  const { operator, field, type, externalType } = filter
+  let { value } = filter
 
   // Default the value for noValue fields to ensure they are correctly added
   // to the final query
@@ -432,16 +426,36 @@ const buildCondition = (expression: LegacyFilter) => {
   return query
 }
 
+export interface LegacyFilterSplit {
+  allOr?: boolean
+  onEmptyFilter?: EmptyFilterOption
+  filters: SearchFilter[]
+}
+
+export function splitFiltersArray(filters: LegacyFilter[]) {
+  const split: LegacyFilterSplit = {
+    filters: [],
+  }
+
+  for (const filter of filters) {
+    if ("operator" in filter && filter.operator === "allOr") {
+      split.allOr = true
+    } else if ("onEmptyFilter" in filter) {
+      split.onEmptyFilter = filter.onEmptyFilter
+    } else {
+      split.filters.push(filter)
+    }
+  }
+
+  return split
+}
+
 /**
  * Converts a **UISearchFilter** filter definition into a grouped
  * search query of type **SearchFilters**
  *
  * Legacy support remains for the old **SearchFilter[]** format.
  * These will be migrated to an appropriate **SearchFilters** object, if encountered
- *
- * @param filter
- *
- * @returns {SearchFilters}
  */
 export function buildQuery(filter: undefined): undefined
 export function buildQuery(
@@ -454,26 +468,33 @@ export function buildQuery(
     return
   }
 
-  let parsedFilter: UISearchFilter
   if (Array.isArray(filter)) {
-    parsedFilter = processSearchFilters(filter)
-  } else {
-    parsedFilter = filter
+    filter = processSearchFilters(filter)
   }
 
   const operator = logicalOperatorFromUI(
-    parsedFilter.logicalOperator || UILogicalOperator.ALL
+    filter.logicalOperator || UILogicalOperator.ALL
   )
-  const groups = parsedFilter.groups || []
-  const conditions: SearchFilters[] = groups.map(group => {
-    const filters = group.filters || []
-    return { [operator]: { conditions: filters.map(x => buildCondition(x)) } }
-  })
 
-  return {
-    onEmptyFilter: parsedFilter.onEmptyFilter,
-    [operator]: { conditions },
+  const query: SearchFilters = {}
+  if (filter.onEmptyFilter) {
+    query.onEmptyFilter = filter.onEmptyFilter
   }
+
+  query[operator] = {
+    conditions: (filter.groups || []).map(group => {
+      const { allOr, onEmptyFilter, filters } = splitFiltersArray(
+        group.filters || []
+      )
+      if (onEmptyFilter) {
+        query.onEmptyFilter = onEmptyFilter
+      }
+      const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
+      return { [operator]: { conditions: filters.map(buildCondition) } }
+    }),
+  }
+
+  return query
 }
 
 function logicalOperatorFromUI(operator: UILogicalOperator): LogicalOperator {
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 4f3c4f3deb5..dce1b8b960d 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -6,15 +6,22 @@ import {
   BasicOperator,
   ArrayOperator,
   isLogicalSearchOperator,
-  EmptyFilterOption,
-  SearchFilterGroup,
+  SearchFilter,
 } from "@budibase/types"
 import * as Constants from "./constants"
-import { removeKeyNumbering } from "./filters"
-
-// an array of keys from filter type to properties that are in the type
-// this can then be converted using .fromEntries to an object
-type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+import { removeKeyNumbering, splitFiltersArray } from "./filters"
+import _ from "lodash"
+
+const FILTER_ALLOWED_KEYS = [
+  "field",
+  "operator",
+  "value",
+  "type",
+  "externalType",
+  "valueType",
+  "noValue",
+  "formulaType",
+]
 
 export function unreachable(
   value: never,
@@ -130,88 +137,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
   return true
 }
 
-/**
- * Processes the filter config. Filters are migrated from
- * SearchFilter[] to UISearchFilter
- *
- * If filters is not an array, the migration is skipped
- *
- * @param {LegacyFilter[] | UISearchFilter} filters
- */
 export const processSearchFilters = (
-  filters: LegacyFilter[]
+  filterArray: LegacyFilter[]
 ): UISearchFilter => {
-  // Base search config.
-  const defaultCfg: UISearchFilter = {
-    logicalOperator: UILogicalOperator.ALL,
-    onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-    groups: [],
-  }
-
-  const filterAllowedKeys = [
-    "field",
-    "operator",
-    "value",
-    "type",
-    "externalType",
-    "valueType",
-    "noValue",
-    "formulaType",
-  ]
-
-  let baseGroup: SearchFilterGroup = {
-    logicalOperator: UILogicalOperator.ALL,
-  }
-
-  return filters.reduce((acc: UISearchFilter, filter: LegacyFilter) => {
-    // Sort the properties for easier debugging
-    const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
-      .sort((a, b) => {
-        return a.localeCompare(b)
-      })
-      .filter(key => filter[key])
-
-    if (filterPropertyKeys.length == 1) {
-      const key = filterPropertyKeys[0],
-        value = filter[key]
-      // Global
-      if (key === "onEmptyFilter") {
-        // unset otherwise
-        acc.onEmptyFilter = value
-      } else if (key === "operator" && value === "allOr") {
-        // Group 1 logical operator
-        baseGroup.logicalOperator = UILogicalOperator.ANY
-      }
-
-      return acc
-    }
-
-    const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
-      (acc: AllowedFilters, key) => {
-        const value = filter[key]
-        if (filterAllowedKeys.includes(key)) {
-          if (key === "field") {
-            acc.push([key, removeKeyNumbering(value)])
-          } else {
-            acc.push([key, value])
-          }
-        }
-        return acc
+  const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray)
+  return {
+    onEmptyFilter,
+    groups: [
+      {
+        logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL,
+        filters: filters.map(filter => {
+          filter.field = removeKeyNumbering(filter.field)
+          return _.pick(filter, FILTER_ALLOWED_KEYS) as SearchFilter
+        }),
       },
-      []
-    )
-
-    const migratedFilter: LegacyFilter = Object.fromEntries(
-      allowedFilterSettings
-    ) as LegacyFilter
-
-    baseGroup.filters!.push(migratedFilter)
-
-    if (!acc.groups || !acc.groups.length) {
-      // init the base group
-      acc.groups = [baseGroup]
-    }
-
-    return acc
-  }, defaultCfg)
+    ],
+  }
 }
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index 428d5d364af..b21484f6edb 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -1,17 +1,41 @@
 import { FieldType } from "../../documents"
-import { EmptyFilterOption, UILogicalOperator, SearchFilters } from "../../sdk"
+import {
+  EmptyFilterOption,
+  UILogicalOperator,
+  BasicOperator,
+  RangeOperator,
+  ArrayOperator,
+} from "../../sdk"
 
-// Prior to v2, this is the type the frontend sent us when filters were
-// involved. We convert this to a SearchFilters before use with the search SDK.
-export type LegacyFilter = {
-  operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
-  onEmptyFilter?: EmptyFilterOption
+type AllOr = {
+  operator: "allOr"
+}
+
+type OnEmptyFilter = {
+  onEmptyFilter: EmptyFilterOption
+}
+
+// TODO(samwho): this could be broken down further
+export type SearchFilter = {
+  operator:
+    | BasicOperator
+    | RangeOperator
+    | ArrayOperator
+    | "rangeLow"
+    | "rangeHigh"
+  // Field name will often have a numerical prefix when coming from the frontend,
+  // use the ColumnSplitter class to remove it.
   field: string
-  type?: FieldType
   value: any
+  type?: FieldType
   externalType?: string
+  noValue?: boolean
 }
 
+// Prior to v2, this is the type the frontend sent us when filters were
+// involved. We convert this to a SearchFilters before use with the search SDK.
+export type LegacyFilter = AllOr | OnEmptyFilter | SearchFilter
+
 export type SearchFilterGroup = {
   logicalOperator?: UILogicalOperator
   groups?: SearchFilterGroup[]

From f53e68f52667777846122ceee9c2508eab09b4ca Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 18 Oct 2024 14:53:20 +0100
Subject: [PATCH 287/455] Fix allOr test

---
 .../server/src/api/routes/tests/viewV2.spec.ts | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 6ba6fb8cb8e..a738479f7da 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -3712,7 +3712,7 @@ describe.each([
             query: UISearchFilter
             insert: Row[]
             expected: Row[]
-            searchOpts?: SearchViewRowRequest
+            searchOpts?: Partial<SearchViewRowRequest>
           }
 
           function simpleQuery(...filters: LegacyFilter[]): UISearchFilter {
@@ -3758,7 +3758,7 @@ describe.each([
             },
             {
               name: "allOr",
-              insert: [{ string: "foo" }, { string: "bar" }],
+              insert: [{ string: "bar" }, { string: "foo" }],
               query: simpleQuery(
                 {
                   operator: BasicOperator.EQUAL,
@@ -3774,7 +3774,11 @@ describe.each([
                   operator: "allOr",
                 }
               ),
-              expected: [{ string: "foo" }, { string: "bar" }],
+              searchOpts: {
+                sort: "string",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ string: "bar" }, { string: "foo" }],
             },
           ]
 
@@ -3792,10 +3796,10 @@ describe.each([
                 },
               })
 
-              const { rows } = await config.api.viewV2.search(
-                view.id,
-                searchOpts
-              )
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {},
+                ...searchOpts,
+              })
               expect(rows).toEqual(
                 expected.map(r => expect.objectContaining(r))
               )

From 4b6a66627a865ef793d4484def99e9c214af2fbc Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 18 Oct 2024 15:28:03 +0100
Subject: [PATCH 288/455] More view search tests.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index a738479f7da..a2251c6ffc6 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -3780,6 +3780,81 @@ describe.each([
               },
               expected: [{ string: "bar" }, { string: "foo" }],
             },
+            {
+              name: "can find rows with fuzzy search",
+              insert: [{ string: "foo" }, { string: "bar" }],
+              query: simpleQuery({
+                operator: BasicOperator.FUZZY,
+                field: "string",
+                value: "fo",
+              }),
+              expected: [{ string: "foo" }],
+            },
+            {
+              name: "can find nothing with fuzzy search",
+              insert: [{ string: "foo" }, { string: "bar" }],
+              query: simpleQuery({
+                operator: BasicOperator.FUZZY,
+                field: "string",
+                value: "baz",
+              }),
+              expected: [],
+            },
+            {
+              name: "can find numeric rows",
+              insert: [{ number: 1 }, { number: 2 }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "number",
+                value: 1,
+              }),
+              expected: [{ number: 1 }],
+            },
+            {
+              name: "can find numeric values with rangeHigh",
+              insert: [{ number: 1 }, { number: 2 }, { number: 3 }],
+              query: simpleQuery({
+                operator: "rangeHigh",
+                field: "number",
+                value: 2,
+              }),
+              searchOpts: {
+                sort: "number",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ number: 1 }, { number: 2 }],
+            },
+            {
+              name: "can find numeric values with rangeLow",
+              insert: [{ number: 1 }, { number: 2 }, { number: 3 }],
+              query: simpleQuery({
+                operator: "rangeLow",
+                field: "number",
+                value: 2,
+              }),
+              searchOpts: {
+                sort: "number",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ number: 2 }, { number: 3 }],
+            },
+            {
+              name: "can find numeric values with full range",
+              insert: [{ number: 1 }, { number: 2 }, { number: 3 }],
+              query: simpleQuery(
+                {
+                  operator: "rangeHigh",
+                  field: "number",
+                  value: 2,
+                },
+                {
+                  operator: "rangeLow",
+                  field: "number",
+                  value: 2,
+                }
+              ),
+              expected: [{ number: 2 }],
+            },
           ]
 
           it.only.each(testCases)(
@@ -3793,6 +3868,16 @@ describe.each([
                 queryUI: query,
                 schema: {
                   string: { visible: true },
+                  longform: { visible: true },
+                  options: { visible: true },
+                  array: { visible: true },
+                  number: { visible: true },
+                  bigint: { visible: true },
+                  datetime: { visible: true },
+                  timeOnly: { visible: true },
+                  boolean: { visible: true },
+                  user: { visible: true },
+                  users: { visible: true },
                 },
               })
 

From c6aa83d988b860e54250237130dac9cd1276e696 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 18 Oct 2024 16:52:40 +0100
Subject: [PATCH 289/455] Fix labelling when sorting by calculation columns in
 tables

---
 .../components/grid/cells/HeaderCell.svelte   | 12 +++++++---
 packages/frontend-core/src/fetch/DataFetch.js | 22 ++++++++++++-------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 6c1c025fcdd..bae7bbf40bb 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -52,7 +52,7 @@
   $: sortedBy = column.name === $sort.column
   $: canMoveLeft = orderable && idx > 0
   $: canMoveRight = orderable && idx < $scrollableColumns.length - 1
-  $: sortingLabels = getSortingLabels(column.schema?.type)
+  $: sortingLabels = getSortingLabels(column)
   $: searchable = isColumnSearchable(column)
   $: resetSearchValue(column.name)
   $: searching = searchValue != null
@@ -66,8 +66,14 @@
     editIsOpen = false
   }
 
-  const getSortingLabels = type => {
-    switch (type) {
+  const getSortingLabels = column => {
+    if (column.calculationType) {
+      return {
+        ascending: "low-high",
+        descending: "high-low",
+      }
+    }
+    switch (column?.schema?.type) {
       case FieldType.NUMBER:
       case FieldType.BIGINT:
         return {
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index a056cdff5d2..fb1dbd58856 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { QueryUtils } from "../utils"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
+import { FieldType, SortOrder, SortType } from "@budibase/types"
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 
@@ -37,7 +38,7 @@ export default class DataFetch {
 
       // Sorting config
       sortColumn: null,
-      sortOrder: "ascending",
+      sortOrder: SortOrder.ASCENDING,
       sortType: null,
 
       // Pagination config
@@ -162,17 +163,22 @@ export default class DataFetch {
     // If we don't have a sort column specified then just ensure we don't set
     // any sorting params
     if (!this.options.sortColumn) {
-      this.options.sortOrder = "ascending"
+      this.options.sortOrder = SortOrder.ASCENDING
       this.options.sortType = null
     } else {
       // Otherwise determine what sort type to use base on sort column
-      const type = schema?.[this.options.sortColumn]?.type
-      this.options.sortType =
-        type === "number" || type === "bigint" ? "number" : "string"
-
+      this.options.sortType = SortType.STRING
+      const fieldSchema = schema?.[this.options.sortColumn]
+      if (
+        fieldSchema?.type === FieldType.NUMBER ||
+        fieldSchema?.type === FieldType.BIGINT ||
+        fieldSchema?.calculationType
+      ) {
+        this.options.sortType = SortType.NUMBER
+      }
       // If no sort order, default to ascending
       if (!this.options.sortOrder) {
-        this.options.sortOrder = "ascending"
+        this.options.sortOrder = SortOrder.ASCENDING
       }
     }
 
@@ -310,7 +316,7 @@ export default class DataFetch {
     let jsonAdditions = {}
     Object.keys(schema).forEach(fieldKey => {
       const fieldSchema = schema[fieldKey]
-      if (fieldSchema?.type === "json") {
+      if (fieldSchema?.type === FieldType.JSON) {
         const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, {
           squashObjects: true,
         })

From 80875fd9a0375ee44598f8cf110b4d5e6ffd246d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 18 Oct 2024 17:50:08 +0100
Subject: [PATCH 290/455] More tests.

---
 .../src/api/routes/tests/search.spec.ts       |   4 +-
 .../src/api/routes/tests/viewV2.spec.ts       | 137 ++++++++++++++++--
 packages/shared-core/src/filters.ts           |  10 +-
 3 files changed, 133 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index e0143c59388..f9032b8b610 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1602,7 +1602,7 @@ describe.each([
         })
       })
 
-    describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
+    describe("arrays", () => {
       beforeAll(async () => {
         tableOrViewId = await createTableOrView({
           numbers: {
@@ -2192,7 +2192,7 @@ describe.each([
       })
 
       describe("contains", () => {
-        it("successfully finds a row", async () => {
+        it.only("successfully finds a row", async () => {
           await expectQuery({
             contains: { users: [user1._id] },
           }).toContainExactly([
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index a2251c6ffc6..52e28c879f3 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -33,6 +33,7 @@ import {
   UISearchFilter,
   LegacyFilter,
   SearchViewRowRequest,
+  ArrayOperator,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -3686,11 +3687,6 @@ describe.each([
                   number: { name: "number", type: FieldType.NUMBER },
                   bigint: { name: "bigint", type: FieldType.BIGINT },
                   datetime: { name: "datetime", type: FieldType.DATETIME },
-                  timeOnly: {
-                    name: "timeOnly",
-                    type: FieldType.DATETIME,
-                    timeOnly: true,
-                  },
                   boolean: { name: "boolean", type: FieldType.BOOLEAN },
                   user: {
                     name: "user",
@@ -3701,6 +3697,9 @@ describe.each([
                     name: "users",
                     type: FieldType.BB_REFERENCE,
                     subtype: BBReferenceFieldSubType.USER,
+                    constraints: {
+                      type: JsonFieldSubType.ARRAY,
+                    },
                   },
                 },
               })
@@ -3709,9 +3708,9 @@ describe.each([
 
           interface TestCase {
             name: string
-            query: UISearchFilter
-            insert: Row[]
-            expected: Row[]
+            query: UISearchFilter | (() => UISearchFilter)
+            insert: Row[] | (() => Row[])
+            expected: Row[] | (() => Row[])
             searchOpts?: Partial<SearchViewRowRequest>
           }
 
@@ -3855,11 +3854,130 @@ describe.each([
               ),
               expected: [{ number: 2 }],
             },
+            {
+              name: "can find longform values",
+              insert: [{ longform: "foo" }, { longform: "bar" }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "longform",
+                value: "foo",
+              }),
+              expected: [{ longform: "foo" }],
+            },
+            {
+              name: "can find options values",
+              insert: [{ options: "a" }, { options: "b" }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "options",
+                value: "a",
+              }),
+              expected: [{ options: "a" }],
+            },
+            {
+              name: "can find array values",
+              insert: [
+                // Number field here is just to guarantee order.
+                { number: 1, array: ["a"] },
+                { number: 2, array: ["b"] },
+                { number: 3, array: ["a", "c"] },
+              ],
+              query: simpleQuery({
+                operator: ArrayOperator.CONTAINS,
+                field: "array",
+                value: "a",
+              }),
+              searchOpts: {
+                sort: "number",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ array: ["a"] }, { array: ["a", "c"] }],
+            },
+            {
+              name: "can find bigint values",
+              insert: [{ bigint: "1" }, { bigint: "2" }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "bigint",
+                type: FieldType.BIGINT,
+                value: "1",
+              }),
+              expected: [{ bigint: "1" }],
+            },
+            {
+              name: "can find datetime values",
+              insert: [
+                { datetime: "2021-01-01T00:00:00.000Z" },
+                { datetime: "2021-01-02T00:00:00.000Z" },
+              ],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "datetime",
+                type: FieldType.DATETIME,
+                value: "2021-01-01",
+              }),
+              expected: [{ datetime: "2021-01-01T00:00:00.000Z" }],
+            },
+            {
+              name: "can find boolean values",
+              insert: [{ boolean: true }, { boolean: false }],
+              query: simpleQuery({
+                operator: BasicOperator.EQUAL,
+                field: "boolean",
+                value: true,
+              }),
+              expected: [{ boolean: true }],
+            },
+            {
+              name: "can find user values",
+              insert: () => [{ user: config.getUser() }],
+              query: () =>
+                simpleQuery({
+                  operator: BasicOperator.EQUAL,
+                  field: "user",
+                  value: config.getUser()._id,
+                }),
+              expected: () => [
+                {
+                  user: expect.objectContaining({ _id: config.getUser()._id }),
+                },
+              ],
+            },
+            {
+              name: "can find users values",
+              insert: () => [{ users: [config.getUser()] }],
+              query: () =>
+                simpleQuery({
+                  operator: ArrayOperator.CONTAINS,
+                  field: "users",
+                  value: [config.getUser()._id],
+                }),
+              expected: () => [
+                {
+                  users: [
+                    expect.objectContaining({ _id: config.getUser()._id }),
+                  ],
+                },
+              ],
+            },
           ]
 
-          it.only.each(testCases)(
+          it.each(testCases)(
             "$name",
             async ({ query, insert, expected, searchOpts }) => {
+              // Some values can't be specified outside of a test (e.g. getting
+              // config.getUser(), it won't be initialised), so we use functions
+              // in those cases.
+              if (typeof insert === "function") {
+                insert = insert()
+              }
+              if (typeof expected === "function") {
+                expected = expected()
+              }
+              if (typeof query === "function") {
+                query = query()
+              }
+
               await config.api.row.bulkImport(table._id!, { rows: insert })
 
               const view = await config.api.viewV2.create({
@@ -3874,7 +3992,6 @@ describe.each([
                   number: { visible: true },
                   bigint: { visible: true },
                   datetime: { visible: true },
-                  timeOnly: { visible: true },
                   boolean: { visible: true },
                   user: { visible: true },
                   users: { visible: true },
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 2d01c243638..efdd3b0df2a 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -339,11 +339,7 @@ function buildCondition(filter?: SearchFilter): SearchFilters | undefined {
         if (!value) {
           return
         }
-        try {
-          value = new Date(value).toISOString()
-        } catch (error) {
-          return
-        }
+        value = new Date(value).toISOString()
       }
       break
     case FieldType.NUMBER:
@@ -490,7 +486,9 @@ export function buildQuery(
         query.onEmptyFilter = onEmptyFilter
       }
       const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
-      return { [operator]: { conditions: filters.map(buildCondition) } }
+      return {
+        [operator]: { conditions: filters.map(buildCondition).filter(f => f) },
+      }
     }),
   }
 

From b46626c926341b7c3fce5e2a46661c1ff506d67b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 18 Oct 2024 18:23:01 +0100
Subject: [PATCH 291/455] Removing ts-ignore.

---
 packages/server/src/api/routes/tests/row.spec.ts  | 1 -
 packages/server/src/tests/utilities/structures.ts | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fa288081c7b..66e931d0ab0 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2087,7 +2087,6 @@ describe.each([
         )
         tableId = table._id!
 
-        // @ts-ignore - until AI implemented
         const rowValues: Record<keyof typeof fullSchema, any> = {
           [FieldType.STRING]: generator.guid(),
           [FieldType.LONGFORM]: generator.paragraph(),
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index 1198dc514d4..b38cc6484fa 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -606,7 +606,6 @@ export function fullSchemaWithoutLinks({
 }): {
   [type in Exclude<FieldType, FieldType.LINK>]: FieldSchema & { type: type }
 } {
-  // @ts-ignore - until AI implemented
   return {
     [FieldType.STRING]: {
       name: "string",

From 795948f3a19d75d27f0681d2264b9ba8f2e36bab Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 21 Oct 2024 08:52:56 +0100
Subject: [PATCH 292/455] Use enums for sorting

---
 .gitignore                                         |  3 +--
 .../src/components/grid/cells/HeaderCell.svelte    | 14 ++++++++------
 .../components/grid/stores/datasources/nonPlus.js  |  5 +++--
 .../components/grid/stores/datasources/table.js    |  5 +++--
 .../components/grid/stores/datasources/viewV2.js   |  7 ++++---
 .../src/components/grid/stores/sort.js             | 10 +++++++---
 packages/frontend-core/src/fetch/TableFetch.js     |  3 ++-
 7 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/.gitignore b/.gitignore
index 32d1416f4ae..bac643e5df1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,11 +4,10 @@ packages/server/runtime_apps/
 .idea/
 bb-airgapped.tar.gz
 *.iml
-
 packages/server/build/oldClientVersions/**/*
 packages/builder/src/components/deploy/clientVersions.json
-
 packages/server/src/integrations/tests/utils/*.lock
+packages/builder/vite.config.mjs.timestamp*
 
 # Logs
 logs
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index bae7bbf40bb..93db2b6317d 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -6,7 +6,7 @@
   import { getColumnIcon } from "../../../utils/schema"
   import MigrationModal from "../controls/MigrationModal.svelte"
   import { debounce } from "../../../utils/utils"
-  import { FieldType, FormulaType } from "@budibase/types"
+  import { FieldType, FormulaType, SortOrder } from "@budibase/types"
   import { TableNames } from "../../../constants"
   import GridPopover from "../overlays/GridPopover.svelte"
 
@@ -143,7 +143,7 @@
   const sortAscending = () => {
     sort.set({
       column: column.name,
-      order: "ascending",
+      order: SortOrder.ASCENDING,
     })
     open = false
   }
@@ -151,7 +151,7 @@
   const sortDescending = () => {
     sort.set({
       column: column.name,
-      order: "descending",
+      order: SortOrder.DESCENDING,
     })
     open = false
   }
@@ -324,7 +324,7 @@
           <Icon
             hoverable
             size="S"
-            name={$sort.order === "descending"
+            name={$sort.order === SortOrder.DESCENDING
               ? "SortOrderDown"
               : "SortOrderUp"}
           />
@@ -372,7 +372,8 @@
           icon="SortOrderUp"
           on:click={sortAscending}
           disabled={!canBeSortColumn(column.schema) ||
-            (column.name === $sort.column && $sort.order === "ascending")}
+            (column.name === $sort.column &&
+              $sort.order === SortOrder.ASCENDING)}
         >
           Sort {sortingLabels.ascending}
         </MenuItem>
@@ -380,7 +381,8 @@
           icon="SortOrderDown"
           on:click={sortDescending}
           disabled={!canBeSortColumn(column.schema) ||
-            (column.name === $sort.column && $sort.order === "descending")}
+            (column.name === $sort.column &&
+              $sort.order === SortOrder.DESCENDING)}
         >
           Sort {sortingLabels.descending}
         </MenuItem>
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
index f3b8a6e02d8..ea558d6236c 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
@@ -1,3 +1,4 @@
+import { SortOrder } from "@budibase/types"
 import { get } from "svelte/store"
 
 export const createActions = context => {
@@ -84,7 +85,7 @@ export const initialise = context => {
     inlineFilters.set([])
     sort.set({
       column: get(initialSortColumn),
-      order: get(initialSortOrder) || "ascending",
+      order: get(initialSortOrder) || SortOrder.ASCENDING,
     })
 
     // Update fetch when filter changes
@@ -110,7 +111,7 @@ export const initialise = context => {
           return
         }
         $fetch.update({
-          sortOrder: $sort.order || "ascending",
+          sortOrder: $sort.order || SortOrder.ASCENDING,
           sortColumn: $sort.column,
         })
       })
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js
index 2ba95e3a74f..e415f5914b3 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/table.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/table.js
@@ -1,3 +1,4 @@
+import { SortOrder } from "@budibase/types"
 import { get } from "svelte/store"
 
 const SuppressErrors = true
@@ -93,7 +94,7 @@ export const initialise = context => {
     inlineFilters.set([])
     sort.set({
       column: get(initialSortColumn),
-      order: get(initialSortOrder) || "ascending",
+      order: get(initialSortOrder) || SortOrder.ASCENDING,
     })
 
     // Update fetch when filter changes
@@ -119,7 +120,7 @@ export const initialise = context => {
           return
         }
         $fetch.update({
-          sortOrder: $sort.order || "ascending",
+          sortOrder: $sort.order || SortOrder.ASCENDING,
           sortColumn: $sort.column,
         })
       })
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 56a7479b44c..46456034798 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -1,5 +1,6 @@
 import { get } from "svelte/store"
 import { dataFilters } from "@budibase/shared-core"
+import { SortOrder } from "@budibase/types"
 
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
@@ -117,7 +118,7 @@ export const initialise = context => {
     inlineFilters.set([])
     sort.set({
       column: get(initialSortColumn),
-      order: get(initialSortOrder) || "ascending",
+      order: get(initialSortOrder) || SortOrder.ASCENDING,
     })
 
     // Keep sort and filter state in line with the view definition when in builder
@@ -133,7 +134,7 @@ export const initialise = context => {
         if (!get(initialSortColumn)) {
           sort.set({
             column: $definition.sort?.field,
-            order: $definition.sort?.order || "ascending",
+            order: $definition.sort?.order || SortOrder.ASCENDING,
           })
         }
         // Only override filter state if we don't have an initial filter
@@ -166,7 +167,7 @@ export const initialise = context => {
             ...$view,
             sort: {
               field: $sort.column,
-              order: $sort.order || "ascending",
+              order: $sort.order || SortOrder.ASCENDING,
             },
           })
         }
diff --git a/packages/frontend-core/src/components/grid/stores/sort.js b/packages/frontend-core/src/components/grid/stores/sort.js
index 336570d012e..9ab393b11fd 100644
--- a/packages/frontend-core/src/components/grid/stores/sort.js
+++ b/packages/frontend-core/src/components/grid/stores/sort.js
@@ -1,5 +1,6 @@
 import { derived, get } from "svelte/store"
 import { memo } from "../../../utils"
+import { SortOrder } from "@budibase/types"
 
 export const createStores = context => {
   const { props } = context
@@ -8,7 +9,7 @@ export const createStores = context => {
   // Initialise to default props
   const sort = memo({
     column: $props.initialSortColumn,
-    order: $props.initialSortOrder || "ascending",
+    order: $props.initialSortOrder || SortOrder.ASCENDING,
   })
 
   return {
@@ -24,7 +25,10 @@ export const initialise = context => {
     sort.update(state => ({ ...state, column: newSortColumn }))
   })
   initialSortOrder.subscribe(newSortOrder => {
-    sort.update(state => ({ ...state, order: newSortOrder || "ascending" }))
+    sort.update(state => ({
+      ...state,
+      order: newSortOrder || SortOrder.ASCENDING,
+    }))
   })
 
   // Derive if the current sort column exists in the schema
@@ -40,7 +44,7 @@ export const initialise = context => {
     if (!exists) {
       sort.set({
         column: null,
-        order: "ascending",
+        order: SortOrder.ASCENDING,
       })
     }
   })
diff --git a/packages/frontend-core/src/fetch/TableFetch.js b/packages/frontend-core/src/fetch/TableFetch.js
index a13b1bd186a..ed17c20c795 100644
--- a/packages/frontend-core/src/fetch/TableFetch.js
+++ b/packages/frontend-core/src/fetch/TableFetch.js
@@ -1,5 +1,6 @@
 import { get } from "svelte/store"
 import DataFetch from "./DataFetch.js"
+import { SortOrder } from "@budibase/types"
 
 export default class TableFetch extends DataFetch {
   determineFeatureFlags() {
@@ -23,7 +24,7 @@ export default class TableFetch extends DataFetch {
         query,
         limit,
         sort: sortColumn,
-        sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+        sortOrder: sortOrder?.toLowerCase() ?? SortOrder.ASCENDING,
         sortType,
         paginate,
         bookmark: cursor,

From e4702070d84e6c63e36ddf357dc1d62c4372f2c4 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 21 Oct 2024 09:08:34 +0100
Subject: [PATCH 293/455] Lint. Remove duplicate entry

---
 packages/builder/src/constants/backend/automations.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/constants/backend/automations.js b/packages/builder/src/constants/backend/automations.js
index 1c40c925ac9..37ae35aea06 100644
--- a/packages/builder/src/constants/backend/automations.js
+++ b/packages/builder/src/constants/backend/automations.js
@@ -6,7 +6,6 @@ export const TriggerStepID = {
   WEBHOOK: "WEBHOOK",
   APP: "APP",
   CRON: "CRON",
-  ROW_ACTION: "ROW_ACTION",
 }
 
 export const ActionStepID = {

From e9fd5415be8b195fe117869b89bdf71f198492ca Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 21 Oct 2024 09:10:22 +0100
Subject: [PATCH 294/455] Bump Pro Version

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 1a749caba9c..297fdc937e9 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1a749caba9c85aab2645e5d00db479eb53d3f80f
+Subproject commit 297fdc937e9c650b4964fc1a942b60022b195865

From 915202badad5df9b43f4c47f3bcdd151150d7bf3 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 09:48:51 +0100
Subject: [PATCH 295/455] More tests.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 41 +++++++++++++++++++
 packages/server/src/sdk/app/views/internal.ts |  1 +
 2 files changed, 42 insertions(+)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 52e28c879f3..5f9997c189f 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -219,6 +219,47 @@ describe.each([
         expect(res).toEqual(expected)
       })
 
+      it.only("can create a view with just a query field, no queryUI, for backwards compatibility", async () => {
+        const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
+          name: generator.name(),
+          tableId: table._id!,
+          primaryDisplay: "id",
+          query: [
+            {
+              operator: BasicOperator.EQUAL,
+              field: "field",
+              value: "value",
+            },
+          ],
+          sort: {
+            field: "fieldToSort",
+            order: SortOrder.DESCENDING,
+            type: SortType.STRING,
+          },
+          schema: {
+            id: { visible: true },
+            Price: {
+              visible: true,
+            },
+          },
+        }
+        const res = await config.api.viewV2.create(newView)
+
+        const expected: ViewV2 = {
+          ...newView,
+          schema: {
+            id: { visible: true },
+            Price: {
+              visible: true,
+            },
+          },
+          id: expect.any(String),
+          version: 2,
+        }
+
+        expect(res).toEqual(expected)
+      })
+
       it("persist only UI schema overrides", async () => {
         const newView: CreateViewRequest = {
           name: generator.name(),
diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts
index 33b68759d72..fc6b59fefef 100644
--- a/packages/server/src/sdk/app/views/internal.ts
+++ b/packages/server/src/sdk/app/views/internal.ts
@@ -41,6 +41,7 @@ export async function create(
   }
 
   ensureQuerySet(view)
+  ensureQueryUISet(view)
 
   const db = context.getAppDB()
 

From 064721cefa16617bf4c6121ee15c4ca7d9897566 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 11:30:28 +0100
Subject: [PATCH 296/455] Make sure you can still create views with a query but
 no queryUI

---
 .../server/src/api/routes/tests/viewV2.spec.ts     | 14 ++++++++++++++
 packages/server/src/sdk/app/views/external.ts      |  1 +
 2 files changed, 15 insertions(+)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index fdb680c64e3..f1be2c1ae21 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -253,6 +253,20 @@ describe.each([
               visible: true,
             },
           },
+          queryUI: {
+            groups: [
+              {
+                logicalOperator: UILogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "field",
+                    value: "value",
+                  },
+                ],
+              },
+            ],
+          },
           id: expect.any(String),
           version: 2,
         }
diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts
index d5251122c9c..2484932232f 100644
--- a/packages/server/src/sdk/app/views/external.ts
+++ b/packages/server/src/sdk/app/views/external.ts
@@ -50,6 +50,7 @@ export async function create(
   }
 
   ensureQuerySet(view)
+  ensureQueryUISet(view)
 
   const db = context.getAppDB()
 

From 205d6d0671b6041f881630bdd482aba18c3fc6ab Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 11:39:22 +0100
Subject: [PATCH 297/455] Re-enable tests.

---
 packages/server/src/api/routes/tests/viewV2.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index f1be2c1ae21..8104d124c18 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -219,7 +219,7 @@ describe.each([
         expect(res).toEqual(expected)
       })
 
-      it.only("can create a view with just a query field, no queryUI, for backwards compatibility", async () => {
+      it("can create a view with just a query field, no queryUI, for backwards compatibility", async () => {
         const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
           name: generator.name(),
           tableId: table._id!,

From 67b814a5cadc73ba87fa97ffa72c5b432b5da27e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 13:35:53 +0100
Subject: [PATCH 298/455] Yet more tests.

---
 .../src/api/routes/tests/search.spec.ts       |   2 +-
 .../src/api/routes/tests/viewV2.spec.ts       | 238 +++++++++++++++++-
 packages/server/src/sdk/app/tables/getters.ts |  20 ++
 packages/server/src/sdk/app/views/external.ts |   1 +
 packages/server/src/sdk/app/views/internal.ts |   2 +
 packages/server/src/sdk/app/views/utils.ts    |   8 +-
 .../server/src/tests/utilities/api/viewV2.ts  |   4 +-
 7 files changed, 264 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index f9032b8b610..0aa2a80afe5 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2192,7 +2192,7 @@ describe.each([
       })
 
       describe("contains", () => {
-        it.only("successfully finds a row", async () => {
+        it("successfully finds a row", async () => {
           await expectQuery({
             contains: { users: [user1._id] },
           }).toContainExactly([
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 8104d124c18..a061a370ba4 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -39,7 +39,7 @@ import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import { db, roles, features } from "@budibase/backend-core"
+import { db, roles, features, context } from "@budibase/backend-core"
 
 describe.each([
   ["lucene", undefined],
@@ -56,7 +56,8 @@ describe.each([
   const isInternal = isSqs || isLucene
 
   let table: Table
-  let datasource: Datasource
+  let rawDatasource: Datasource | undefined
+  let datasource: Datasource | undefined
   let envCleanup: (() => void) | undefined
 
   function saveTableRequest(
@@ -113,8 +114,9 @@ describe.each([
     })
 
     if (dsProvider) {
+      rawDatasource = await dsProvider
       datasource = await config.createDatasource({
-        datasource: await dsProvider,
+        datasource: rawDatasource,
       })
     }
     table = await config.api.table.save(priceTable())
@@ -923,6 +925,7 @@ describe.each([
 
     describe("update", () => {
       let view: ViewV2
+      let table: Table
 
       beforeEach(async () => {
         table = await config.api.table.save(priceTable())
@@ -955,6 +958,21 @@ describe.each([
             query: [
               { operator: "equal", field: "newField", value: "thatValue" },
             ],
+            // Should also update queryUI because query was not previously set.
+            queryUI: {
+              groups: [
+                {
+                  logicalOperator: "all",
+                  filters: [
+                    {
+                      operator: "equal",
+                      field: "newField",
+                      value: "thatValue",
+                    },
+                  ],
+                },
+              ],
+            },
             schema: expect.anything(),
           },
         })
@@ -974,8 +992,8 @@ describe.each([
           query: [
             {
               operator: BasicOperator.EQUAL,
-              field: generator.word(),
-              value: generator.word(),
+              field: "newField",
+              value: "newValue",
             },
           ],
           sort: {
@@ -999,6 +1017,21 @@ describe.each([
         expect((await config.api.table.get(tableId)).views).toEqual({
           [view.name]: {
             ...updatedData,
+            // queryUI gets generated from query
+            queryUI: {
+              groups: [
+                {
+                  logicalOperator: "all",
+                  filters: [
+                    {
+                      operator: "equal",
+                      field: "newField",
+                      value: "newValue",
+                    },
+                  ],
+                },
+              ],
+            },
             schema: {
               ...table.schema,
               id: expect.objectContaining({
@@ -1244,6 +1277,145 @@ describe.each([
           )
         })
 
+      it("can update queryUI field and query gets regenerated", async () => {
+        await config.api.viewV2.update({
+          ...view,
+          queryUI: {
+            groups: [
+              {
+                logicalOperator: UILogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "field",
+                    value: "value",
+                  },
+                ],
+              },
+            ],
+          },
+        })
+
+        let updatedView = await config.api.viewV2.get(view.id)
+        expect(updatedView.query).toEqual({
+          $and: {
+            conditions: [
+              {
+                $and: {
+                  conditions: [
+                    {
+                      equal: { field: "value" },
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        })
+
+        await config.api.viewV2.update({
+          ...updatedView,
+          queryUI: {
+            groups: [
+              {
+                logicalOperator: UILogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "newField",
+                    value: "newValue",
+                  },
+                ],
+              },
+            ],
+          },
+        })
+
+        updatedView = await config.api.viewV2.get(view.id)
+        expect(updatedView.query).toEqual({
+          $and: {
+            conditions: [
+              {
+                $and: {
+                  conditions: [
+                    {
+                      equal: { newField: "newValue" },
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        })
+      })
+
+      it("can delete either query and it will get regenerated from queryUI", async () => {
+        await config.api.viewV2.update({
+          ...view,
+          query: [
+            {
+              operator: BasicOperator.EQUAL,
+              field: "field",
+              value: "value",
+            },
+          ],
+        })
+
+        let updatedView = await config.api.viewV2.get(view.id)
+        expect(updatedView.queryUI).toBeDefined()
+
+        await config.api.viewV2.update({
+          ...updatedView,
+          query: undefined,
+        })
+
+        updatedView = await config.api.viewV2.get(view.id)
+        expect(updatedView.query).toBeDefined()
+      })
+
+      // This is because the conversion from queryUI -> query loses data, so you
+      // can't accurately reproduce the original queryUI from the query. If
+      // query is a LegacyFilter[] we allow it, because for Budibase v3
+      // everything in the db had query set to a LegacyFilter[], and there's no
+      // loss of information converting from a LegacyFilter[] to a
+      // UISearchFilter. But we convert to a SearchFilters and that can't be
+      // accurately converted to a UISearchFilter.
+      it("can't regenerate queryUI from a query once it has been generated from a queryUI", async () => {
+        await config.api.viewV2.update({
+          ...view,
+          queryUI: {
+            groups: [
+              {
+                logicalOperator: UILogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "field",
+                    value: "value",
+                  },
+                ],
+              },
+            ],
+          },
+        })
+
+        let updatedView = await config.api.viewV2.get(view.id)
+        expect(updatedView.query).toBeDefined()
+
+        await config.api.viewV2.update(
+          {
+            ...updatedView,
+            queryUI: undefined,
+          },
+          {
+            status: 400,
+            body: {
+              message: "view is missing queryUI field",
+            },
+          }
+        )
+      })
+
       !isLucene &&
         describe("calculation views", () => {
           let table: Table
@@ -1597,6 +1769,62 @@ describe.each([
           expect.objectContaining({ visible: true, readonly: true })
         )
       })
+
+      it("should fill in the queryUI field if it's missing", async () => {
+        const res = await config.api.viewV2.create({
+          name: generator.name(),
+          tableId: tableId,
+          query: [
+            {
+              operator: BasicOperator.EQUAL,
+              field: "one",
+              value: "1",
+            },
+          ],
+          schema: {
+            id: { visible: true },
+            one: { visible: true },
+          },
+        })
+
+        const table = await config.api.table.get(tableId)
+        const rawView = table.views![res.name] as ViewV2
+        delete rawView.queryUI
+
+        await context.doInAppContext(config.getAppId(), async () => {
+          const db = context.getAppDB()
+
+          if (!rawDatasource) {
+            await db.put(table)
+          } else {
+            const ds = await config.api.datasource.get(datasource!._id!)
+            ds.entities![table.name] = table
+            const updatedDs = {
+              ...rawDatasource,
+              _id: ds._id,
+              _rev: ds._rev,
+              entities: ds.entities,
+            }
+            await db.put(updatedDs)
+          }
+        })
+
+        const view = await getDelegate(res)
+        expect(view.queryUI).toEqual({
+          groups: [
+            {
+              logicalOperator: UILogicalOperator.ALL,
+              filters: [
+                {
+                  operator: BasicOperator.EQUAL,
+                  field: "one",
+                  value: "1",
+                },
+              ],
+            },
+          ],
+        })
+      })
     })
 
     describe("updating table schema", () => {
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 49944bce850..fa87d3cfadf 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -12,9 +12,26 @@ import {
   TableResponse,
   TableSourceType,
   TableViewsResponse,
+  View,
+  ViewV2,
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
+import { ensureQueryUISet } from "../views/utils"
+import { isV2 } from "../views"
+
+function processView(view: ViewV2 | View) {
+  if (!isV2(view)) {
+    return
+  }
+  ensureQueryUISet(view)
+}
+
+function processViews(views: (ViewV2 | View)[]) {
+  for (const view of views) {
+    processView(view)
+  }
+}
 
 export async function processTable(table: Table): Promise<Table> {
   if (!table) {
@@ -22,6 +39,9 @@ export async function processTable(table: Table): Promise<Table> {
   }
 
   table = { ...table }
+  if (table.views) {
+    processViews(Object.values(table.views))
+  }
   if (table._id && isExternalTableID(table._id)) {
     // Old created external tables via Budibase might have a missing field name breaking some UI such as filters
     if (table.schema["id"] && !table.schema["id"].name) {
diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts
index 2484932232f..74f2248f58a 100644
--- a/packages/server/src/sdk/app/views/external.ts
+++ b/packages/server/src/sdk/app/views/external.ts
@@ -81,6 +81,7 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   }
 
   ensureQuerySet(view)
+  ensureQueryUISet(view)
 
   delete views[existingView.name]
   views[view.name] = view
diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts
index fc6b59fefef..efac4a9f8a8 100644
--- a/packages/server/src/sdk/app/views/internal.ts
+++ b/packages/server/src/sdk/app/views/internal.ts
@@ -5,6 +5,7 @@ import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
 import { ensureQuerySet, ensureQueryUISet } from "./utils"
+import { processTable } from "../tables/getters"
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -70,6 +71,7 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   }
 
   ensureQuerySet(view)
+  ensureQueryUISet(view)
 
   delete table.views[existingView.name]
   table.views[view.name] = view
diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
index a110acf0722..ec942c617a5 100644
--- a/packages/server/src/sdk/app/views/utils.ts
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -1,6 +1,7 @@
 import { ViewV2 } from "@budibase/types"
 import { utils, dataFilters } from "@budibase/shared-core"
 import { isPlainObject } from "lodash"
+import { HTTPError } from "@budibase/backend-core"
 
 function isEmptyObject(obj: any) {
   return obj && isPlainObject(obj) && Object.keys(obj).length === 0
@@ -21,7 +22,7 @@ export function ensureQueryUISet(view: ViewV2) {
       // So despite the type saying that `view.query` is a LegacyFilter[] |
       // SearchFilters, it will never be a SearchFilters when a `view.queryUI`
       // is specified, making it "safe" to throw an error here.
-      throw new Error("view is missing queryUI field")
+      throw new HTTPError("view is missing queryUI field", 400)
     }
 
     view.queryUI = utils.processSearchFilters(view.query)
@@ -29,7 +30,10 @@ export function ensureQueryUISet(view: ViewV2) {
 }
 
 export function ensureQuerySet(view: ViewV2) {
-  if (!view.query && view.queryUI && !isEmptyObject(view.queryUI)) {
+  // We consider queryUI to be the source of truth, so we don't check for the
+  // presence of query here. We will overwrite it regardless of whether it is
+  // present or not.
+  if (view.queryUI && !isEmptyObject(view.queryUI)) {
     view.query = dataFilters.buildQuery(view.queryUI)
   }
 }
diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts
index ba07dbe5f04..9741240f27b 100644
--- a/packages/server/src/tests/utilities/api/viewV2.ts
+++ b/packages/server/src/tests/utilities/api/viewV2.ts
@@ -10,9 +10,7 @@ import { Expectations, TestAPI } from "./base"
 
 export class ViewV2API extends TestAPI {
   create = async (
-    // The frontend changed in v3 from sending query to sending only queryUI,
-    // making sure tests reflect that.
-    view: Omit<CreateViewRequest, "query">,
+    view: CreateViewRequest,
     expectations?: Expectations
   ): Promise<ViewV2> => {
     const exp: Expectations = {

From 439565486a56b141c4944032c1f82cd881c40e70 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 13:41:28 +0100
Subject: [PATCH 299/455] Fix lint.

---
 packages/server/src/sdk/app/views/internal.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts
index efac4a9f8a8..77a81a16231 100644
--- a/packages/server/src/sdk/app/views/internal.ts
+++ b/packages/server/src/sdk/app/views/internal.ts
@@ -5,7 +5,6 @@ import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
 import { ensureQuerySet, ensureQueryUISet } from "./utils"
-import { processTable } from "../tables/getters"
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)

From f0fd81b752104b0b5f282e1fb3943ce9ebdf988f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 15:41:27 +0100
Subject: [PATCH 300/455] Update pro reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index fc4c7f49251..297fdc937e9 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit fc4c7f4925139af078480217965c3d6338dc0a7f
+Subproject commit 297fdc937e9c650b4964fc1a942b60022b195865

From 49477330c23f635b2157d1329d247349f0a9adcd Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 21 Oct 2024 16:18:39 +0100
Subject: [PATCH 301/455] Fix broken frontend test.

---
 .../backend/DataTable/modals/ExportModal.test.js          | 2 ++
 packages/shared-core/src/utils.ts                         | 8 ++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
index a5d24d74355..4927f51a8d5 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
+++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
@@ -79,6 +79,8 @@ describe("Export Modal", () => {
       props: propsCfg,
     })
 
+    expect(propsCfg.filters[0].field).toBe("1:Cost")
+
     expect(screen.getByTestId("filters-applied")).toBeVisible()
     expect(screen.getByTestId("filters-applied").textContent).toBe(
       "Filters applied"
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index dce1b8b960d..2bfd166414d 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -147,8 +147,12 @@ export const processSearchFilters = (
       {
         logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL,
         filters: filters.map(filter => {
-          filter.field = removeKeyNumbering(filter.field)
-          return _.pick(filter, FILTER_ALLOWED_KEYS) as SearchFilter
+          const trimmedFilter = _.pick(
+            filter,
+            FILTER_ALLOWED_KEYS
+          ) as SearchFilter
+          trimmedFilter.field = removeKeyNumbering(trimmedFilter.field)
+          return trimmedFilter
         }),
       },
     ],

From 21c580e95665a0926e05acd313ae2fcccf029f10 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 21 Oct 2024 16:29:03 +0100
Subject: [PATCH 302/455] Lint

---
 .../DataTable/buttons/grid/GridViewCalculationButton.svelte     | 2 +-
 .../data/table/[tableId]/_components/CreateViewButton.svelte    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index f9833a981b8..72216f3b3be 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -109,7 +109,7 @@
   // Gets the available fields to group by
   const getGroupByOptions = schema => {
     return Object.entries(schema)
-      .filter(([field, fieldSchema]) => {
+      .filter(([_, fieldSchema]) => {
         // Don't allow grouping by calculations
         if (fieldSchema.calculationType) {
           return false
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
index e531e54d87b..ecc85e76228 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte
@@ -3,7 +3,7 @@
   import { Input, notifications, Button, Icon, ListItem } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { viewsV2 } from "stores/builder"
-  import { CalculationType, ViewV2Type } from "@budibase/types"
+  import { ViewV2Type } from "@budibase/types"
 
   export let table
   export let firstView = false

From 87dbe66cccf5515f4b10de72956e1af8eadbd687 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 10:23:27 +0200
Subject: [PATCH 303/455] Update user column data on import

---
 .../server/src/api/controllers/application.ts | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 59f67540fec..1af797a14ed 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -23,6 +23,7 @@ import {
   cache,
   context,
   db as dbCore,
+  docIds,
   env as envCore,
   ErrorCode,
   events,
@@ -54,6 +55,10 @@ import {
   DuplicateAppResponse,
   UpdateAppRequest,
   UpdateAppResponse,
+  Database,
+  FieldType,
+  BBReferenceFieldSubType,
+  Row,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -279,6 +284,11 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
     const instance = await createInstance(appId, instanceConfig)
     const db = context.getAppDB()
 
+    const isTemplate = instanceConfig.templateString
+    if (!isTemplate) {
+      await updateUserColumns(appId, db, ctx.user._id!)
+    }
+
     const newApplication: App = {
       _id: DocumentType.APP_METADATA,
       _rev: undefined,
@@ -375,6 +385,66 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   })
 }
 
+async function updateUserColumns(
+  appId: string,
+  db: Database,
+  toUserId: string
+) {
+  await context.doInAppContext(appId, async () => {
+    const allTables = await sdk.tables.getAllTables()
+    const tablesWithUserColumns = []
+    for (const table of allTables) {
+      const userColumns = Object.values(table.schema).filter(
+        f =>
+          (f.type === FieldType.BB_REFERENCE ||
+            f.type === FieldType.BB_REFERENCE_SINGLE) &&
+          f.subtype === BBReferenceFieldSubType.USER
+      )
+      if (!userColumns.length) {
+        continue
+      }
+
+      tablesWithUserColumns.push({
+        tableId: table._id!,
+        columns: userColumns.map(c => c.name),
+      })
+    }
+
+    const docsToUpdate = []
+
+    for (const { tableId, columns } of tablesWithUserColumns) {
+      const docs = await db.allDocs<Row>(
+        docIds.getRowParams(tableId, null, { include_docs: true })
+      )
+      const rows = docs.rows.map(d => d.doc!)
+
+      for (const row of rows) {
+        let shouldUpdate = false
+        const updatedColumns = columns.reduce<Row>((newColumns, column) => {
+          if (row[column]) {
+            shouldUpdate = true
+            if (Array.isArray(row[column])) {
+              newColumns[column] = row[column]?.map(() => toUserId)
+            } else if (row[column]) {
+              newColumns[column] = toUserId
+            }
+          }
+          return newColumns
+        }, {})
+
+        if (shouldUpdate) {
+          docsToUpdate.push({
+            ...row,
+            ...updatedColumns,
+          })
+        }
+      }
+    }
+
+    await db.bulkDocs(docsToUpdate)
+  })
+}
+
 async function creationEvents(request: any, app: App) {
   let creationFns: ((app: App) => Promise<void>)[] = []
 

From cb5082a0c0ec47129926681a1fc8eb4eff60231f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 10:25:09 +0200
Subject: [PATCH 304/455] Fix check

---
 packages/server/src/api/controllers/application.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 1af797a14ed..d0021db2cce 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -284,8 +284,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
     const instance = await createInstance(appId, instanceConfig)
     const db = context.getAppDB()
 
-    const isTemplate = instanceConfig.templateString
-    if (!isTemplate) {
+    if (instanceConfig.useTemplate) {
       await updateUserColumns(appId, db, ctx.user._id!)
     }
 

From 2cd3ab662793b19b182b3ef9aa540fbe7b97d742 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 22 Oct 2024 10:35:09 +0100
Subject: [PATCH 305/455] Respond to PR comments.

---
 packages/shared-core/src/utils.ts          | 2 +-
 packages/types/src/api/web/searchFilter.ts | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 2bfd166414d..0af27d4e81f 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -12,7 +12,7 @@ import * as Constants from "./constants"
 import { removeKeyNumbering, splitFiltersArray } from "./filters"
 import _ from "lodash"
 
-const FILTER_ALLOWED_KEYS = [
+const FILTER_ALLOWED_KEYS: (keyof SearchFilter)[] = [
   "field",
   "operator",
   "value",
diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts
index b21484f6edb..7a6920fb607 100644
--- a/packages/types/src/api/web/searchFilter.ts
+++ b/packages/types/src/api/web/searchFilter.ts
@@ -30,6 +30,8 @@ export type SearchFilter = {
   type?: FieldType
   externalType?: string
   noValue?: boolean
+  valueType?: string
+  formulaType?: string
 }
 
 // Prior to v2, this is the type the frontend sent us when filters were

From 8dee7b2c45504bfe7a9121f3ae9801d2d0f271c4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 11:56:19 +0200
Subject: [PATCH 306/455] Remove obsolete templateString

---
 .../server/src/api/controllers/application.ts | 22 +++----------------
 .../src/api/routes/tests/application.spec.ts  |  1 -
 .../server/src/api/routes/utils/validators.ts |  8 ++-----
 packages/server/src/utilities/index.ts        | 12 ----------
 packages/types/src/api/web/application.ts     |  1 -
 5 files changed, 5 insertions(+), 39 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index d0021db2cce..1be301dd9d6 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -36,7 +36,6 @@ import {
 import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
-import { stringToReadStream } from "../../utilities"
 import { doesUserHaveLock } from "../../utilities/redis"
 import { cleanupAutomations } from "../../automations/utils"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
@@ -128,7 +127,6 @@ function checkAppName(
 }
 
 interface AppTemplate {
-  templateString?: string
   useTemplate?: string
   file?: {
     type?: string
@@ -153,14 +151,7 @@ async function createInstance(appId: string, template: AppTemplate) {
   await createRoutingView()
   await createAllSearchIndex()
 
-  // replicate the template data to the instance DB
-  // this is currently very hard to test, downloading and importing template files
-  if (template && template.templateString) {
-    const { ok } = await db.load(stringToReadStream(template.templateString))
-    if (!ok) {
-      throw "Error loading database dump from memory."
-    }
-  } else if (template && template.useTemplate === "true") {
+  if (template && template.useTemplate === "true") {
     await sdk.backups.importApp(appId, db, template)
   } else {
     // create the users table
@@ -248,14 +239,8 @@ export async function fetchAppPackage(
 
 async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
-  const {
-    name,
-    url,
-    encryptionPassword,
-    useTemplate,
-    templateKey,
-    templateString,
-  } = ctx.request.body
+  const { name, url, encryptionPassword, useTemplate, templateKey } =
+    ctx.request.body
 
   checkAppName(ctx, apps, name)
   const appUrl = sdk.applications.getAppUrl({ name, url })
@@ -264,7 +249,6 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const instanceConfig: AppTemplate = {
     useTemplate,
     key: templateKey,
-    templateString,
   }
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = {
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index fe8250bde51..f10f4089bf7 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -141,7 +141,6 @@ describe("/applications", () => {
         name: utils.newid(),
         useTemplate: "true",
         templateKey: "test",
-        templateString: "{}",
       })
       expect(app._id).toBeDefined()
       expect(events.app.created).toHaveBeenCalledTimes(1)
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index d9dbd96b16f..02a1a2d0600 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -355,9 +355,7 @@ export function applicationValidator(opts = { isCreate: true }) {
     _id: OPTIONAL_STRING,
     _rev: OPTIONAL_STRING,
     url: OPTIONAL_STRING,
-    template: Joi.object({
-      templateString: OPTIONAL_STRING,
-    }),
+    template: Joi.object({}),
   }
 
   const appNameValidator = Joi.string()
@@ -390,9 +388,7 @@ export function applicationValidator(opts = { isCreate: true }) {
       _rev: OPTIONAL_STRING,
       name: appNameValidator,
       url: OPTIONAL_STRING,
-      template: Joi.object({
-        templateString: OPTIONAL_STRING,
-      }).unknown(true),
+      template: Joi.object({}).unknown(true),
       snippets: snippetValidator,
     }).unknown(true)
   )
diff --git a/packages/server/src/utilities/index.ts b/packages/server/src/utilities/index.ts
index ce6f2345ca0..129137a72e2 100644
--- a/packages/server/src/utilities/index.ts
+++ b/packages/server/src/utilities/index.ts
@@ -2,9 +2,6 @@ import env from "../environment"
 import { context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"
 import { Document } from "@budibase/types"
-import stream from "stream"
-
-const Readable = stream.Readable
 
 export function wait(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms))
@@ -98,15 +95,6 @@ export function escapeDangerousCharacters(string: string) {
     .replace(/[\t]/g, "\\t")
 }
 
-export function stringToReadStream(string: string) {
-  return new Readable({
-    read() {
-      this.push(string)
-      this.push(null)
-    },
-  })
-}
-
 export function formatBytes(bytes: string) {
   const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
   const byteIncrements = 1024
diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
index bb4d8c7f729..8fa417fb64a 100644
--- a/packages/types/src/api/web/application.ts
+++ b/packages/types/src/api/web/application.ts
@@ -10,7 +10,6 @@ export interface CreateAppRequest {
   templateFile?: string
   includeSampleData?: boolean
   encryptionPassword?: string
-  templateString?: string
   file?: { path: string }
 }
 

From ea1d893ab8a127d5364e3e168935fa6a95a0f79a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 12:06:24 +0200
Subject: [PATCH 307/455] Remove unused fields

---
 packages/types/src/api/web/application.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
index 8fa417fb64a..e4a70c7fd47 100644
--- a/packages/types/src/api/web/application.ts
+++ b/packages/types/src/api/web/application.ts
@@ -8,7 +8,6 @@ export interface CreateAppRequest {
   templateName?: string
   templateKey?: string
   templateFile?: string
-  includeSampleData?: boolean
   encryptionPassword?: string
   file?: { path: string }
 }

From 5fe2a1d33e8935d0dd185b627d163aa0c73cc039 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 12:23:36 +0200
Subject: [PATCH 308/455] Rename and pass just required data

---
 .../src/components/start/CreateAppModal.svelte      | 13 ++++++++-----
 packages/server/src/api/controllers/application.ts  |  4 ++--
 .../server/src/api/routes/tests/application.spec.ts |  4 ++--
 .../server/src/tests/utilities/api/application.ts   |  4 ++--
 packages/types/src/api/web/application.ts           |  2 +-
 5 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte
index b5fed9cdc1d..83ed0890060 100644
--- a/packages/builder/src/components/start/CreateAppModal.svelte
+++ b/packages/builder/src/components/start/CreateAppModal.svelte
@@ -118,14 +118,17 @@
       if ($values.url) {
         data.append("url", $values.url.trim())
       }
-      data.append("useTemplate", template != null)
-      if (template) {
-        data.append("templateName", template.name)
-        data.append("templateKey", template.key)
-        data.append("templateFile", $values.file)
+
+      if (template?.fromFile) {
+        data.append("useTemplate", true)
+        data.append("fileToImport", $values.file)
         if ($values.encryptionPassword?.trim()) {
           data.append("encryptionPassword", $values.encryptionPassword.trim())
         }
+      } else if (template) {
+        data.append("useTemplate", true)
+        data.append("templateName", template.name)
+        data.append("templateKey", template.key)
       }
 
       // Create App
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 1be301dd9d6..fbcf51c1de1 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -250,9 +250,9 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
     useTemplate,
     key: templateKey,
   }
-  if (ctx.request.files && ctx.request.files.templateFile) {
+  if (ctx.request.files && ctx.request.files.fileToImport) {
     instanceConfig.file = {
-      ...(ctx.request.files.templateFile as any),
+      ...(ctx.request.files.fileToImport as any),
       password: encryptionPassword,
     }
   } else if (typeof ctx.request.body.file?.path === "string") {
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index f10f4089bf7..b654e577ba1 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -151,7 +151,7 @@ describe("/applications", () => {
       const app = await config.api.application.create({
         name: utils.newid(),
         useTemplate: "true",
-        templateFile: "src/api/routes/tests/data/export.txt",
+        fileToImport: "src/api/routes/tests/data/export.txt",
       })
       expect(app._id).toBeDefined()
       expect(events.app.created).toHaveBeenCalledTimes(1)
@@ -171,7 +171,7 @@ describe("/applications", () => {
       const app = await config.api.application.create({
         name: utils.newid(),
         useTemplate: "true",
-        templateFile: "src/api/routes/tests/data/old-app.txt",
+        fileToImport: "src/api/routes/tests/data/old-app.txt",
       })
       expect(app._id).toBeDefined()
       expect(app.navigation).toBeDefined()
diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts
index 516af5c973c..9dabc8cfe88 100644
--- a/packages/server/src/tests/utilities/api/application.ts
+++ b/packages/server/src/tests/utilities/api/application.ts
@@ -17,8 +17,8 @@ export class ApplicationAPI extends TestAPI {
     app: CreateAppRequest,
     expectations?: Expectations
   ): Promise<App> => {
-    const files = app.templateFile ? { templateFile: app.templateFile } : {}
-    delete app.templateFile
+    const files = app.fileToImport ? { fileToImport: app.fileToImport } : {}
+    delete app.fileToImport
     return await this._post<App>("/api/applications", {
       fields: app,
       files,
diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
index e4a70c7fd47..57422ceabca 100644
--- a/packages/types/src/api/web/application.ts
+++ b/packages/types/src/api/web/application.ts
@@ -7,7 +7,7 @@ export interface CreateAppRequest {
   useTemplate?: string
   templateName?: string
   templateKey?: string
-  templateFile?: string
+  fileToImport?: string
   encryptionPassword?: string
   file?: { path: string }
 }

From 77c4140e76ccb008e276ab5449cdfd2083d0ce8a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 12:49:34 +0200
Subject: [PATCH 309/455] Type useTemplate as bool

---
 packages/server/scripts/load/utils.js         |  2 +-
 .../server/src/api/controllers/application.ts | 27 +++++++++----------
 .../src/api/routes/tests/application.spec.ts  |  6 ++---
 .../src/api/routes/tests/templates.spec.ts    |  2 +-
 packages/types/src/api/web/application.ts     |  2 +-
 5 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 1dabdcec9ab..57e03c471ce 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -29,7 +29,7 @@ exports.createApp = async apiKey => {
   const body = {
     name,
     url: `/${name}`,
-    useTemplate: "true",
+    useTemplate: true,
     templateKey: "app/school-admin-panel",
     templateName: "School Admin Panel",
   }
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index fbcf51c1de1..2484484ef8f 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -58,6 +58,7 @@ import {
   FieldType,
   BBReferenceFieldSubType,
   Row,
+  BBRequest,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -127,7 +128,7 @@ function checkAppName(
 }
 
 interface AppTemplate {
-  useTemplate?: string
+  useTemplate?: boolean
   file?: {
     type?: string
     path: string
@@ -151,7 +152,7 @@ async function createInstance(appId: string, template: AppTemplate) {
   await createRoutingView()
   await createAllSearchIndex()
 
-  if (template && template.useTemplate === "true") {
+  if (template && template.useTemplate) {
     await sdk.backups.importApp(appId, db, template)
   } else {
     // create the users table
@@ -428,21 +429,21 @@ async function updateUserColumns(
   })
 }
 
-async function creationEvents(request: any, app: App) {
+async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
   let creationFns: ((app: App) => Promise<void>)[] = []
 
-  const body = request.body
-  if (body.useTemplate === "true") {
+  const { useTemplate, templateKey, file } = request.body
+  if (useTemplate) {
     // from template
-    if (body.templateKey && body.templateKey !== "undefined") {
-      creationFns.push(a => events.app.templateImported(a, body.templateKey))
+    if (templateKey && templateKey !== "undefined") {
+      creationFns.push(a => events.app.templateImported(a, templateKey))
     }
     // from file
     else if (request.files?.templateFile) {
       creationFns.push(a => events.app.fileImported(a))
     }
     // from server file path
-    else if (request.body.file) {
+    else if (file) {
       // explicitly pass in the newly created app id
       creationFns.push(a => events.app.duplicated(a, app.appId))
     }
@@ -452,16 +453,14 @@ async function creationEvents(request: any, app: App) {
     }
   }
 
-  if (!request.duplicate) {
-    creationFns.push(a => events.app.created(a))
-  }
+  creationFns.push(a => events.app.created(a))
 
   for (let fn of creationFns) {
     await fn(app)
   }
 }
 
-async function appPostCreate(ctx: UserCtx, app: App) {
+async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
   const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
@@ -472,7 +471,7 @@ async function appPostCreate(ctx: UserCtx, app: App) {
   await creationEvents(ctx.request, app)
 
   // app import, template creation and duplication
-  if (ctx.request.body.useTemplate === "true") {
+  if (ctx.request.body.useTemplate) {
     const { rows } = await getUniqueRows([app.appId])
     const rowCount = rows ? rows.length : 0
     if (rowCount) {
@@ -742,7 +741,7 @@ export async function duplicateApp(
   const createRequestBody: CreateAppRequest = {
     name: appName,
     url: possibleUrl,
-    useTemplate: "true",
+    useTemplate: true,
     // The app export path
     file: {
       path: tmpPath,
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index b654e577ba1..e905a86ed95 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -139,7 +139,7 @@ describe("/applications", () => {
     it("creates app from template", async () => {
       const app = await config.api.application.create({
         name: utils.newid(),
-        useTemplate: "true",
+        useTemplate: true,
         templateKey: "test",
       })
       expect(app._id).toBeDefined()
@@ -150,7 +150,7 @@ describe("/applications", () => {
     it("creates app from file", async () => {
       const app = await config.api.application.create({
         name: utils.newid(),
-        useTemplate: "true",
+        useTemplate: true,
         fileToImport: "src/api/routes/tests/data/export.txt",
       })
       expect(app._id).toBeDefined()
@@ -170,7 +170,7 @@ describe("/applications", () => {
     it("migrates navigation settings from old apps", async () => {
       const app = await config.api.application.create({
         name: utils.newid(),
-        useTemplate: "true",
+        useTemplate: true,
         fileToImport: "src/api/routes/tests/data/old-app.txt",
       })
       expect(app._id).toBeDefined()
diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts
index d5483c54b4b..4d9ccb2c170 100644
--- a/packages/server/src/api/routes/tests/templates.spec.ts
+++ b/packages/server/src/api/routes/tests/templates.spec.ts
@@ -95,7 +95,7 @@ describe("/templates", () => {
             const app = await config.api.application.create({
               name,
               url,
-              useTemplate: "true",
+              useTemplate: true,
               templateName: "Agency Client Portal",
               templateKey: "app/agency-client-portal",
             })
diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
index 57422ceabca..2606860fde9 100644
--- a/packages/types/src/api/web/application.ts
+++ b/packages/types/src/api/web/application.ts
@@ -4,7 +4,7 @@ import type { Layout, App, Screen } from "../../documents"
 export interface CreateAppRequest {
   name: string
   url?: string
-  useTemplate?: string
+  useTemplate?: boolean
   templateName?: string
   templateKey?: string
   fileToImport?: string

From 838d11e2534ee9f92be8d43671f3cca9bf773f44 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 13:01:33 +0200
Subject: [PATCH 310/455] Update only for templates

---
 packages/server/src/api/controllers/application.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 2484484ef8f..1d8724ed751 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -269,7 +269,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
     const instance = await createInstance(appId, instanceConfig)
     const db = context.getAppDB()
 
-    if (instanceConfig.useTemplate) {
+    if (instanceConfig.useTemplate && !instanceConfig.file) {
       await updateUserColumns(appId, db, ctx.user._id!)
     }
 

From 020927ae6c193d5cce8d906bb5f712ef9d4d2a34 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 13:19:04 +0200
Subject: [PATCH 311/455] Fix types

---
 .../server/src/api/controllers/application.ts     | 15 +++++++++++----
 packages/types/src/api/web/application.ts         |  2 +-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 1d8724ed751..50878861a15 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -240,8 +240,15 @@ export async function fetchAppPackage(
 
 async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
-  const { name, url, encryptionPassword, useTemplate, templateKey } =
-    ctx.request.body
+  const { body } = ctx.request
+  const { name, url, encryptionPassword, templateKey } = body
+
+  let useTemplate
+  if (typeof body.useTemplate === "string") {
+    useTemplate = body.useTemplate === "true"
+  } else if (typeof body.useTemplate === "boolean") {
+    useTemplate = body.useTemplate
+  }
 
   checkAppName(ctx, apps, name)
   const appUrl = sdk.applications.getAppUrl({ name, url })
@@ -256,9 +263,9 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
       ...(ctx.request.files.fileToImport as any),
       password: encryptionPassword,
     }
-  } else if (typeof ctx.request.body.file?.path === "string") {
+  } else if (typeof body.file?.path === "string") {
     instanceConfig.file = {
-      path: ctx.request.body.file?.path,
+      path: body.file?.path,
     }
   }
 
diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
index 2606860fde9..57422ceabca 100644
--- a/packages/types/src/api/web/application.ts
+++ b/packages/types/src/api/web/application.ts
@@ -4,7 +4,7 @@ import type { Layout, App, Screen } from "../../documents"
 export interface CreateAppRequest {
   name: string
   url?: string
-  useTemplate?: boolean
+  useTemplate?: string
   templateName?: string
   templateKey?: string
   fileToImport?: string

From e1640cccf93a061155b72e9b4e5decc4fac6c827 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 14:04:57 +0200
Subject: [PATCH 312/455] Change api type back

---
 packages/server/src/api/controllers/application.ts       | 2 +-
 packages/server/src/api/routes/tests/application.spec.ts | 6 +++---
 packages/server/src/api/routes/tests/templates.spec.ts   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 50878861a15..8102254a79a 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -748,7 +748,7 @@ export async function duplicateApp(
   const createRequestBody: CreateAppRequest = {
     name: appName,
     url: possibleUrl,
-    useTemplate: true,
+    useTemplate: "true",
     // The app export path
     file: {
       path: tmpPath,
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index e905a86ed95..b654e577ba1 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -139,7 +139,7 @@ describe("/applications", () => {
     it("creates app from template", async () => {
       const app = await config.api.application.create({
         name: utils.newid(),
-        useTemplate: true,
+        useTemplate: "true",
         templateKey: "test",
       })
       expect(app._id).toBeDefined()
@@ -150,7 +150,7 @@ describe("/applications", () => {
     it("creates app from file", async () => {
       const app = await config.api.application.create({
         name: utils.newid(),
-        useTemplate: true,
+        useTemplate: "true",
         fileToImport: "src/api/routes/tests/data/export.txt",
       })
       expect(app._id).toBeDefined()
@@ -170,7 +170,7 @@ describe("/applications", () => {
     it("migrates navigation settings from old apps", async () => {
       const app = await config.api.application.create({
         name: utils.newid(),
-        useTemplate: true,
+        useTemplate: "true",
         fileToImport: "src/api/routes/tests/data/old-app.txt",
       })
       expect(app._id).toBeDefined()
diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts
index 4d9ccb2c170..d5483c54b4b 100644
--- a/packages/server/src/api/routes/tests/templates.spec.ts
+++ b/packages/server/src/api/routes/tests/templates.spec.ts
@@ -95,7 +95,7 @@ describe("/templates", () => {
             const app = await config.api.application.create({
               name,
               url,
-              useTemplate: true,
+              useTemplate: "true",
               templateName: "Agency Client Portal",
               templateKey: "app/agency-client-portal",
             })

From 00fe28e5576684930485221dde872b9fa1ff4a8d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 13:05:22 +0100
Subject: [PATCH 313/455] Simplifying retrieval of role hierarchy for
 accessible endpoint.

---
 packages/server/src/api/controllers/role.ts | 27 +++++----------------
 1 file changed, 6 insertions(+), 21 deletions(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index d09fcf28f60..2d8556b5813 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -226,35 +226,20 @@ export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
   if (!roleId) {
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
   }
+  // If a custom role is provided in the header, filter out higher level roles
+  const roleHeader = ctx.header?.[Header.PREVIEW_ROLE] as string
+  const isBuilder = ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)
   let roleIds: string[] = []
-  if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
+  if (!roleHeader && isBuilder) {
     const appId = context.getAppId()
     if (appId) {
       roleIds = await roles.getAllRoleIds(appId)
     }
+  } else if (isBuilder && roleHeader) {
+    roleIds = await roles.getUserRoleIdHierarchy(roleHeader)
   } else {
     roleIds = await roles.getUserRoleIdHierarchy(roleId!)
   }
 
-  // If a custom role is provided in the header, filter out higher level roles
-  const roleHeader = ctx.header?.[Header.PREVIEW_ROLE] as string
-  if (roleHeader && !Object.keys(roles.BUILTIN_ROLE_IDS).includes(roleHeader)) {
-    const role = await roles.getRole(roleHeader)
-    const inherits = role?.inherits
-    const orderedRoles = roleIds.reverse()
-    let filteredRoles = [roleHeader]
-    for (let role of orderedRoles) {
-      filteredRoles = [role, ...filteredRoles]
-      if (
-        (Array.isArray(inherits) && inherits.includes(role)) ||
-        role === inherits
-      ) {
-        break
-      }
-    }
-    filteredRoles.pop()
-    roleIds = [roleHeader, ...filteredRoles]
-  }
-
   ctx.body = roleIds.map(roleId => roles.getExternalRoleID(roleId))
 }

From 86620b9a26a3b41ce529da76e6e8acc5f423f73c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 22 Oct 2024 13:47:38 +0100
Subject: [PATCH 314/455] Remove permissionId when creating roles and ensure
 all roles inherit basic

---
 .../src/components/backend/RoleEditor/RoleFlow.svelte        | 2 +-
 packages/builder/src/components/backend/RoleEditor/utils.js  | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
index 152946fa96e..90c0a9fca8d 100644
--- a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
+++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte
@@ -95,7 +95,7 @@
         color: "var(--spectrum-global-color-gray-700)",
         description: "Custom role",
       },
-      permissionId: "write",
+      inherits: [Roles.BASIC],
     })
     await tick()
     layoutAndFit()
diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js
index 0142b7b36ff..a958fc6401b 100644
--- a/packages/builder/src/components/backend/RoleEditor/utils.js
+++ b/packages/builder/src/components/backend/RoleEditor/utils.js
@@ -201,7 +201,10 @@ export const roleToNode = role => {
 // Converts a node structure back into a role doc
 export const nodeToRole = ({ node, edges }) => ({
   ...get(roles).find(role => role._id === node.id),
-  inherits: edges.filter(x => x.target === node.id).map(x => x.source),
+  inherits: edges
+    .filter(x => x.target === node.id)
+    .map(x => x.source)
+    .concat(Roles.BASIC),
   uiMetadata: {
     displayName: node.data.displayName,
     color: node.data.color,

From f3d54f1b7d6e6d92956e2533efaa05cb7889570a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 13:54:54 +0100
Subject: [PATCH 315/455] Adding test cases.

---
 packages/backend-core/src/security/roles.ts   | 16 +++++++++++++
 packages/server/src/api/controllers/role.ts   | 13 +++++++++++
 .../server/src/api/routes/tests/role.spec.ts  | 23 +++++++++++++++++++
 3 files changed, 52 insertions(+)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index c14178cacb1..a301abea8be 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -213,6 +213,22 @@ export function getBuiltinRole(roleId: string): Role | undefined {
   return cloneDeep(role)
 }
 
+export function validInherits(
+  allRoles: RoleDoc[],
+  inherits?: string | string[]
+): boolean {
+  if (!inherits) {
+    return false
+  }
+  const find = (id: string) => allRoles.find(r => compareRoleIds(r._id!, id))
+  if (Array.isArray(inherits)) {
+    const filtered = inherits.filter(roleId => find(roleId))
+    return inherits.length !== 0 && filtered.length === inherits.length
+  } else {
+    return !!find(inherits)
+  }
+}
+
 /**
  * Works through the inheritance ranks to see how far up the builtin stack this ID is.
  */
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 2d8556b5813..fca0d287eba 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -18,10 +18,12 @@ import {
   UserCtx,
   UserMetadata,
   DocumentType,
+  PermissionLevel,
 } from "@budibase/types"
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
+import { validInherits } from "@budibase/backend-core/src/security/roles"
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -125,6 +127,17 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     ctx.throw(400, "Cannot change custom role name")
   }
 
+  // custom roles should always inherit basic - if they don't inherit anything else
+  if (!inherits && roles.validInherits(allRoles, dbRole?.inherits)) {
+    inherits = dbRole?.inherits
+  } else if (!roles.validInherits(allRoles, inherits)) {
+    inherits = [roles.BUILTIN_ROLE_IDS.BASIC]
+  }
+  // assume write permission level for newly created roles
+  if (isCreate && !permissionId) {
+    permissionId = PermissionLevel.WRITE
+  }
+
   const role = new roles.Role(_id, name, permissionId, {
     displayName: uiMetadata?.displayName || name,
     description: uiMetadata?.description || "Custom role",
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 56682953427..2c9ce329373 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -38,6 +38,16 @@ describe("/roles", () => {
         _id: dbCore.prefixRoleID(res._id!),
       })
     })
+
+    it("handle a role with no inherits", async () => {
+      const role = basicRole()
+      role.inherits = []
+
+      const res = await config.api.roles.save(role, {
+        status: 200,
+      })
+      expect(res.inherits).toEqual([BUILTIN_ROLE_IDS.BASIC])
+    })
   })
 
   describe("update", () => {
@@ -149,6 +159,19 @@ describe("/roles", () => {
         { status: 400, body: { message: LOOP_ERROR } }
       )
     })
+
+    it("handle updating a role, without its inherits", async () => {
+      const res = await config.api.roles.save({
+        ...basicRole(),
+        inherits: [BUILTIN_ROLE_IDS.ADMIN],
+      })
+      // save again
+      const updatedRes = await config.api.roles.save({
+        ...res,
+        inherits: undefined,
+      })
+      expect(updatedRes.inherits).toEqual([BUILTIN_ROLE_IDS.ADMIN])
+    })
   })
 
   describe("fetch", () => {

From cf3ef6473471c44a07ec0b6d2a709c3f3b0648e6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 15:03:13 +0100
Subject: [PATCH 316/455] Clean up role header implementation in currentapp.ts

---
 packages/server/src/middleware/currentapp.ts | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts
index d6163772980..a8ef8bb2512 100644
--- a/packages/server/src/middleware/currentapp.ts
+++ b/packages/server/src/middleware/currentapp.ts
@@ -56,22 +56,9 @@ export default async (ctx: UserCtx, next: any) => {
       ctx.request &&
       (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string)
     if (isBuilder && isDevApp && roleHeader) {
-      // Ensure the role is valid by ensuring a definition exists
-      try {
-        if (roleHeader) {
-          const role = await roles.getRole(roleHeader)
-          if (role) {
-            roleId = roleHeader
-
-            // Delete admin and builder flags so that the specified role is honoured
-            ctx.user = users.removePortalUserPermissions(
-              ctx.user
-            ) as ContextUser
-          }
-        }
-      } catch (error) {
-        // Swallow error and do nothing
-      }
+      roleId = roleHeader
+      // Delete admin and builder flags so that the specified role is honoured
+      ctx.user = users.removePortalUserPermissions(ctx.user) as ContextUser
     }
   }
 

From ebc440a0b67dd5c8f6c38b10d3de6ec81f8af1f1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 15:03:47 +0100
Subject: [PATCH 317/455] Another test case.

---
 .../server/src/api/routes/tests/role.spec.ts  | 17 ++++++++++++++
 .../server/src/tests/utilities/structures.ts  | 23 +++++++++----------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 2c9ce329373..e1e5231beff 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -321,6 +321,23 @@ describe("/roles", () => {
         }
       )
     })
+
+    it("should fetch preview role correctly even without basic specified", async () => {
+      const role = await config.api.roles.save(basicRole())
+      // have to forcefully delete the inherits from DB - technically can't
+      // happen anymore - but good test case
+      await dbCore.getDB(config.appId!).put({
+        ...role,
+        _id: dbCore.prefixRoleID(role._id!),
+        inherits: [],
+      })
+      await config.withHeaders({ "x-budibase-role": role.name }, async () => {
+        const res = await config.api.roles.accessible({
+          status: 200,
+        })
+        expect(res).toEqual([role.name])
+      })
+    })
   })
 
   describe("accessible - multi-inheritance", () => {
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index b38cc6484fa..b63d6d1a78b 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -8,31 +8,31 @@ import {
 } from "../../automations"
 import {
   AIOperationEnum,
+  AutoFieldSubType,
   Automation,
   AutomationActionStepId,
+  AutomationEventType,
   AutomationResults,
   AutomationStatus,
   AutomationStep,
   AutomationStepType,
   AutomationTrigger,
   AutomationTriggerStepId,
+  BBReferenceFieldSubType,
+  CreateViewRequest,
   Datasource,
+  FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  JsonFieldSubType,
+  LoopStepType,
+  Query,
+  Role,
   SourceName,
   Table,
-  INTERNAL_TABLE_SOURCE_ID,
   TableSourceType,
-  Query,
   Webhook,
   WebhookActionType,
-  AutomationEventType,
-  LoopStepType,
-  FieldSchema,
-  BBReferenceFieldSubType,
-  JsonFieldSubType,
-  AutoFieldSubType,
-  Role,
-  CreateViewRequest,
 } from "@budibase/types"
 import { LoopInput } from "../../definitions/automations"
 import { merge } from "lodash"
@@ -439,7 +439,7 @@ export function updateRowAutomationWithFilters(
   appId: string,
   tableId: string
 ): Automation {
-  const automation: Automation = {
+  return {
     name: "updateRowWithFilters",
     type: "automation",
     appId,
@@ -472,7 +472,6 @@ export function updateRowAutomationWithFilters(
       },
     },
   }
-  return automation
 }
 
 export function basicAutomationResults(

From fe35ef74e72064930c5e348cf2dd290bfd8ca4dd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 15:06:50 +0100
Subject: [PATCH 318/455] final test case.

---
 .../src/api/routes/tests/screen.spec.ts       | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/screen.spec.ts b/packages/server/src/api/routes/tests/screen.spec.ts
index 5dfe3d2a44b..894710ca272 100644
--- a/packages/server/src/api/routes/tests/screen.spec.ts
+++ b/packages/server/src/api/routes/tests/screen.spec.ts
@@ -86,7 +86,6 @@ describe("/screens", () => {
             status: 200,
           }
         )
-        // basic and role1 screen
         expect(res.screens.length).toEqual(screenIds.length)
         expect(res.screens.map(s => s._id).sort()).toEqual(screenIds.sort())
       })
@@ -107,6 +106,25 @@ describe("/screens", () => {
         screen2._id!,
       ])
     })
+
+    it("should be able to fetch basic and screen 1 with role1 in role header", async () => {
+      await config.withHeaders(
+        {
+          "x-budibase-role": role1._id!,
+        },
+        async () => {
+          const res = await config.api.application.getDefinition(
+            config.prodAppId!,
+            {
+              status: 200,
+            }
+          )
+          const screenIds = [screen._id!, screen1._id!]
+          expect(res.screens.length).toEqual(screenIds.length)
+          expect(res.screens.map(s => s._id).sort()).toEqual(screenIds.sort())
+        }
+      )
+    })
   })
 
   describe("save", () => {

From eb349b5fb309662da4fe560a6f57666ecb1b79f9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 15:15:18 +0100
Subject: [PATCH 319/455] Linting.

---
 packages/server/src/api/controllers/role.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index fca0d287eba..728038e3a3a 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -23,7 +23,6 @@ import {
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { validInherits } from "@budibase/backend-core/src/security/roles"
 
 const UpdateRolesOptions = {
   CREATED: "created",

From 3ae5ce980207824dfda5c23ae71244cdc2f5ee88 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 22 Oct 2024 15:16:07 +0100
Subject: [PATCH 320/455] Use role display names for picking preview roles

---
 .../components/devtools/DevToolsHeader.svelte | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/packages/client/src/components/devtools/DevToolsHeader.svelte b/packages/client/src/components/devtools/DevToolsHeader.svelte
index 55b705e7175..f283defc7c6 100644
--- a/packages/client/src/components/devtools/DevToolsHeader.svelte
+++ b/packages/client/src/components/devtools/DevToolsHeader.svelte
@@ -2,27 +2,28 @@
   import { Heading, Select, ActionButton } from "@budibase/bbui"
   import { devToolsStore, appStore, roleStore } from "../../stores"
   import { getContext, onMount } from "svelte"
+  import { API } from "api"
 
   const context = getContext("context")
   const SELF_ROLE = "self"
 
-  let staticRoleList
+  let roles
 
-  $: previewOptions = buildRoleList(staticRoleList)
+  $: previewOptions = buildRoleList(roles)
 
-  function buildRoleList(roleIds) {
+  function buildRoleList(roles) {
     const list = []
     list.push({
       label: "View as yourself",
       value: SELF_ROLE,
     })
-    if (!roleIds) {
+    if (!roles) {
       return list
     }
-    for (let roleId of roleIds) {
+    for (let role of roles) {
       list.push({
-        label: `View as ${roleId.toLowerCase()} user`,
-        value: roleId,
+        label: `View as ${role.uiMetadata?.displayName || role.name}`,
+        value: role._id,
       })
     }
     return list
@@ -31,12 +32,12 @@
   onMount(async () => {
     // make sure correct before starting
     await devToolsStore.actions.changeRole(SELF_ROLE)
-    staticRoleList = await roleStore.actions.fetchAccessibleRoles()
+    roles = await API.getRoles()
   })
 </script>
 
 <div class="dev-preview-header" class:mobile={$context.device.mobile}>
-  <Heading size="XS">Preview</Heading>
+  <Heading size="XS">Previ werewr eww</Heading>
   <Select
     quiet
     options={previewOptions}

From b85a28fdf612bd3fcdd179ccf514e0e869a24d0d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 16:19:25 +0200
Subject: [PATCH 321/455] Fixes

---
 packages/server/src/api/controllers/application.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 8102254a79a..d8d9d75336b 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -440,13 +440,13 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
   let creationFns: ((app: App) => Promise<void>)[] = []
 
   const { useTemplate, templateKey, file } = request.body
-  if (useTemplate) {
+  if (useTemplate === "true") {
     // from template
     if (templateKey && templateKey !== "undefined") {
       creationFns.push(a => events.app.templateImported(a, templateKey))
     }
     // from file
-    else if (request.files?.templateFile) {
+    else if (request.files?.fileToImport) {
       creationFns.push(a => events.app.fileImported(a))
     }
     // from server file path

From 1e4f105901d7897651639f25e10c922fe2a1ef8d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 22 Oct 2024 15:25:15 +0100
Subject: [PATCH 322/455] Fix labels and add colours to access level selctor
 for queries

---
 .../design/settings/controls/RoleSelect.svelte     |  2 ++
 .../integration/AccessLevelSelect.svelte           | 14 ++++----------
 .../components/integration/RestQueryViewer.svelte  |  2 +-
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
index ad466cbce76..b99e58f2058 100644
--- a/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/RoleSelect.svelte
@@ -5,6 +5,7 @@
   export let value
   export let error
   export let placeholder = null
+  export let autoWidth = false
 </script>
 
 <Select
@@ -16,4 +17,5 @@
   getOptionColour={role => role.uiMetadata.color}
   {placeholder}
   {error}
+  {autoWidth}
 />
diff --git a/packages/builder/src/components/integration/AccessLevelSelect.svelte b/packages/builder/src/components/integration/AccessLevelSelect.svelte
index 05b336c3b3c..02ea50e7800 100644
--- a/packages/builder/src/components/integration/AccessLevelSelect.svelte
+++ b/packages/builder/src/components/integration/AccessLevelSelect.svelte
@@ -1,7 +1,8 @@
 <script>
-  import { Label, notifications, Select } from "@budibase/bbui"
-  import { permissions, roles } from "stores/builder"
+  import { Label, notifications } from "@budibase/bbui"
+  import { permissions } from "stores/builder"
   import { Constants } from "@budibase/frontend-core"
+  import RoleSelect from "components/design/settings/controls/RoleSelect.svelte"
 
   export let query
   export let label
@@ -52,12 +53,5 @@
   {#if label}
     <Label>{label}</Label>
   {/if}
-  <Select
-    value={roleId}
-    on:change={e => updateRole(e.detail)}
-    options={$roles}
-    getOptionLabel={x => x.name}
-    getOptionValue={x => x._id}
-    autoWidth
-  />
+  <RoleSelect value={roleId} on:change={e => updateRole(e.detail)} autoWidth />
 {/if}
diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte
index 04891678238..3b1356efe22 100644
--- a/packages/builder/src/components/integration/RestQueryViewer.svelte
+++ b/packages/builder/src/components/integration/RestQueryViewer.svelte
@@ -503,7 +503,7 @@
             on:save={saveQuery}
           />
           <div class="access">
-            <Label>Access level</Label>
+            <Label>Access</Label>
             <AccessLevelSelect {query} {saveId} />
           </div>
         </div>

From 62d87da024a0f251ab5910bdaad2da5fd39eccb7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 22 Oct 2024 15:28:40 +0100
Subject: [PATCH 323/455] Fix role colours in user app list portal page

---
 .../portal/users/users/_components/AppRoleTableRenderer.svelte | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
index 7ed36eb3a4e..03a3e626a4d 100644
--- a/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte
@@ -20,7 +20,8 @@
 {:else}
   <StatusLight
     square
-    color={role?.color || "var(--spectrum-global-color-static-magenta-400)"}
+    color={role?.uiMetadata.color ||
+      "var(--spectrum-global-color-static-magenta-400)"}
   >
     Can use as {getRoleLabel(value)}
   </StatusLight>

From 80e532f59d3a70799503fa237b3a593e52b8c412 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 22 Oct 2024 15:33:21 +0100
Subject: [PATCH 324/455] Fix app count in groups list

---
 .../users/groups/_components/GroupAppsTableRenderer.svelte | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupAppsTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupAppsTableRenderer.svelte
index 48b1bfbce66..37e44a277e1 100644
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupAppsTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupAppsTableRenderer.svelte
@@ -4,15 +4,16 @@
 
   export let value
   export let row
-  $: count = getCount(Object.keys(value || {}).length)
 
-  const getCount = () => {
+  $: count = getCount(row, value)
+
+  const getCount = (row, value) => {
     return sdk.users.hasAppBuilderPermissions(row)
       ? row.builder.apps.length +
           Object.keys(row.roles || {}).filter(appId =>
             row.builder.apps.includes(appId)
           ).length
-      : value?.length || 0
+      : Object.keys(value || {}).length
   }
 </script>
 

From f07cb6bd74b31dcf5fd8a753836ec7527162c05a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 22 Oct 2024 16:37:58 +0200
Subject: [PATCH 325/455] Fix test

---
 .../server/src/api/routes/tests/application.spec.ts    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index b654e577ba1..47b6776610a 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -21,6 +21,7 @@ import tk from "timekeeper"
 import * as uuid from "uuid"
 import { structures } from "@budibase/backend-core/tests"
 import nock from "nock"
+import path from "path"
 
 describe("/applications", () => {
   let config = setup.getConfig()
@@ -137,10 +138,17 @@ describe("/applications", () => {
     })
 
     it("creates app from template", async () => {
+      nock("https://prod-budi-templates.s3-eu-west-1.amazonaws.com")
+        .get(`/templates/app/agency-client-portal.tar.gz`)
+        .replyWithFile(
+          200,
+          path.resolve(__dirname, "data", "agency-client-portal.tar.gz")
+        )
+
       const app = await config.api.application.create({
         name: utils.newid(),
         useTemplate: "true",
-        templateKey: "test",
+        templateKey: "app/agency-client-portal",
       })
       expect(app._id).toBeDefined()
       expect(events.app.created).toHaveBeenCalledTimes(1)

From 4129169463e0940c4e45a0bf57c967c34f97cbb1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 22 Oct 2024 15:40:53 +0100
Subject: [PATCH 326/455] Lint

---
 packages/client/src/components/devtools/DevToolsHeader.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/devtools/DevToolsHeader.svelte b/packages/client/src/components/devtools/DevToolsHeader.svelte
index f283defc7c6..89113318b75 100644
--- a/packages/client/src/components/devtools/DevToolsHeader.svelte
+++ b/packages/client/src/components/devtools/DevToolsHeader.svelte
@@ -1,6 +1,6 @@
 <script>
   import { Heading, Select, ActionButton } from "@budibase/bbui"
-  import { devToolsStore, appStore, roleStore } from "../../stores"
+  import { devToolsStore, appStore } from "../../stores"
   import { getContext, onMount } from "svelte"
   import { API } from "api"
 

From 74870663e26b4316ec1d84036826e741cd3bbfbc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 16:02:20 +0100
Subject: [PATCH 327/455] Addressing PR comments.

---
 packages/backend-core/src/security/roles.ts |  8 ++++----
 packages/server/src/api/controllers/role.ts | 10 +++++++---
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index a301abea8be..7d9a11db483 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -220,7 +220,7 @@ export function validInherits(
   if (!inherits) {
     return false
   }
-  const find = (id: string) => allRoles.find(r => compareRoleIds(r._id!, id))
+  const find = (id: string) => allRoles.find(r => roleIDsAreEqual(r._id!, id))
   if (Array.isArray(inherits)) {
     const filtered = inherits.filter(roleId => find(roleId))
     return inherits.length !== 0 && filtered.length === inherits.length
@@ -306,7 +306,7 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
     : roleId1
 }
 
-export function compareRoleIds(roleId1: string, roleId2: string) {
+export function roleIDsAreEqual(roleId1: string, roleId2: string) {
   // make sure both role IDs are prefixed correctly
   return prefixRoleID(roleId1) === prefixRoleID(roleId2)
 }
@@ -339,7 +339,7 @@ export function findRole(
     roleId = prefixRoleID(roleId)
   }
   const dbRole = roles.find(
-    role => role._id && compareRoleIds(role._id, roleId)
+    role => role._id && roleIDsAreEqual(role._id, roleId)
   )
   if (!dbRole && !isBuiltin(roleId) && opts?.defaultPublic) {
     return cloneDeep(BUILTIN_ROLES.PUBLIC)
@@ -573,7 +573,7 @@ export class AccessController {
     }
 
     return (
-      roleIds?.find(roleId => compareRoleIds(roleId, tryingRoleId)) !==
+      roleIds?.find(roleId => roleIDsAreEqual(roleId, tryingRoleId)) !==
       undefined
     )
   }
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 728038e3a3a..eb452b9997d 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -23,6 +23,7 @@ import {
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
+import { roleIDsAreEqual } from "@budibase/backend-core/src/security/roles"
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -36,11 +37,11 @@ async function removeRoleFromOthers(roleId: string) {
     let changed = false
     if (Array.isArray(role.inherits)) {
       const newInherits = role.inherits.filter(
-        id => !roles.compareRoleIds(id, roleId)
+        id => !roles.roleIDsAreEqual(id, roleId)
       )
       changed = role.inherits.length !== newInherits.length
       role.inherits = newInherits
-    } else if (role.inherits && roles.compareRoleIds(role.inherits, roleId)) {
+    } else if (role.inherits && roles.roleIDsAreEqual(role.inherits, roleId)) {
       role.inherits = roles.BUILTIN_ROLE_IDS.PUBLIC
       changed = true
     }
@@ -239,7 +240,10 @@ export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
   }
   // If a custom role is provided in the header, filter out higher level roles
-  const roleHeader = ctx.header?.[Header.PREVIEW_ROLE] as string
+  const roleHeader = ctx.header[Header.PREVIEW_ROLE]
+  if (Array.isArray(roleHeader)) {
+    ctx.throw(400, `Too many roles specified in ${Header.PREVIEW_ROLE} header`)
+  }
   const isBuilder = ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)
   let roleIds: string[] = []
   if (!roleHeader && isBuilder) {

From 99d867fe603cf37cbe3f901d6ce7b8f27f78fa8a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 16:04:18 +0100
Subject: [PATCH 328/455] Updating test case.

---
 packages/server/src/api/routes/tests/role.spec.ts | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index e1e5231beff..07b88733af7 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -165,11 +165,9 @@ describe("/roles", () => {
         ...basicRole(),
         inherits: [BUILTIN_ROLE_IDS.ADMIN],
       })
-      // save again
-      const updatedRes = await config.api.roles.save({
-        ...res,
-        inherits: undefined,
-      })
+      // remove the roles so that it will default back to DB roles, then save again
+      delete res.inherits
+      const updatedRes = await config.api.roles.save(res)
       expect(updatedRes.inherits).toEqual([BUILTIN_ROLE_IDS.ADMIN])
     })
   })

From f91e0394028d67875e6907fe5703ff4c35a6c5b6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 16:06:26 +0100
Subject: [PATCH 329/455] Adding test with invalid role IDs.#

---
 packages/server/src/api/routes/tests/role.spec.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 07b88733af7..adb83ca7932 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -39,6 +39,16 @@ describe("/roles", () => {
       })
     })
 
+    it("handle a role with invalid inherits", async () => {
+      const role = basicRole()
+      role.inherits = ["not_real", "some_other_not_real"]
+
+      const res = await config.api.roles.save(role, {
+        status: 200,
+      })
+      expect(res.inherits).toEqual([BUILTIN_ROLE_IDS.BASIC])
+    })
+
     it("handle a role with no inherits", async () => {
       const role = basicRole()
       role.inherits = []

From 0b2ac578c05e9dada12297cb2378b7ec9e9484c4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 22 Oct 2024 16:27:53 +0100
Subject: [PATCH 330/455] linting.

---
 packages/server/src/api/controllers/role.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index eb452b9997d..10477119837 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -23,7 +23,6 @@ import {
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { roleIDsAreEqual } from "@budibase/backend-core/src/security/roles"
 
 const UpdateRolesOptions = {
   CREATED: "created",

From 807ecf603966e81ecc9311478510f59324341904 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 22 Oct 2024 18:24:10 +0100
Subject: [PATCH 331/455] BB AI QA testing fixes

---
 .../builder/portal/settings/ai/index.svelte   |  5 +-
 packages/pro                                  |  2 +-
 packages/server/src/automations/actions.ts    | 15 +++--
 .../src/api/controllers/global/configs.ts     | 57 ++++++++++++-------
 4 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 42de9f19ae5..9a70fd289a8 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -32,9 +32,7 @@
   async function fetchAIConfig() {
     try {
       const aiDoc = await API.getConfig(ConfigTypes.AI)
-      if (aiDoc._id) {
-        fullAIConfig = aiDoc
-      }
+      fullAIConfig = aiDoc
     } catch (error) {
       notifications.error("Error fetching AI config")
     }
@@ -66,6 +64,7 @@
       }
       // Add new or update existing custom AI Config
       fullAIConfig.config[id] = editingAIConfig
+      fullAIConfig.type = ConfigTypes.AI
     }
 
     try {
diff --git a/packages/pro b/packages/pro
index 297fdc937e9..7bd0350ed3f 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 297fdc937e9c650b4964fc1a942b60022b195865
+Subproject commit 7bd0350ed3f55da30d07a7c67fd746727ce93a9a
diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts
index e921e569c9c..a6c9be41c5c 100644
--- a/packages/server/src/automations/actions.ts
+++ b/packages/server/src/automations/actions.ts
@@ -18,6 +18,7 @@ import * as loop from "./steps/loop"
 import * as collect from "./steps/collect"
 import * as branch from "./steps/branch"
 import * as triggerAutomationRun from "./steps/triggerAutomationRun"
+import * as openai from "./steps/openai"
 import env from "../environment"
 import {
   PluginType,
@@ -89,21 +90,27 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
 // ran at all
 if (env.SELF_HOSTED) {
   const bash = require("./steps/bash")
-  const openai = require("./steps/openai")
 
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
   BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
-  // @ts-ignore
-  ACTION_IMPLS.OPENAI = openai.run
-  BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
 }
 
 export async function getActionDefinitions() {
   if (await features.flags.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
     BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition
   }
+  if (
+    env.SELF_HOSTED ||
+    (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
+    (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
+  ) {
+    BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
+    // @ts-ignore
+    ACTION_IMPLS["OPENAI"] = openai.run
+  }
+
   const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
   if (env.SELF_HOSTED) {
     const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index e6e80ff3a5b..6c09da77de8 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -35,6 +35,7 @@ import {
   AIInnerConfig,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
+import { generateConfigID } from "@budibase/backend-core/src/configs"
 
 const getEventFns = async (config: Config, existing?: Config) => {
   const fns = []
@@ -44,9 +45,7 @@ const getEventFns = async (config: Config, existing?: Config) => {
       fns.push(events.email.SMTPCreated)
     } else if (isAIConfig(config)) {
       fns.push(() => events.ai.AIConfigCreated)
-      fns.push(() =>
-        pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length)
-      )
+      fns.push(() => pro.quotas.addCustomAIConfig)
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE))
       if (config.config.activated) {
@@ -85,9 +84,6 @@ const getEventFns = async (config: Config, existing?: Config) => {
       fns.push(events.email.SMTPUpdated)
     } else if (isAIConfig(config)) {
       fns.push(() => events.ai.AIConfigUpdated)
-      fns.push(() =>
-        pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length)
-      )
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOUpdated(ConfigType.GOOGLE))
       if (!existing.config.activated && config.config.activated) {
@@ -253,7 +249,7 @@ export async function save(ctx: UserCtx<Config>) {
         if (existingConfig) {
           await verifyAIConfig(config, existingConfig)
         }
-        await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length)
+        await pro.quotas.addCustomAIConfig()
         break
     }
   } catch (err: any) {
@@ -342,29 +338,43 @@ export async function find(ctx: UserCtx) {
     let scopedConfig = await configs.getConfig(type)
 
     if (scopedConfig) {
-      if (type === ConfigType.OIDC_LOGOS) {
-        enrichOIDCLogos(scopedConfig)
-      }
-
-      if (type === ConfigType.AI) {
-        await pro.sdk.ai.enrichAIConfig(scopedConfig)
-        // Strip out the API Keys from the response so they don't show in the UI
-        for (const key in scopedConfig.config) {
-          if (scopedConfig.config[key].apiKey) {
-            scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT
-          }
-        }
-      }
-      ctx.body = scopedConfig
+      await handleConfigType(type, scopedConfig)
+    } else if (type === ConfigType.AI) {
+      scopedConfig = { config: {} } as AIConfig
+      await handleAIConfig(scopedConfig)
     } else {
-      // don't throw an error, there simply is nothing to return
+      // If no config found and not AI type, just return an empty body
       ctx.body = {}
+      return
     }
+
+    ctx.body = scopedConfig
   } catch (err: any) {
     ctx.throw(err?.status || 400, err)
   }
 }
 
+async function handleConfigType(type: ConfigType, config: Config) {
+  if (type === ConfigType.OIDC_LOGOS) {
+    enrichOIDCLogos(config)
+  } else if (type === ConfigType.AI) {
+    await handleAIConfig(config)
+  }
+}
+
+async function handleAIConfig(config: AIConfig) {
+  await pro.sdk.ai.enrichAIConfig(config)
+  stripApiKeys(config)
+}
+
+function stripApiKeys(config: AIConfig) {
+  for (const key in config?.config) {
+    if (config.config[key].apiKey) {
+      config.config[key].apiKey = PASSWORD_REPLACEMENT
+    }
+  }
+}
+
 export async function publicOidc(ctx: Ctx<void, GetPublicOIDCConfigResponse>) {
   try {
     // Find the config with the most granular scope based on context
@@ -508,6 +518,9 @@ export async function destroy(ctx: UserCtx) {
   try {
     await db.remove(id, rev)
     await cache.destroy(cache.CacheKey.CHECKLIST)
+    if (id === generateConfigID(ConfigType.AI)) {
+      await pro.quotas.removeCustomAIConfig()
+    }
     ctx.body = { message: "Config deleted successfully" }
   } catch (err: any) {
     ctx.throw(err.status, err)

From 29ab9bef7df3728c20d4db9a68ae1bc043f359d0 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 22 Oct 2024 18:31:07 +0100
Subject: [PATCH 332/455] fix lint

---
 packages/pro                                          | 2 +-
 packages/worker/src/api/controllers/global/configs.ts | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/pro b/packages/pro
index 7bd0350ed3f..1ca626aa65b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 7bd0350ed3f55da30d07a7c67fd746727ce93a9a
+Subproject commit 1ca626aa65bdbddf940a69947be6654b533cb726
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 6c09da77de8..5d9768443b1 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -35,7 +35,6 @@ import {
   AIInnerConfig,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
-import { generateConfigID } from "@budibase/backend-core/src/configs"
 
 const getEventFns = async (config: Config, existing?: Config) => {
   const fns = []
@@ -518,7 +517,7 @@ export async function destroy(ctx: UserCtx) {
   try {
     await db.remove(id, rev)
     await cache.destroy(cache.CacheKey.CHECKLIST)
-    if (id === generateConfigID(ConfigType.AI)) {
+    if (id === configs.generateConfigID(ConfigType.AI)) {
       await pro.quotas.removeCustomAIConfig()
     }
     ctx.body = { message: "Config deleted successfully" }

From 4f5e831b22473e0a9fd17d11b901d40dbbb58a58 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 22 Oct 2024 21:48:03 +0100
Subject: [PATCH 333/455] PR comments

---
 .../builder/src/pages/builder/portal/settings/ai/index.svelte  | 3 +--
 packages/worker/src/api/controllers/global/configs.ts          | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 9a70fd289a8..bbdf46a24e4 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -31,8 +31,7 @@
 
   async function fetchAIConfig() {
     try {
-      const aiDoc = await API.getConfig(ConfigTypes.AI)
-      fullAIConfig = aiDoc
+      fullAIConfig = await API.getConfig(ConfigTypes.AI)
     } catch (error) {
       notifications.error("Error fetching AI config")
     }
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 5d9768443b1..750b02088c9 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -44,7 +44,7 @@ const getEventFns = async (config: Config, existing?: Config) => {
       fns.push(events.email.SMTPCreated)
     } else if (isAIConfig(config)) {
       fns.push(() => events.ai.AIConfigCreated)
-      fns.push(() => pro.quotas.addCustomAIConfig)
+      fns.push(() => pro.quotas.addCustomAIConfig())
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE))
       if (config.config.activated) {

From 6541f6d31f384f90114537d9be9b01485e5b2125 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 22 Oct 2024 21:49:02 +0100
Subject: [PATCH 334/455] update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 1ca626aa65b..60411963053 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1ca626aa65bdbddf940a69947be6654b533cb726
+Subproject commit 60411963053b98e0415a1e9285b721fc26c628c8

From 07312b903b2da46239ef0da49f4a7d2c4a61a1fa Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 09:58:49 +0100
Subject: [PATCH 335/455]  Revert testing stuff

---
 packages/client/src/components/devtools/DevToolsHeader.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/devtools/DevToolsHeader.svelte b/packages/client/src/components/devtools/DevToolsHeader.svelte
index 89113318b75..eca085a88a6 100644
--- a/packages/client/src/components/devtools/DevToolsHeader.svelte
+++ b/packages/client/src/components/devtools/DevToolsHeader.svelte
@@ -37,7 +37,7 @@
 </script>
 
 <div class="dev-preview-header" class:mobile={$context.device.mobile}>
-  <Heading size="XS">Previ werewr eww</Heading>
+  <Heading size="XS">Preview</Heading>
   <Select
     quiet
     options={previewOptions}

From 8f0a234ae3f36a4d66f331292c05da04c6160b4a Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 23 Oct 2024 11:13:52 +0100
Subject: [PATCH 336/455] fix tests

---
 packages/server/src/automations/actions.ts                | 3 +--
 packages/types/src/documents/app/automation/schema.ts     | 8 ++++----
 .../src/api/controllers/global/tests/configs.spec.ts      | 4 ----
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts
index a6c9be41c5c..a8b69fa7d7f 100644
--- a/packages/server/src/automations/actions.ts
+++ b/packages/server/src/automations/actions.ts
@@ -51,6 +51,7 @@ const ACTION_IMPLS: ActionImplType = {
   QUERY_ROWS: queryRow.run,
   COLLECT: collect.run,
   TRIGGER_AUTOMATION_RUN: triggerAutomationRun.run,
+  OPENAI: openai.run,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.run,
   slack: slack.run,
@@ -107,8 +108,6 @@ export async function getActionDefinitions() {
     (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
   ) {
     BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
-    // @ts-ignore
-    ACTION_IMPLS["OPENAI"] = openai.run
   }
 
   const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
diff --git a/packages/types/src/documents/app/automation/schema.ts b/packages/types/src/documents/app/automation/schema.ts
index 599256694df..b8a19b7b450 100644
--- a/packages/types/src/documents/app/automation/schema.ts
+++ b/packages/types/src/documents/app/automation/schema.ts
@@ -126,16 +126,16 @@ export type ActionImplementations<T extends Hosting> = {
     n8nStepInputs,
     ExternalAppStepOutputs
   >
+  [AutomationActionStepId.OPENAI]: ActionImplementation<
+    OpenAIStepInputs,
+    OpenAIStepOutputs
+  >
 } & (T extends "self"
   ? {
       [AutomationActionStepId.EXECUTE_BASH]: ActionImplementation<
         BashStepInputs,
         BashStepOutputs
       >
-      [AutomationActionStepId.OPENAI]: ActionImplementation<
-        OpenAIStepInputs,
-        OpenAIStepOutputs
-      >
     }
   : {})
 
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 9091f292472..857f499dcc1 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -13,10 +13,6 @@ describe("Global configs controller", () => {
     await config.afterAll()
   })
 
-  afterEach(() => {
-    jest.resetAllMocks()
-  })
-
   it("Should strip secrets when pulling AI config", async () => {
     const data = structures.configs.ai()
     await config.api.configs.saveConfig(data)

From 713500febbab0c1c15dd35ff22ddc2a6ed21ecb5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 12:18:53 +0200
Subject: [PATCH 337/455] Add sdk.automation.find

---
 packages/server/src/sdk/app/automations/crud.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/server/src/sdk/app/automations/crud.ts b/packages/server/src/sdk/app/automations/crud.ts
index 3888e6882aa..61134999150 100644
--- a/packages/server/src/sdk/app/automations/crud.ts
+++ b/packages/server/src/sdk/app/automations/crud.ts
@@ -99,6 +99,12 @@ export async function get(automationId: string) {
   return trimUnexpectedObjectFields(result)
 }
 
+export async function find(ids: string[]) {
+  const db = getDb()
+  const result = await db.getMultiple<PersistedAutomation>(ids)
+  return result.map(trimUnexpectedObjectFields)
+}
+
 export async function create(automation: Automation) {
   automation = trimUnexpectedObjectFields(automation)
   const db = getDb()

From 1f01277ee8f544e03d629ac338b4ed313730abab Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 12:22:07 +0200
Subject: [PATCH 338/455] Clean code

---
 packages/server/src/sdk/app/rowActions.ts | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index 08a173ee13e..fa720792235 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -248,13 +248,8 @@ export async function run(
     throw new HTTPError("Table not found", 404)
   }
 
-  const rowActions = await getAll(tableId)
-  const rowAction = rowActions?.actions[rowActionId]
-  if (!rowAction) {
-    throw new HTTPError("Row action not found", 404)
-  }
-
-  const automation = await sdk.automations.get(rowAction.automationId)
+  const { automationId } = await get(tableId, rowActionId)
+  const automation = await sdk.automations.get(automationId)
 
   const row = await sdk.rows.find(tableId, rowId)
   await triggers.externalTrigger(

From 076f21108f0e928734ec4d5b5a3babdd2752c270 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 12:30:23 +0200
Subject: [PATCH 339/455] Remove unnecessary row action update endpoint

---
 .../src/api/controllers/rowAction/crud.ts     | 20 -------------------
 packages/server/src/api/routes/rowAction.ts   |  6 ------
 packages/server/src/sdk/app/rowActions.ts     | 14 -------------
 3 files changed, 40 deletions(-)

diff --git a/packages/server/src/api/controllers/rowAction/crud.ts b/packages/server/src/api/controllers/rowAction/crud.ts
index ca84b2ea304..71fbce16e6c 100644
--- a/packages/server/src/api/controllers/rowAction/crud.ts
+++ b/packages/server/src/api/controllers/rowAction/crud.ts
@@ -68,26 +68,6 @@ export async function create(
   ctx.status = 201
 }
 
-export async function update(
-  ctx: Ctx<UpdateRowActionRequest, RowActionResponse>
-) {
-  const table = await getTable(ctx)
-  const tableId = table._id!
-  const { actionId } = ctx.params
-
-  const action = await sdk.rowActions.update(tableId, actionId, {
-    name: ctx.request.body.name,
-  })
-
-  ctx.body = {
-    tableId,
-    id: action.id,
-    name: action.name,
-    automationId: action.automationId,
-    allowedSources: flattenAllowedSources(tableId, action.permissions),
-  }
-}
-
 export async function remove(ctx: Ctx<void, void>) {
   const table = await getTable(ctx)
   const { actionId } = ctx.params
diff --git a/packages/server/src/api/routes/rowAction.ts b/packages/server/src/api/routes/rowAction.ts
index 3d146335094..b4b595164c2 100644
--- a/packages/server/src/api/routes/rowAction.ts
+++ b/packages/server/src/api/routes/rowAction.ts
@@ -40,12 +40,6 @@ router
     rowActionValidator(),
     rowActionController.create
   )
-  .put(
-    "/api/tables/:tableId/actions/:actionId",
-    authorized(BUILDER),
-    rowActionValidator(),
-    rowActionController.update
-  )
   .delete(
     "/api/tables/:tableId/actions/:actionId",
     authorized(BUILDER),
diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index fa720792235..f1064396fc3 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -159,20 +159,6 @@ async function updateDoc(
   }
 }
 
-export async function update(
-  tableId: string,
-  rowActionId: string,
-  rowActionData: { name: string }
-) {
-  const newName = rowActionData.name.trim()
-
-  return await updateDoc(tableId, rowActionId, actionsDoc => {
-    ensureUniqueAndThrow(actionsDoc, newName, rowActionId)
-    actionsDoc.actions[rowActionId].name = newName
-    return actionsDoc
-  })
-}
-
 async function guardView(tableId: string, viewId: string) {
   let view
   if (docIds.isViewId(viewId)) {

From 6f4e1dd711a8d0e85078b9ebcbb1bae10dbec994 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 23 Oct 2024 11:34:55 +0100
Subject: [PATCH 340/455] Respond to PR comment.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 167 +++++++++---------
 packages/server/src/sdk/app/tables/getters.ts |  22 +--
 packages/server/src/sdk/app/views/external.ts |  21 +--
 packages/server/src/sdk/app/views/internal.ts |  21 +--
 packages/server/src/sdk/app/views/utils.ts    |  10 +-
 packages/shared-core/src/filters.ts           |   2 +
 packages/shared-core/src/utils.ts             |   6 +-
 7 files changed, 122 insertions(+), 127 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index a061a370ba4..2af11b513bc 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -27,13 +27,14 @@ import {
   ViewV2Schema,
   ViewV2Type,
   JsonTypes,
-  UILogicalOperator,
   EmptyFilterOption,
   JsonFieldSubType,
   UISearchFilter,
   LegacyFilter,
   SearchViewRowRequest,
   ArrayOperator,
+  UILogicalOperator,
+  SearchFilters,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -159,11 +160,8 @@ describe.each([
           tableId: table._id!,
           primaryDisplay: "id",
           queryUI: {
-            logicalOperator: UILogicalOperator.ALL,
-            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
             groups: [
               {
-                logicalOperator: UILogicalOperator.ALL,
                 filters: [
                   {
                     operator: BasicOperator.EQUAL,
@@ -256,6 +254,8 @@ describe.each([
             },
           },
           queryUI: {
+            logicalOperator: UILogicalOperator.ALL,
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
             groups: [
               {
                 logicalOperator: UILogicalOperator.ALL,
@@ -952,29 +952,37 @@ describe.each([
           ],
         })
 
-        expect((await config.api.table.get(tableId)).views).toEqual({
-          [view.name]: {
-            ...view,
-            query: [
-              { operator: "equal", field: "newField", value: "thatValue" },
-            ],
-            // Should also update queryUI because query was not previously set.
-            queryUI: {
-              groups: [
-                {
-                  logicalOperator: "all",
-                  filters: [
-                    {
-                      operator: "equal",
-                      field: "newField",
-                      value: "thatValue",
-                    },
-                  ],
-                },
-              ],
+        const expected: ViewV2 = {
+          ...view,
+          query: [
+            {
+              operator: BasicOperator.EQUAL,
+              field: "newField",
+              value: "thatValue",
             },
-            schema: expect.anything(),
+          ],
+          // Should also update queryUI because query was not previously set.
+          queryUI: {
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+            logicalOperator: UILogicalOperator.ALL,
+            groups: [
+              {
+                logicalOperator: UILogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "newField",
+                    value: "thatValue",
+                  },
+                ],
+              },
+            ],
           },
+          schema: expect.anything(),
+        }
+
+        expect((await config.api.table.get(tableId)).views).toEqual({
+          [view.name]: expected,
         })
       })
 
@@ -1014,38 +1022,42 @@ describe.each([
         }
         await config.api.viewV2.update(updatedData)
 
-        expect((await config.api.table.get(tableId)).views).toEqual({
-          [view.name]: {
-            ...updatedData,
-            // queryUI gets generated from query
-            queryUI: {
-              groups: [
-                {
-                  logicalOperator: "all",
-                  filters: [
-                    {
-                      operator: "equal",
-                      field: "newField",
-                      value: "newValue",
-                    },
-                  ],
-                },
-              ],
-            },
-            schema: {
-              ...table.schema,
-              id: expect.objectContaining({
-                visible: true,
-              }),
-              Category: expect.objectContaining({
-                visible: false,
-              }),
-              Price: expect.objectContaining({
-                visible: true,
-                readonly: true,
-              }),
-            },
+        const expected: ViewV2 = {
+          ...updatedData,
+          // queryUI gets generated from query
+          queryUI: {
+            logicalOperator: UILogicalOperator.ALL,
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+            groups: [
+              {
+                logicalOperator: UILogicalOperator.ALL,
+                filters: [
+                  {
+                    operator: BasicOperator.EQUAL,
+                    field: "newField",
+                    value: "newValue",
+                  },
+                ],
+              },
+            ],
+          },
+          schema: {
+            ...table.schema,
+            id: expect.objectContaining({
+              visible: true,
+            }),
+            Category: expect.objectContaining({
+              visible: false,
+            }),
+            Price: expect.objectContaining({
+              visible: true,
+              readonly: true,
+            }),
           },
+        }
+
+        expect((await config.api.table.get(tableId)).views).toEqual({
+          [view.name]: expected,
         })
       })
 
@@ -1283,7 +1295,6 @@ describe.each([
           queryUI: {
             groups: [
               {
-                logicalOperator: UILogicalOperator.ALL,
                 filters: [
                   {
                     operator: BasicOperator.EQUAL,
@@ -1297,7 +1308,8 @@ describe.each([
         })
 
         let updatedView = await config.api.viewV2.get(view.id)
-        expect(updatedView.query).toEqual({
+        let expected: SearchFilters = {
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
           $and: {
             conditions: [
               {
@@ -1311,14 +1323,14 @@ describe.each([
               },
             ],
           },
-        })
+        }
+        expect(updatedView.query).toEqual(expected)
 
         await config.api.viewV2.update({
           ...updatedView,
           queryUI: {
             groups: [
               {
-                logicalOperator: UILogicalOperator.ALL,
                 filters: [
                   {
                     operator: BasicOperator.EQUAL,
@@ -1332,7 +1344,8 @@ describe.each([
         })
 
         updatedView = await config.api.viewV2.get(view.id)
-        expect(updatedView.query).toEqual({
+        expected = {
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
           $and: {
             conditions: [
               {
@@ -1346,7 +1359,8 @@ describe.each([
               },
             ],
           },
-        })
+        }
+        expect(updatedView.query).toEqual(expected)
       })
 
       it("can delete either query and it will get regenerated from queryUI", async () => {
@@ -1386,7 +1400,6 @@ describe.each([
           queryUI: {
             groups: [
               {
-                logicalOperator: UILogicalOperator.ALL,
                 filters: [
                   {
                     operator: BasicOperator.EQUAL,
@@ -1810,7 +1823,9 @@ describe.each([
         })
 
         const view = await getDelegate(res)
-        expect(view.queryUI).toEqual({
+        const expected: UISearchFilter = {
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+          logicalOperator: UILogicalOperator.ALL,
           groups: [
             {
               logicalOperator: UILogicalOperator.ALL,
@@ -1823,7 +1838,8 @@ describe.each([
               ],
             },
           ],
-        })
+        }
+        expect(view.queryUI).toEqual(expected)
       })
     })
 
@@ -2669,11 +2685,8 @@ describe.each([
             tableId: table._id!,
             name: generator.guid(),
             queryUI: {
-              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.EQUAL,
@@ -2733,11 +2746,8 @@ describe.each([
             tableId: table._id!,
             name: generator.guid(),
             queryUI: {
-              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.EQUAL,
@@ -3022,11 +3032,8 @@ describe.each([
             tableId: table._id!,
             name: generator.guid(),
             queryUI: {
-              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.NOT_EQUAL,
@@ -3080,11 +3087,8 @@ describe.each([
             tableId: table._id!,
             name: generator.guid(),
             queryUI: {
-              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.NOT_EQUAL,
@@ -3175,11 +3179,8 @@ describe.each([
               tableId: table._id!,
               name: generator.guid(),
               queryUI: {
-                onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-                logicalOperator: UILogicalOperator.ALL,
                 groups: [
                   {
-                    logicalOperator: UILogicalOperator.ALL,
                     filters: [
                       {
                         operator: BasicOperator.EQUAL,
@@ -3619,11 +3620,8 @@ describe.each([
                 name: generator.guid(),
                 type: ViewV2Type.CALCULATION,
                 queryUI: {
-                  onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-                  logicalOperator: UILogicalOperator.ALL,
                   groups: [
                     {
-                      logicalOperator: UILogicalOperator.ALL,
                       filters: [
                         {
                           operator: BasicOperator.EQUAL,
@@ -3912,11 +3910,8 @@ describe.each([
             tableId: table._id!,
             name: generator.guid(),
             queryUI: {
-              onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-              logicalOperator: UILogicalOperator.ALL,
               groups: [
                 {
-                  logicalOperator: UILogicalOperator.ALL,
                   filters: [
                     {
                       operator: BasicOperator.EQUAL,
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 270ae7fee09..a8ad606647f 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -12,8 +12,6 @@ import {
   TableResponse,
   TableSourceType,
   TableViewsResponse,
-  View,
-  ViewV2,
   FeatureFlag,
 } from "@budibase/types"
 import datasources from "../datasources"
@@ -21,19 +19,6 @@ import sdk from "../../../sdk"
 import { ensureQueryUISet } from "../views/utils"
 import { isV2 } from "../views"
 
-function processView(view: ViewV2 | View) {
-  if (!isV2(view)) {
-    return
-  }
-  ensureQueryUISet(view)
-}
-
-function processViews(views: (ViewV2 | View)[]) {
-  for (const view of views) {
-    processView(view)
-  }
-}
-
 export async function processTable(table: Table): Promise<Table> {
   if (!table) {
     return table
@@ -41,7 +26,12 @@ export async function processTable(table: Table): Promise<Table> {
 
   table = { ...table }
   if (table.views) {
-    processViews(Object.values(table.views))
+    for (const [key, view] of Object.entries(table.views)) {
+      if (!isV2(view)) {
+        continue
+      }
+      table.views[key] = ensureQueryUISet(view)
+    }
   }
   if (table._id && isExternalTableID(table._id)) {
     // Old created external tables via Budibase might have a missing field name breaking some UI such as filters
diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts
index 74f2248f58a..bee153a9107 100644
--- a/packages/server/src/sdk/app/views/external.ts
+++ b/packages/server/src/sdk/app/views/external.ts
@@ -19,8 +19,7 @@ export async function get(viewId: string): Promise<ViewV2> {
   if (!found) {
     throw new Error("No view found")
   }
-  ensureQueryUISet(found)
-  return found
+  return ensureQueryUISet(found)
 }
 
 export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
@@ -35,22 +34,21 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   if (!found) {
     throw new Error("No view found")
   }
-  ensureQueryUISet(found)
-  return await enrichSchema(found, table.schema)
+  return await enrichSchema(ensureQueryUISet(found), table.schema)
 }
 
 export async function create(
   tableId: string,
   viewRequest: Omit<ViewV2, "id" | "version">
 ): Promise<ViewV2> {
-  const view: ViewV2 = {
+  let view: ViewV2 = {
     ...viewRequest,
     id: utils.generateViewID(tableId),
     version: 2,
   }
 
-  ensureQuerySet(view)
-  ensureQueryUISet(view)
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
 
   const db = context.getAppDB()
 
@@ -62,7 +60,10 @@ export async function create(
   return view
 }
 
-export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
+export async function update(
+  tableId: string,
+  view: Readonly<ViewV2>
+): Promise<ViewV2> {
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -80,8 +81,8 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`Cannot update view type after creation`, 400)
   }
 
-  ensureQuerySet(view)
-  ensureQueryUISet(view)
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
 
   delete views[existingView.name]
   views[view.name] = view
diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts
index 77a81a16231..63807bcfd44 100644
--- a/packages/server/src/sdk/app/views/internal.ts
+++ b/packages/server/src/sdk/app/views/internal.ts
@@ -14,8 +14,7 @@ export async function get(viewId: string): Promise<ViewV2> {
   if (!found) {
     throw new Error("No view found")
   }
-  ensureQueryUISet(found)
-  return found
+  return ensureQueryUISet(found)
 }
 
 export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
@@ -26,22 +25,21 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   if (!found) {
     throw new Error("No view found")
   }
-  ensureQueryUISet(found)
-  return await enrichSchema(found, table.schema)
+  return await enrichSchema(ensureQueryUISet(found), table.schema)
 }
 
 export async function create(
   tableId: string,
   viewRequest: Omit<ViewV2, "id" | "version">
 ): Promise<ViewV2> {
-  const view: ViewV2 = {
+  let view: ViewV2 = {
     ...viewRequest,
     id: utils.generateViewID(tableId),
     version: 2,
   }
 
-  ensureQuerySet(view)
-  ensureQueryUISet(view)
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
 
   const db = context.getAppDB()
 
@@ -53,7 +51,10 @@ export async function create(
   return view
 }
 
-export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
+export async function update(
+  tableId: string,
+  view: Readonly<ViewV2>
+): Promise<ViewV2> {
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
   table.views ??= {}
@@ -69,8 +70,8 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`Cannot update view type after creation`, 400)
   }
 
-  ensureQuerySet(view)
-  ensureQueryUISet(view)
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
 
   delete table.views[existingView.name]
   table.views[view.name] = view
diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
index ec942c617a5..8ca5fd1098f 100644
--- a/packages/server/src/sdk/app/views/utils.ts
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -1,13 +1,14 @@
 import { ViewV2 } from "@budibase/types"
 import { utils, dataFilters } from "@budibase/shared-core"
-import { isPlainObject } from "lodash"
+import { cloneDeep, isPlainObject } from "lodash"
 import { HTTPError } from "@budibase/backend-core"
 
 function isEmptyObject(obj: any) {
   return obj && isPlainObject(obj) && Object.keys(obj).length === 0
 }
 
-export function ensureQueryUISet(view: ViewV2) {
+export function ensureQueryUISet(viewArg: Readonly<ViewV2>): ViewV2 {
+  const view = cloneDeep<ViewV2>(viewArg)
   if (!view.queryUI && view.query && !isEmptyObject(view.query)) {
     if (!Array.isArray(view.query)) {
       // In practice this should not happen. `view.query`, at the time this code
@@ -27,13 +28,16 @@ export function ensureQueryUISet(view: ViewV2) {
 
     view.queryUI = utils.processSearchFilters(view.query)
   }
+  return view
 }
 
-export function ensureQuerySet(view: ViewV2) {
+export function ensureQuerySet(viewArg: Readonly<ViewV2>): ViewV2 {
+  const view = cloneDeep<ViewV2>(viewArg)
   // We consider queryUI to be the source of truth, so we don't check for the
   // presence of query here. We will overwrite it regardless of whether it is
   // present or not.
   if (view.queryUI && !isEmptyObject(view.queryUI)) {
     view.query = dataFilters.buildQuery(view.queryUI)
   }
+  return view
 }
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index efdd3b0df2a..a5d4e7ca4de 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -475,6 +475,8 @@ export function buildQuery(
   const query: SearchFilters = {}
   if (filter.onEmptyFilter) {
     query.onEmptyFilter = filter.onEmptyFilter
+  } else {
+    query.onEmptyFilter = EmptyFilterOption.RETURN_ALL
   }
 
   query[operator] = {
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 0af27d4e81f..5db4ead1dc7 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -7,6 +7,7 @@ import {
   ArrayOperator,
   isLogicalSearchOperator,
   SearchFilter,
+  EmptyFilterOption,
 } from "@budibase/types"
 import * as Constants from "./constants"
 import { removeKeyNumbering, splitFiltersArray } from "./filters"
@@ -139,10 +140,11 @@ export function isSupportedUserSearch(query: SearchFilters) {
 
 export const processSearchFilters = (
   filterArray: LegacyFilter[]
-): UISearchFilter => {
+): Required<UISearchFilter> => {
   const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray)
   return {
-    onEmptyFilter,
+    logicalOperator: UILogicalOperator.ALL,
+    onEmptyFilter: onEmptyFilter || EmptyFilterOption.RETURN_ALL,
     groups: [
       {
         logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL,

From 7781e75491b5430850452db05a9b691c4d2bd875 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 12:47:06 +0200
Subject: [PATCH 341/455] Simplify row action permissions response

---
 .../src/api/controllers/rowAction/crud.ts     | 34 +++++++------------
 packages/types/src/api/web/app/rowAction.ts   | 12 +++++--
 2 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/packages/server/src/api/controllers/rowAction/crud.ts b/packages/server/src/api/controllers/rowAction/crud.ts
index 71fbce16e6c..d6eadb65b13 100644
--- a/packages/server/src/api/controllers/rowAction/crud.ts
+++ b/packages/server/src/api/controllers/rowAction/crud.ts
@@ -2,9 +2,9 @@ import {
   CreateRowActionRequest,
   Ctx,
   RowActionPermissions,
+  RowActionPermissionsResponse,
   RowActionResponse,
   RowActionsResponse,
-  UpdateRowActionRequest,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 
@@ -76,22 +76,22 @@ export async function remove(ctx: Ctx<void, void>) {
   ctx.status = 204
 }
 
-export async function setTablePermission(ctx: Ctx<void, RowActionResponse>) {
+export async function setTablePermission(
+  ctx: Ctx<void, RowActionPermissionsResponse>
+) {
   const table = await getTable(ctx)
   const tableId = table._id!
   const { actionId } = ctx.params
 
   const action = await sdk.rowActions.setTablePermission(tableId, actionId)
   ctx.body = {
-    tableId,
-    id: action.id,
-    name: action.name,
-    automationId: action.automationId,
     allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
 
-export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
+export async function unsetTablePermission(
+  ctx: Ctx<void, RowActionPermissionsResponse>
+) {
   const table = await getTable(ctx)
   const tableId = table._id!
   const { actionId } = ctx.params
@@ -99,15 +99,13 @@ export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
   const action = await sdk.rowActions.unsetTablePermission(tableId, actionId)
 
   ctx.body = {
-    tableId,
-    id: action.id,
-    name: action.name,
-    automationId: action.automationId,
     allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
 
-export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
+export async function setViewPermission(
+  ctx: Ctx<void, RowActionPermissionsResponse>
+) {
   const table = await getTable(ctx)
   const tableId = table._id!
   const { actionId, viewId } = ctx.params
@@ -118,15 +116,13 @@ export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
     viewId
   )
   ctx.body = {
-    tableId,
-    id: action.id,
-    name: action.name,
-    automationId: action.automationId,
     allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
 
-export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
+export async function unsetViewPermission(
+  ctx: Ctx<void, RowActionPermissionsResponse>
+) {
   const table = await getTable(ctx)
   const tableId = table._id!
   const { actionId, viewId } = ctx.params
@@ -138,10 +134,6 @@ export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
   )
 
   ctx.body = {
-    tableId,
-    id: action.id,
-    name: action.name,
-    automationId: action.automationId,
     allowedSources: flattenAllowedSources(tableId, action.permissions),
   }
 }
diff --git a/packages/types/src/api/web/app/rowAction.ts b/packages/types/src/api/web/app/rowAction.ts
index 0061be275ec..84e4e529edc 100644
--- a/packages/types/src/api/web/app/rowAction.ts
+++ b/packages/types/src/api/web/app/rowAction.ts
@@ -1,14 +1,17 @@
 interface RowActionData {
   name: string
 }
+interface RowActionPermissionsData {
+  allowedSources: string[] | undefined
+}
 export interface CreateRowActionRequest extends RowActionData {}
-export interface UpdateRowActionRequest extends RowActionData {}
 
-export interface RowActionResponse extends RowActionData {
+export interface RowActionResponse
+  extends RowActionData,
+    RowActionPermissionsData {
   id: string
   tableId: string
   automationId: string
-  allowedSources: string[] | undefined
 }
 
 export interface RowActionsResponse {
@@ -18,3 +21,6 @@ export interface RowActionsResponse {
 export interface RowActionTriggerRequest {
   rowId: string
 }
+
+export interface RowActionPermissionsResponse
+  extends RowActionPermissionsData {}

From 015854f1a2fd207f40b4552b4038aa4238d107c3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 12:49:42 +0200
Subject: [PATCH 342/455] Dont persist row action name

---
 .../src/api/controllers/rowAction/crud.ts     | 12 +++++-
 .../server/src/sdk/app/automations/utils.ts   | 17 +-------
 packages/server/src/sdk/app/rowActions.ts     | 39 ++++++++++++-------
 packages/types/src/documents/app/rowAction.ts |  1 -
 4 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/packages/server/src/api/controllers/rowAction/crud.ts b/packages/server/src/api/controllers/rowAction/crud.ts
index d6eadb65b13..014ebcf426c 100644
--- a/packages/server/src/api/controllers/rowAction/crud.ts
+++ b/packages/server/src/api/controllers/rowAction/crud.ts
@@ -30,6 +30,16 @@ export async function find(ctx: Ctx<void, RowActionsResponse>) {
   }
 
   const { actions } = rowActions
+  const automations = await sdk.automations.find(
+    Object.values(actions).map(({ automationId }) => automationId)
+  )
+  const automationNames = automations.reduce<Record<string, string>>(
+    (names, a) => {
+      names[a._id] = a.name
+      return names
+    },
+    {}
+  )
   const result: RowActionsResponse = {
     actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
       (acc, [key, action]) => ({
@@ -37,7 +47,7 @@ export async function find(ctx: Ctx<void, RowActionsResponse>) {
         [key]: {
           id: key,
           tableId,
-          name: action.name,
+          name: automationNames[action.automationId],
           automationId: action.automationId,
           allowedSources: flattenAllowedSources(tableId, action.permissions),
         },
diff --git a/packages/server/src/sdk/app/automations/utils.ts b/packages/server/src/sdk/app/automations/utils.ts
index 8b397df75f7..f5ae8bf9484 100644
--- a/packages/server/src/sdk/app/automations/utils.ts
+++ b/packages/server/src/sdk/app/automations/utils.ts
@@ -2,7 +2,6 @@ import {
   Automation,
   AutomationActionStepId,
   AutomationBuilderData,
-  TableRowActions,
 } from "@budibase/types"
 import { sdk as coreSdk } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -26,15 +25,6 @@ export async function getBuilderData(
     return tableNameCache[tableId]
   }
 
-  const rowActionNameCache: Record<string, TableRowActions | undefined> = {}
-  async function getRowActionName(tableId: string, rowActionId: string) {
-    if (!rowActionNameCache[tableId]) {
-      rowActionNameCache[tableId] = await sdk.rowActions.getAll(tableId)
-    }
-
-    return rowActionNameCache[tableId]?.actions[rowActionId]?.name
-  }
-
   const result: Record<string, AutomationBuilderData> = {}
   for (const automation of automations) {
     const isRowAction = coreSdk.automations.isRowAction(automation)
@@ -49,12 +39,7 @@ export async function getBuilderData(
     }
 
     const tableName = await getTableName(tableId)
-    const rowActionName = await getRowActionName(tableId, rowActionId)
-
-    if (!rowActionName) {
-      throw new Error(`Row action not found: ${rowActionId}`)
-    }
-
+    const rowActionName = automation.name
     result[automation._id!] = {
       displayName: rowActionName,
       triggerInfo: {
diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index f1064396fc3..c6182ed38bd 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -13,16 +13,17 @@ import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInf
 import * as triggers from "../../automations/triggers"
 import sdk from ".."
 
-function ensureUniqueAndThrow(
+async function ensureUniqueAndThrow(
   doc: TableRowActions,
   name: string,
   existingRowActionId?: string
 ) {
+  const names = await getNames(doc)
+  name = name.toLowerCase()
+
   if (
-    Object.entries(doc.actions).find(
-      ([id, a]) =>
-        a.name.toLowerCase() === name.toLowerCase() &&
-        id !== existingRowActionId
+    Object.entries(names).find(
+      ([id, name]) => name === name && id !== existingRowActionId
     )
   ) {
     throw new HTTPError("A row action with the same name already exists.", 409)
@@ -34,18 +35,12 @@ export async function create(tableId: string, rowAction: { name: string }) {
 
   const db = context.getAppDB()
   const rowActionsId = generateRowActionsID(tableId)
-  let doc: TableRowActions
-  try {
-    doc = await db.get<TableRowActions>(rowActionsId)
-  } catch (e: any) {
-    if (e.status !== 404) {
-      throw e
-    }
-
+  let doc = await db.tryGet<TableRowActions>(rowActionsId)
+  if (!doc) {
     doc = { _id: rowActionsId, actions: {} }
   }
 
-  ensureUniqueAndThrow(doc, action.name)
+  await ensureUniqueAndThrow(doc, action.name)
 
   const appId = context.getAppId()
   if (!appId) {
@@ -74,7 +69,6 @@ export async function create(tableId: string, rowAction: { name: string }) {
   })
 
   doc.actions[newRowActionId] = {
-    name: action.name,
     automationId: automation._id!,
     permissions: {
       table: { runAllowed: true },
@@ -85,6 +79,7 @@ export async function create(tableId: string, rowAction: { name: string }) {
 
   return {
     id: newRowActionId,
+    name: automation.name,
     ...doc.actions[newRowActionId],
   }
 }
@@ -253,3 +248,17 @@ export async function run(
     { getResponses: true }
   )
 }
+
+export async function getNames(actions: TableRowActions) {
+  const automations = await sdk.automations.find(
+    Object.values(actions).map(({ automationId }) => automationId)
+  )
+  const automationNames = automations.reduce<Record<string, string>>(
+    (names, a) => {
+      names[a._id] = a.name
+      return names
+    },
+    {}
+  )
+  return automationNames
+}
diff --git a/packages/types/src/documents/app/rowAction.ts b/packages/types/src/documents/app/rowAction.ts
index 071b480b1c0..db3637b708b 100644
--- a/packages/types/src/documents/app/rowAction.ts
+++ b/packages/types/src/documents/app/rowAction.ts
@@ -6,7 +6,6 @@ export interface TableRowActions extends Document {
 }
 
 export interface RowActionData {
-  name: string
   automationId: string
   permissions: RowActionPermissions
 }

From dab38265dbc942364ec601be85e33e5cc298b4f2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 12:54:32 +0200
Subject: [PATCH 343/455] Allow renaming row actions

---
 packages/server/src/sdk/app/automations/crud.ts | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/packages/server/src/sdk/app/automations/crud.ts b/packages/server/src/sdk/app/automations/crud.ts
index 61134999150..b982d5b45e9 100644
--- a/packages/server/src/sdk/app/automations/crud.ts
+++ b/packages/server/src/sdk/app/automations/crud.ts
@@ -1,4 +1,3 @@
-import { sdk } from "@budibase/shared-core"
 import {
   Automation,
   RequiredKeys,
@@ -295,13 +294,6 @@ function guardInvalidUpdatesAndThrow(
       }
     })
   }
-
-  if (
-    sdk.automations.isRowAction(automation) &&
-    automation.name !== oldAutomation.name
-  ) {
-    throw new Error("Row actions cannot be renamed")
-  }
 }
 
 function trimUnexpectedObjectFields<T extends Automation>(automation: T): T {

From 2868c840e628e519978e8ff6c274fc5e66c4f5bd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:02:25 +0200
Subject: [PATCH 344/455] Remove duplicated modals

---
 .../AutomationPanel/AutomationNavItem.svelte  |  9 +-
 .../UpdateRowActionModal.svelte               | 83 -------------------
 2 files changed, 2 insertions(+), 90 deletions(-)
 delete mode 100644 packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index ec9b9561904..14857da3736 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -17,7 +17,6 @@
 
   let confirmDeleteDialog
   let updateAutomationDialog
-  let updateRowActionDialog
 
   $: isRowAction = sdk.automations.isRowAction(automation)
 
@@ -92,7 +91,7 @@
           name: "Edit",
           keyBind: null,
           visible: true,
-          callback: updateRowActionDialog.show,
+          callback: updateAutomationDialog.show,
         },
         del,
       ]
@@ -135,8 +134,4 @@
   This action cannot be undone.
 </ConfirmDialog>
 
-{#if isRowAction}
-  <UpdateRowActionModal {automation} bind:this={updateRowActionDialog} />
-{:else}
-  <UpdateAutomationModal {automation} bind:this={updateAutomationDialog} />
-{/if}
+<UpdateAutomationModal {automation} bind:this={updateAutomationDialog} />
diff --git a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte b/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
deleted file mode 100644
index 20f06608dc4..00000000000
--- a/packages/builder/src/components/automation/AutomationPanel/UpdateRowActionModal.svelte
+++ /dev/null
@@ -1,83 +0,0 @@
-<script>
-  import { rowActions } from "stores/builder"
-  import {
-    notifications,
-    Icon,
-    Input,
-    ModalContent,
-    Modal,
-  } from "@budibase/bbui"
-
-  export let automation
-  export let onCancel = undefined
-
-  let name
-  let error = ""
-  let modal
-
-  export const show = () => {
-    name = automation?.displayName
-    modal.show()
-  }
-  export const hide = () => {
-    modal.hide()
-  }
-
-  async function saveAutomation() {
-    try {
-      await rowActions.rename(
-        automation.definition.trigger.inputs.tableId,
-        automation.definition.trigger.inputs.rowActionId,
-        name
-      )
-      notifications.success(`Row action updated successfully`)
-      hide()
-    } catch (error) {
-      notifications.error("Error saving row action")
-    }
-  }
-
-  function checkValid(evt) {
-    name = evt.target.value
-    if (!name) {
-      error = "Name is required"
-      return
-    }
-    error = ""
-  }
-</script>
-
-<Modal bind:this={modal} on:hide={onCancel}>
-  <ModalContent
-    title="Edit Row Action"
-    confirmText="Save"
-    size="L"
-    onConfirm={saveAutomation}
-    disabled={error}
-  >
-    <Input bind:value={name} label="Name" on:input={checkValid} {error} />
-    <a
-      slot="footer"
-      target="_blank"
-      href="https://docs.budibase.com/docs/automation-steps"
-    >
-      <Icon name="InfoOutline" />
-      <span>Learn about automations</span>
-    </a>
-  </ModalContent>
-</Modal>
-
-<style>
-  a {
-    color: var(--ink);
-    font-size: 14px;
-    vertical-align: middle;
-    display: flex;
-    align-items: center;
-    text-decoration: none;
-  }
-  a span {
-    text-decoration: underline;
-    margin-left: var(--spectrum-alias-item-padding-s);
-  }
-</style>

From 022ec12b1bdb16ec32c7ac22c80b63fecf03bfab Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:08:15 +0200
Subject: [PATCH 345/455] Clean

---
 .../automation/AutomationPanel/AutomationNavItem.svelte          | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index 14857da3736..50814897b48 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -9,7 +9,6 @@
   import { sdk } from "@budibase/shared-core"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import UpdateAutomationModal from "components/automation/AutomationPanel/UpdateAutomationModal.svelte"
-  import UpdateRowActionModal from "components/automation/AutomationPanel/UpdateRowActionModal.svelte"
   import NavItem from "components/common/NavItem.svelte"
 
   export let automation

From 8f990725523d1ec71a0b62313f91d7789d22d6e4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:08:22 +0200
Subject: [PATCH 346/455] Reuse

---
 packages/server/src/api/controllers/rowAction/crud.ts | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/packages/server/src/api/controllers/rowAction/crud.ts b/packages/server/src/api/controllers/rowAction/crud.ts
index 014ebcf426c..3baaff6fcca 100644
--- a/packages/server/src/api/controllers/rowAction/crud.ts
+++ b/packages/server/src/api/controllers/rowAction/crud.ts
@@ -30,16 +30,7 @@ export async function find(ctx: Ctx<void, RowActionsResponse>) {
   }
 
   const { actions } = rowActions
-  const automations = await sdk.automations.find(
-    Object.values(actions).map(({ automationId }) => automationId)
-  )
-  const automationNames = automations.reduce<Record<string, string>>(
-    (names, a) => {
-      names[a._id] = a.name
-      return names
-    },
-    {}
-  )
+  const automationNames = await sdk.rowActions.getNames(rowActions)
   const result: RowActionsResponse = {
     actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
       (acc, [key, action]) => ({

From b4a979ed97cffc41a32cc23967137eb4917a8036 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:10:50 +0200
Subject: [PATCH 347/455] Fix get names

---
 packages/server/src/sdk/app/rowActions.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index c6182ed38bd..d5c7666edb8 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -249,7 +249,7 @@ export async function run(
   )
 }
 
-export async function getNames(actions: TableRowActions) {
+export async function getNames({ actions }: TableRowActions) {
   const automations = await sdk.automations.find(
     Object.values(actions).map(({ automationId }) => automationId)
   )

From 1fef7838f2ba17efc7a320efd12cc61f9616c819 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:15:07 +0200
Subject: [PATCH 348/455] Check names properly

---
 packages/server/src/sdk/app/rowActions.ts            | 3 ++-
 packages/server/src/tests/utilities/api/rowAction.ts | 9 +++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index d5c7666edb8..6a0a803bd57 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -23,7 +23,8 @@ async function ensureUniqueAndThrow(
 
   if (
     Object.entries(names).find(
-      ([id, name]) => name === name && id !== existingRowActionId
+      ([automationId, automationName]) =>
+        automationName === name && automationId !== existingRowActionId
     )
   ) {
     throw new HTTPError("A row action with the same name already exists.", 409)
diff --git a/packages/server/src/tests/utilities/api/rowAction.ts b/packages/server/src/tests/utilities/api/rowAction.ts
index 0fd59362577..e18f2ce6b67 100644
--- a/packages/server/src/tests/utilities/api/rowAction.ts
+++ b/packages/server/src/tests/utilities/api/rowAction.ts
@@ -1,5 +1,6 @@
 import {
   CreateRowActionRequest,
+  RowActionPermissionsResponse,
   RowActionResponse,
   RowActionsResponse,
   RowActionTriggerRequest,
@@ -78,7 +79,7 @@ export class RowActionAPI extends TestAPI {
     expectations?: Expectations,
     config?: { publicUser?: boolean }
   ) => {
-    return await this._post<RowActionResponse>(
+    return await this._post<RowActionPermissionsResponse>(
       `/api/tables/${tableId}/actions/${rowActionId}/permissions`,
       {
         expectations: {
@@ -96,7 +97,7 @@ export class RowActionAPI extends TestAPI {
     expectations?: Expectations,
     config?: { publicUser?: boolean }
   ) => {
-    return await this._delete<RowActionResponse>(
+    return await this._delete<RowActionPermissionsResponse>(
       `/api/tables/${tableId}/actions/${rowActionId}/permissions`,
       {
         expectations: {
@@ -115,7 +116,7 @@ export class RowActionAPI extends TestAPI {
     expectations?: Expectations,
     config?: { publicUser?: boolean }
   ) => {
-    return await this._post<RowActionResponse>(
+    return await this._post<RowActionPermissionsResponse>(
       `/api/tables/${tableId}/actions/${rowActionId}/permissions/${viewId}`,
       {
         expectations: {
@@ -134,7 +135,7 @@ export class RowActionAPI extends TestAPI {
     expectations?: Expectations,
     config?: { publicUser?: boolean }
   ) => {
-    return await this._delete<RowActionResponse>(
+    return await this._delete<RowActionPermissionsResponse>(
       `/api/tables/${tableId}/actions/${rowActionId}/permissions/${viewId}`,
       {
         expectations: {

From 732d701a868bd7b198cb5a9dcd64b12fdf565cb5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:16:55 +0200
Subject: [PATCH 349/455] Remove row action update tests

---
 .../src/api/routes/tests/rowAction.spec.ts    | 123 ------------------
 .../src/tests/utilities/api/rowAction.ts      |  17 ---
 2 files changed, 140 deletions(-)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index c0b5c4210fd..5cec83fe23d 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -328,129 +328,6 @@ describe("/rowsActions", () => {
     })
   })
 
-  describe("update", () => {
-    unauthorisedTests((expectations, testConfig) =>
-      config.api.rowAction.update(
-        tableId,
-        generator.guid(),
-        createRowActionRequest(),
-        expectations,
-        testConfig
-      )
-    )
-
-    it("can update existing actions", async () => {
-      for (const rowAction of createRowActionRequests(3)) {
-        await createRowAction(tableId, rowAction)
-      }
-
-      const persisted = await config.api.rowAction.find(tableId)
-
-      const [actionId, actionData] = _.sample(
-        Object.entries(persisted.actions)
-      )!
-
-      const updatedName = generator.string()
-
-      const res = await config.api.rowAction.update(tableId, actionId, {
-        name: updatedName,
-      })
-
-      expect(res).toEqual({
-        id: actionId,
-        tableId,
-        name: updatedName,
-        automationId: actionData.automationId,
-        allowedSources: [tableId],
-      })
-
-      expect(await config.api.rowAction.find(tableId)).toEqual(
-        expect.objectContaining({
-          actions: expect.objectContaining({
-            [actionId]: {
-              name: updatedName,
-              id: actionData.id,
-              tableId: actionData.tableId,
-              automationId: actionData.automationId,
-              allowedSources: [tableId],
-            },
-          }),
-        })
-      )
-    })
-
-    it("trims row action names", async () => {
-      const rowAction = await createRowAction(tableId, createRowActionRequest())
-
-      const res = await config.api.rowAction.update(tableId, rowAction.id, {
-        name: "   action  name  ",
-      })
-
-      expect(res).toEqual(expect.objectContaining({ name: "action  name" }))
-
-      expect(await config.api.rowAction.find(tableId)).toEqual(
-        expect.objectContaining({
-          actions: expect.objectContaining({
-            [rowAction.id]: expect.objectContaining({
-              name: "action  name",
-            }),
-          }),
-        })
-      )
-    })
-
-    it("throws Bad Request when trying to update by a non-existing id", async () => {
-      await createRowAction(tableId, createRowActionRequest())
-
-      await config.api.rowAction.update(
-        tableId,
-        generator.guid(),
-        createRowActionRequest(),
-        { status: 400 }
-      )
-    })
-
-    it("throws Bad Request when trying to update by a via another table id", async () => {
-      const otherTable = await config.api.table.save(
-        setup.structures.basicTable()
-      )
-      await createRowAction(otherTable._id!, createRowActionRequest())
-
-      const action = await createRowAction(tableId, createRowActionRequest())
-      await config.api.rowAction.update(
-        otherTable._id!,
-        action.id,
-        createRowActionRequest(),
-        { status: 400 }
-      )
-    })
-
-    it("can not use existing row action names (for the same table)", async () => {
-      const action1 = await createRowAction(tableId, createRowActionRequest())
-      const action2 = await createRowAction(tableId, createRowActionRequest())
-
-      await config.api.rowAction.update(
-        tableId,
-        action1.id,
-        { name: action2.name },
-        {
-          status: 409,
-          body: {
-            message: "A row action with the same name already exists.",
-          },
-        }
-      )
-    })
-
-    it("does not throw with name conflicts for the same row action", async () => {
-      const action1 = await createRowAction(tableId, createRowActionRequest())
-
-      await config.api.rowAction.update(tableId, action1.id, {
-        name: action1.name,
-      })
-    })
-  })
-
   describe("delete", () => {
     unauthorisedTests((expectations, testConfig) =>
       config.api.rowAction.delete(
diff --git a/packages/server/src/tests/utilities/api/rowAction.ts b/packages/server/src/tests/utilities/api/rowAction.ts
index e18f2ce6b67..5d5ef69b8a9 100644
--- a/packages/server/src/tests/utilities/api/rowAction.ts
+++ b/packages/server/src/tests/utilities/api/rowAction.ts
@@ -41,23 +41,6 @@ export class RowActionAPI extends TestAPI {
     )
   }
 
-  update = async (
-    tableId: string,
-    rowActionId: string,
-    rowAction: CreateRowActionRequest,
-    expectations?: Expectations,
-    config?: { publicUser?: boolean }
-  ) => {
-    return await this._put<RowActionResponse>(
-      `/api/tables/${tableId}/actions/${rowActionId}`,
-      {
-        body: rowAction,
-        expectations,
-        ...config,
-      }
-    )
-  }
-
   delete = async (
     tableId: string,
     rowActionId: string,

From 46a7f15fed6d7965e71deab92d2d95db4556f709 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 13:19:24 +0200
Subject: [PATCH 350/455] Fix name checks

---
 packages/server/src/sdk/app/rowActions.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts
index 6a0a803bd57..de73969696a 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions.ts
@@ -19,12 +19,13 @@ async function ensureUniqueAndThrow(
   existingRowActionId?: string
 ) {
   const names = await getNames(doc)
-  name = name.toLowerCase()
+  name = name.toLowerCase().trim()
 
   if (
     Object.entries(names).find(
       ([automationId, automationName]) =>
-        automationName === name && automationId !== existingRowActionId
+        automationName.toLowerCase().trim() === name &&
+        automationId !== existingRowActionId
     )
   ) {
     throw new HTTPError("A row action with the same name already exists.", 409)

From 63daa6bcde63209bf922039cc323d38037022b1f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 14:08:07 +0200
Subject: [PATCH 351/455] Remove obsolete test

---
 .../sdk/app/automations/tests/index.spec.ts   | 21 +------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/packages/server/src/sdk/app/automations/tests/index.spec.ts b/packages/server/src/sdk/app/automations/tests/index.spec.ts
index bf4c886cf0d..6c70392300e 100644
--- a/packages/server/src/sdk/app/automations/tests/index.spec.ts
+++ b/packages/server/src/sdk/app/automations/tests/index.spec.ts
@@ -1,5 +1,5 @@
 import { sample } from "lodash/fp"
-import { Automation, AutomationTriggerStepId } from "@budibase/types"
+import { Automation } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
 import automationSdk from "../"
@@ -26,25 +26,6 @@ describe("automation sdk", () => {
       })
     })
 
-    it("cannot rename row action automations", async () => {
-      await config.doInContext(config.getAppId(), async () => {
-        const automation = structures.newAutomation({
-          trigger: {
-            ...structures.automationTrigger(),
-            stepId: AutomationTriggerStepId.ROW_ACTION,
-          },
-        })
-
-        const response = await automationSdk.create(automation)
-
-        const newName = generator.guid()
-        const update = { ...response, name: newName }
-        await expect(automationSdk.update(update)).rejects.toThrow(
-          "Row actions cannot be renamed"
-        )
-      })
-    })
-
     it.each([
       ["trigger", (a: Automation) => a.definition.trigger],
       ["step", (a: Automation) => a.definition.steps[0]],

From 612bd5177ccf7f5f1f710da8ce2b1ba4b82e27d2 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 23 Oct 2024 13:32:39 +0100
Subject: [PATCH 352/455] update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 60411963053..f6aebba9445 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 60411963053b98e0415a1e9285b721fc26c628c8
+Subproject commit f6aebba94451ce47bba551926e5ad72bd75f71c6

From ff487735cc4a90e9b7e1cd38e41bc2ab8ebf98f4 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 23 Oct 2024 14:13:43 +0100
Subject: [PATCH 353/455] Update frontend type names to match changes I've
 made.

---
 packages/builder/src/stores/builder/viewsV2.js            | 8 ++++----
 .../frontend-core/src/components/CoreFilterBuilder.svelte | 4 ++--
 .../frontend-core/src/components/grid/stores/filter.js    | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index 7c0a940c1b9..3d67686344b 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -4,7 +4,7 @@ import { API } from "api"
 import { dataFilters } from "@budibase/shared-core"
 
 function convertToSearchFilters(view) {
-  // convert from SearchFilterGroup type
+  // convert from UISearchFilter type
   if (view?.query) {
     return {
       ...view,
@@ -15,7 +15,7 @@ function convertToSearchFilters(view) {
   return view
 }
 
-function convertToSearchFilterGroup(view) {
+function convertToUISearchFilter(view) {
   if (view?.queryUI) {
     return {
       ...view,
@@ -36,7 +36,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
+      list = list.concat(views.map(view => convertToUISearchFilter(view)))
     })
     return {
       ...$store,
@@ -77,7 +77,7 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
-    view = convertToSearchFilterGroup(view)
+    view = convertToUISearchFilter(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId
diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index c711a57e1c5..002d6fadb84 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -10,7 +10,7 @@
   } from "@budibase/bbui"
   import {
     FieldType,
-    FilterGroupLogicalOperator,
+    UILogicalOperator,
     EmptyFilterOption,
   } from "@budibase/types"
   import { QueryUtils, Constants } from "@budibase/frontend-core"
@@ -220,7 +220,7 @@
     } else if (addGroup) {
       if (!editable?.groups?.length) {
         editable = {
-          logicalOperator: FilterGroupLogicalOperator.ALL,
+          logicalOperator: UILogicalOperator.ALL,
           onEmptyFilter: EmptyFilterOption.RETURN_NONE,
           groups: [],
         }
diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js
index 6e6c37da87c..e7adc356ae6 100644
--- a/packages/frontend-core/src/components/grid/stores/filter.js
+++ b/packages/frontend-core/src/components/grid/stores/filter.js
@@ -1,5 +1,5 @@
 import { get, derived } from "svelte/store"
-import { FieldType, FilterGroupLogicalOperator } from "@budibase/types"
+import { FieldType, UILogicalOperator } from "@budibase/types"
 import { memo } from "../../../utils/memo"
 
 export const createStores = context => {
@@ -25,10 +25,10 @@ export const deriveStores = context => {
         return $filter
       }
       let allFilters = {
-        logicalOperator: FilterGroupLogicalOperator.ALL,
+        logicalOperator: UILogicalOperator.ALL,
         groups: [
           {
-            logicalOperator: FilterGroupLogicalOperator.ALL,
+            logicalOperator: UILogicalOperator.ALL,
             filters: $inlineFilters,
           },
         ],

From c6a413f3991d660436f0c2b7deb0191bd47df482 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 23 Oct 2024 14:47:06 +0100
Subject: [PATCH 354/455] Quick fix for role validation, permissionId is no
 longer required.

---
 packages/server/src/api/controllers/role.ts        |  4 ++++
 packages/server/src/api/routes/tests/role.spec.ts  | 13 +++++++++++++
 packages/server/src/api/routes/utils/validators.ts |  2 +-
 packages/types/src/api/web/role.ts                 |  2 +-
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 10477119837..35f221293fb 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -137,6 +137,10 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     permissionId = PermissionLevel.WRITE
   }
 
+  if (!permissionId) {
+    ctx.throw(400, "Role requires permissionId to be specified.")
+  }
+
   const role = new roles.Role(_id, name, permissionId, {
     displayName: uiMetadata?.displayName || name,
     description: uiMetadata?.description || "Custom role",
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index adb83ca7932..e3dfcb40f12 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -58,6 +58,19 @@ describe("/roles", () => {
       })
       expect(res.inherits).toEqual([BUILTIN_ROLE_IDS.BASIC])
     })
+
+    it("save role without permissionId", async () => {
+      const res = await config.api.roles.save(
+        {
+          ...basicRole(),
+          permissionId: undefined,
+        },
+        {
+          status: 200,
+        }
+      )
+      expect(res.permissionId).toEqual(PermissionLevel.WRITE)
+    })
   })
 
   describe("update", () => {
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index 02a1a2d0600..962735ded7f 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -215,7 +215,7 @@ export function roleValidator() {
       // this is the base permission ID (for now a built in)
       permissionId: Joi.string()
         .valid(...Object.values(permissions.BuiltinPermissionID))
-        .required(),
+        .optional(),
       permissions: Joi.object()
         .pattern(
           /.*/,
diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts
index df439e84e7b..aa5dd2f31ba 100644
--- a/packages/types/src/api/web/role.ts
+++ b/packages/types/src/api/web/role.ts
@@ -6,7 +6,7 @@ export interface SaveRoleRequest {
   _rev?: string
   name: string
   inherits?: string | string[]
-  permissionId: string
+  permissionId?: string
   permissions?: Record<string, PermissionLevel[]>
   version?: string
   uiMetadata?: RoleUIMetadata

From 4eda072af412498bc08c2fc36ec2d7d8280d4895 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 14:50:46 +0100
Subject: [PATCH 355/455] Add view name to queryUI error

---
 packages/server/src/sdk/app/views/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
index 8ca5fd1098f..a0194b6d1bf 100644
--- a/packages/server/src/sdk/app/views/utils.ts
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -23,7 +23,7 @@ export function ensureQueryUISet(viewArg: Readonly<ViewV2>): ViewV2 {
       // So despite the type saying that `view.query` is a LegacyFilter[] |
       // SearchFilters, it will never be a SearchFilters when a `view.queryUI`
       // is specified, making it "safe" to throw an error here.
-      throw new HTTPError("view is missing queryUI field", 400)
+      throw new HTTPError(`View ${view.name} is missing queryUI field`, 400)
     }
 
     view.queryUI = utils.processSearchFilters(view.query)

From 77bb3cf4dadb7d865c044e2d4a9ab76c1ab5a4e9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 14:52:54 +0100
Subject: [PATCH 356/455] Increase width of create table modal

---
 .../backend/TableNavigator/modals/CreateTableModal.svelte        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
index b62c8af03d4..8a5484e2b21 100644
--- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
@@ -92,6 +92,7 @@
   disabled={error ||
     !name ||
     (rows.length && (!allValid || displayColumn == null))}
+  size="M"
 >
   <Input
     thin

From cb4b7988c503bbfb9e2290106092f75e48b73fce Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 15:06:57 +0100
Subject: [PATCH 357/455] Update usages of queryUI and ensure
 processSearchFilters isn't called with nullish values

---
 .../SetupPanel/AutomationBlockSetup.svelte    |  2 +-
 .../buttons/TableFilterButton.svelte          |  4 +--
 .../buttons/grid/GridFilterButton.svelte      |  2 ++
 .../builder/src/stores/builder/components.js  |  2 +-
 .../builder/src/stores/builder/viewsV2.js     | 27 +------------------
 .../grid/stores/datasources/viewV2.js         | 18 +++----------
 6 files changed, 10 insertions(+), 45 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index cbac1bf2ffc..28872dcaced 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -798,7 +798,7 @@
         break
       }
     }
-    return utils.processSearchFilters(filters)
+    return utils.processSearchFilters(filters || [])
   }
 
   function saveFilters(key) {
diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index e616e274678..68d2ad44476 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -17,7 +17,7 @@
 
   let drawer
 
-  $: localFilters = utils.processSearchFilters(filters)
+  $: localFilters = utils.processSearchFilters(filters || [])
 
   $: schemaFields = search.getFields(
     $tables.list,
@@ -61,7 +61,7 @@
   title="Filtering"
   on:drawerHide
   on:drawerShow={() => {
-    localFilters = utils.processSearchFilters(filters)
+    localFilters = utils.processSearchFilters(filters || [])
   }}
   forceModal
 >
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
index f927f84876d..ea083add873 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
@@ -7,6 +7,8 @@
   const onFilter = e => {
     filter.set(e.detail || [])
   }
+
+  $: console.log($filter)
 </script>
 
 {#key $datasource}
diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 6c2c438f0c7..166122c8855 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -210,7 +210,7 @@ export class ComponentStore extends BudiStore {
 
       if (isLegacy) {
         const processedSetting = utils.processSearchFilters(
-          enrichedComponent[setting.key]
+          enrichedComponent[setting.key] || []
         )
         enrichedComponent[setting.key] = processedSetting
         migrated = true
diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index 3d67686344b..70af03e75ff 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -3,29 +3,6 @@ import { tables } from "./tables"
 import { API } from "api"
 import { dataFilters } from "@budibase/shared-core"
 
-function convertToSearchFilters(view) {
-  // convert from UISearchFilter type
-  if (view?.query) {
-    return {
-      ...view,
-      queryUI: view.query,
-      query: dataFilters.buildQuery(view.query),
-    }
-  }
-  return view
-}
-
-function convertToUISearchFilter(view) {
-  if (view?.queryUI) {
-    return {
-      ...view,
-      query: view.queryUI,
-      queryUI: undefined,
-    }
-  }
-  return view
-}
-
 export function createViewsV2Store() {
   const store = writable({
     selectedViewId: null,
@@ -36,7 +13,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views.map(view => convertToUISearchFilter(view)))
+      list = list.concat(views)
     })
     return {
       ...$store,
@@ -58,7 +35,6 @@ export function createViewsV2Store() {
   }
 
   const create = async view => {
-    view = convertToSearchFilters(view)
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
     replaceView(savedView.id, savedView)
@@ -66,7 +42,6 @@ export function createViewsV2Store() {
   }
 
   const save = async view => {
-    view = convertToSearchFilters(view)
     const res = await API.viewV2.update(view)
     const savedView = res?.data
     replaceView(view.id, savedView)
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 56a7479b44c..281443cbc32 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -1,25 +1,13 @@
 import { get } from "svelte/store"
 import { dataFilters } from "@budibase/shared-core"
 
-function convertToSearchFilters(view) {
-  // convert from SearchFilterGroup type
-  if (view?.query) {
-    return {
-      ...view,
-      queryUI: view.query,
-      query: dataFilters.buildQuery(view.query),
-    }
-  }
-  return view
-}
-
 const SuppressErrors = true
 
 export const createActions = context => {
   const { API, datasource, columns } = context
 
   const saveDefinition = async newDefinition => {
-    await API.viewV2.update(convertToSearchFilters(newDefinition))
+    await API.viewV2.update(newDefinition)
   }
 
   const saveRow = async row => {
@@ -138,7 +126,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          filter.set($definition.queryUI || $definition.query)
+          filter.set($definition.queryUI)
         }
       })
     )
@@ -197,7 +185,7 @@ export const initialise = context => {
         if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) {
           await datasource.actions.saveDefinition({
             ...$view,
-            query: $filter,
+            queryUI: $filter,
           })
 
           // Refresh data since view definition changed

From de26d151af24450485d3b9a3635f155cccab1364 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 15:07:19 +0100
Subject: [PATCH 358/455] Remove log

---
 .../backend/DataTable/buttons/TableFilterButton.svelte         | 3 ---
 .../backend/DataTable/buttons/grid/GridFilterButton.svelte     | 2 --
 2 files changed, 5 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index 68d2ad44476..5bd1a675b53 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -18,18 +18,15 @@
   let drawer
 
   $: localFilters = utils.processSearchFilters(filters || [])
-
   $: schemaFields = search.getFields(
     $tables.list,
     Object.values(schema || {}),
     { allowLinks: true }
   )
-
   $: filterCount =
     localFilters?.groups?.reduce((acc, group) => {
       return (acc += group.filters.filter(filter => filter.field).length)
     }, 0) || 0
-
   $: bindings = [
     {
       type: "context",
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
index ea083add873..f927f84876d 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
@@ -7,8 +7,6 @@
   const onFilter = e => {
     filter.set(e.detail || [])
   }
-
-  $: console.log($filter)
 </script>
 
 {#key $datasource}

From 8f21802e6ea5a3cf0cea9b06f8276d9b59250277 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 23 Oct 2024 15:08:59 +0100
Subject: [PATCH 359/455] Refactor, correct to the BuiltinPermissionID rather
 than PermissionLevel, these are different.

---
 .../backend-core/src/security/permissions.ts  | 14 +++----
 packages/backend-core/src/security/roles.ts   |  7 ++--
 .../src/security/tests/permissions.spec.ts    |  7 ++--
 packages/server/src/api/controllers/role.ts   |  5 ++-
 .../src/api/routes/tests/application.spec.ts  |  6 +--
 .../src/api/routes/tests/permissions.spec.ts  | 15 +++++--
 .../server/src/api/routes/tests/role.spec.ts  | 41 +++++++++++--------
 .../src/api/routes/tests/screen.spec.ts       |  8 ++--
 .../server/src/api/routes/utils/validators.ts |  3 +-
 .../server/src/tests/utilities/structures.ts  |  5 ++-
 .../src/helpers/tests/roles.spec.ts           |  4 +-
 packages/types/src/api/web/role.ts            |  4 +-
 packages/types/src/documents/app/role.ts      |  4 +-
 packages/types/src/sdk/permissions.ts         |  9 ++++
 .../src/api/routes/global/tests/roles.spec.ts |  6 +--
 15 files changed, 83 insertions(+), 55 deletions(-)

diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts
index 4ed2cd39541..929ae929097 100644
--- a/packages/backend-core/src/security/permissions.ts
+++ b/packages/backend-core/src/security/permissions.ts
@@ -1,4 +1,8 @@
-import { PermissionLevel, PermissionType } from "@budibase/types"
+import {
+  PermissionLevel,
+  PermissionType,
+  BuiltinPermissionID,
+} from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
@@ -57,14 +61,6 @@ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] {
   }
 }
 
-export enum BuiltinPermissionID {
-  PUBLIC = "public",
-  READ_ONLY = "read_only",
-  WRITE = "write",
-  ADMIN = "admin",
-  POWER = "power",
-}
-
 export const BUILTIN_PERMISSIONS: {
   [key in keyof typeof BuiltinPermissionID]: {
     _id: (typeof BuiltinPermissionID)[key]
diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 7d9a11db483..df7f41e6cea 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -1,5 +1,4 @@
 import semver from "semver"
-import { BuiltinPermissionID, PermissionLevel } from "./permissions"
 import {
   prefixRoleID,
   getRoleParams,
@@ -14,6 +13,8 @@ import {
   RoleUIMetadata,
   Database,
   App,
+  BuiltinPermissionID,
+  PermissionLevel,
 } from "@budibase/types"
 import cloneDeep from "lodash/fp/cloneDeep"
 import { RoleColor, helpers } from "@budibase/shared-core"
@@ -50,7 +51,7 @@ export class Role implements RoleDoc {
   _id: string
   _rev?: string
   name: string
-  permissionId: string
+  permissionId: BuiltinPermissionID
   inherits?: string | string[]
   version?: string
   permissions: Record<string, PermissionLevel[]> = {}
@@ -59,7 +60,7 @@ export class Role implements RoleDoc {
   constructor(
     id: string,
     name: string,
-    permissionId: string,
+    permissionId: BuiltinPermissionID,
     uiMetadata?: RoleUIMetadata
   ) {
     this._id = id
diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts
index 39348646fb8..f98833c7cd8 100644
--- a/packages/backend-core/src/security/tests/permissions.spec.ts
+++ b/packages/backend-core/src/security/tests/permissions.spec.ts
@@ -1,6 +1,7 @@
 import cloneDeep from "lodash/cloneDeep"
 import * as permissions from "../permissions"
 import { BUILTIN_ROLE_IDS } from "../roles"
+import { BuiltinPermissionID } from "@budibase/types"
 
 describe("levelToNumber", () => {
   it("should return 0 for EXECUTE", () => {
@@ -77,7 +78,7 @@ describe("doesHaveBasePermission", () => {
     const rolesHierarchy = [
       {
         roleId: BUILTIN_ROLE_IDS.ADMIN,
-        permissionId: permissions.BuiltinPermissionID.ADMIN,
+        permissionId: BuiltinPermissionID.ADMIN,
       },
     ]
     expect(
@@ -91,7 +92,7 @@ describe("doesHaveBasePermission", () => {
     const rolesHierarchy = [
       {
         roleId: BUILTIN_ROLE_IDS.PUBLIC,
-        permissionId: permissions.BuiltinPermissionID.PUBLIC,
+        permissionId: BuiltinPermissionID.PUBLIC,
       },
     ]
     expect(
@@ -129,7 +130,7 @@ describe("getBuiltinPermissions", () => {
 describe("getBuiltinPermissionByID", () => {
   it("returns correct permission object for valid ID", () => {
     const expectedPermission = {
-      _id: permissions.BuiltinPermissionID.PUBLIC,
+      _id: BuiltinPermissionID.PUBLIC,
       name: "Public",
       permissions: [
         new permissions.Permission(
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 35f221293fb..f8af71fe5cf 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -19,6 +19,7 @@ import {
   UserMetadata,
   DocumentType,
   PermissionLevel,
+  BuiltinPermissionID,
 } from "@budibase/types"
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
@@ -134,7 +135,9 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   }
   // assume write permission level for newly created roles
   if (isCreate && !permissionId) {
-    permissionId = PermissionLevel.WRITE
+    permissionId = BuiltinPermissionID.WRITE
+  } else if (!permissionId && dbRole?.permissionId) {
+    permissionId = dbRole.permissionId
   }
 
   if (!permissionId) {
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index 47b6776610a..729f8993799 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -16,7 +16,7 @@ import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
 import { events, utils, context, features } from "@budibase/backend-core"
 import env from "../../../environment"
-import { type App } from "@budibase/types"
+import { type App, BuiltinPermissionID } from "@budibase/types"
 import tk from "timekeeper"
 import * as uuid from "uuid"
 import { structures } from "@budibase/backend-core/tests"
@@ -80,7 +80,7 @@ describe("/applications", () => {
       const role = await config.api.roles.save({
         name: "Test",
         inherits: "PUBLIC",
-        permissionId: "read_only",
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
 
@@ -112,7 +112,7 @@ describe("/applications", () => {
       const role = await config.api.roles.save({
         name: roleName,
         inherits: "PUBLIC",
-        permissionId: "read_only",
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
 
diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 7a9bb2f373c..193d018c248 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -1,5 +1,12 @@
 import { roles } from "@budibase/backend-core"
-import { Document, PermissionLevel, Role, Row, Table } from "@budibase/types"
+import {
+  BuiltinPermissionID,
+  Document,
+  PermissionLevel,
+  Role,
+  Row,
+  Table,
+} from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
@@ -304,7 +311,7 @@ describe("/permission", () => {
       role1 = await config.api.roles.save(
         {
           name: "test_1",
-          permissionId: PermissionLevel.WRITE,
+          permissionId: BuiltinPermissionID.WRITE,
           inherits: BUILTIN_ROLE_IDS.BASIC,
         },
         { status: 200 }
@@ -312,7 +319,7 @@ describe("/permission", () => {
       role2 = await config.api.roles.save(
         {
           name: "test_2",
-          permissionId: PermissionLevel.WRITE,
+          permissionId: BuiltinPermissionID.WRITE,
           inherits: BUILTIN_ROLE_IDS.BASIC,
         },
         { status: 200 }
@@ -345,7 +352,7 @@ describe("/permission", () => {
     it("should be able to fetch two tables, with different roles, using multi-inheritance", async () => {
       const role3 = await config.api.roles.save({
         name: "role3",
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
         inherits: [role1._id!, role2._id!],
       })
 
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index e3dfcb40f12..1eb59c5a0c8 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -1,15 +1,9 @@
-import {
-  roles,
-  events,
-  permissions,
-  db as dbCore,
-} from "@budibase/backend-core"
+import { roles, events, db as dbCore } from "@budibase/backend-core"
 import * as setup from "./utilities"
-import { PermissionLevel } from "@budibase/types"
+import { PermissionLevel, BuiltinPermissionID } from "@budibase/types"
 
 const { basicRole } = setup.structures
 const { BUILTIN_ROLE_IDS } = roles
-const { BuiltinPermissionID } = permissions
 
 const LOOP_ERROR = "Role inheritance contains a loop, this is not supported"
 
@@ -162,7 +156,7 @@ describe("/roles", () => {
         _id: id1,
         name: id1,
         permissions: {},
-        permissionId: "write",
+        permissionId: BuiltinPermissionID.WRITE,
         version: "name",
         inherits: ["POWER"],
       })
@@ -170,7 +164,7 @@ describe("/roles", () => {
         _id: id2,
         permissions: {},
         name: id2,
-        permissionId: "write",
+        permissionId: BuiltinPermissionID.WRITE,
         version: "name",
         inherits: [id1],
       })
@@ -189,10 +183,25 @@ describe("/roles", () => {
         inherits: [BUILTIN_ROLE_IDS.ADMIN],
       })
       // remove the roles so that it will default back to DB roles, then save again
-      delete res.inherits
-      const updatedRes = await config.api.roles.save(res)
+      const updatedRes = await config.api.roles.save({
+        ...res,
+        inherits: undefined,
+      })
       expect(updatedRes.inherits).toEqual([BUILTIN_ROLE_IDS.ADMIN])
     })
+
+    it("handle updating a role, without its permissionId", async () => {
+      const res = await config.api.roles.save({
+        ...basicRole(),
+        permissionId: BuiltinPermissionID.READ_ONLY,
+      })
+      // permission ID can be removed during update
+      const updatedRes = await config.api.roles.save({
+        ...res,
+        permissionId: undefined,
+      })
+      expect(updatedRes.permissionId).toEqual(PermissionLevel.READ)
+    })
   })
 
   describe("fetch", () => {
@@ -329,7 +338,7 @@ describe("/roles", () => {
       await config.api.roles.save({
         name: customRoleName,
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
       await config.withHeaders(
@@ -369,19 +378,19 @@ describe("/roles", () => {
       const { _id: roleId1 } = await config.api.roles.save({
         name: role1,
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: permissions.BuiltinPermissionID.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
         version: "name",
       })
       const { _id: roleId2 } = await config.api.roles.save({
         name: role2,
         inherits: roles.BUILTIN_ROLE_IDS.POWER,
-        permissionId: permissions.BuiltinPermissionID.POWER,
+        permissionId: BuiltinPermissionID.POWER,
         version: "name",
       })
       await config.api.roles.save({
         name: role3,
         inherits: [roleId1!, roleId2!],
-        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
       const headers = await config.roleHeaders({
diff --git a/packages/server/src/api/routes/tests/screen.spec.ts b/packages/server/src/api/routes/tests/screen.spec.ts
index 894710ca272..82a550f2fdc 100644
--- a/packages/server/src/api/routes/tests/screen.spec.ts
+++ b/packages/server/src/api/routes/tests/screen.spec.ts
@@ -1,7 +1,7 @@
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { events, roles } from "@budibase/backend-core"
-import { Screen, PermissionLevel, Role } from "@budibase/types"
+import { Screen, Role, BuiltinPermissionID } from "@budibase/types"
 
 const { basicScreen } = setup.structures
 
@@ -40,17 +40,17 @@ describe("/screens", () => {
       role1 = await config.api.roles.save({
         name: "role1",
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
       })
       role2 = await config.api.roles.save({
         name: "role2",
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
       })
       multiRole = await config.api.roles.save({
         name: "multiRole",
         inherits: [role1._id!, role2._id!],
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
       })
       screen1 = await config.api.screen.save(
         {
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index 962735ded7f..862d8c30c5c 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -8,6 +8,7 @@ import {
   SearchFilters,
   Table,
   WebhookActionType,
+  BuiltinPermissionID,
 } from "@budibase/types"
 import Joi, { CustomValidator } from "joi"
 import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
@@ -214,7 +215,7 @@ export function roleValidator() {
       }).optional(),
       // this is the base permission ID (for now a built in)
       permissionId: Joi.string()
-        .valid(...Object.values(permissions.BuiltinPermissionID))
+        .valid(...Object.values(BuiltinPermissionID))
         .optional(),
       permissions: Joi.object()
         .pattern(
diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts
index b63d6d1a78b..49046d9eda0 100644
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@@ -1,4 +1,4 @@
-import { permissions, roles, utils } from "@budibase/backend-core"
+import { roles, utils } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
@@ -33,6 +33,7 @@ import {
   TableSourceType,
   Webhook,
   WebhookActionType,
+  BuiltinPermissionID,
 } from "@budibase/types"
 import { LoopInput } from "../../definitions/automations"
 import { merge } from "lodash"
@@ -515,7 +516,7 @@ export function basicRole(): Role {
   return {
     name: `NewRole_${utils.newid()}`,
     inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-    permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+    permissionId: BuiltinPermissionID.WRITE,
     permissions: {},
     version: "name",
   }
diff --git a/packages/shared-core/src/helpers/tests/roles.spec.ts b/packages/shared-core/src/helpers/tests/roles.spec.ts
index 051a196a5bc..003b37f8f24 100644
--- a/packages/shared-core/src/helpers/tests/roles.spec.ts
+++ b/packages/shared-core/src/helpers/tests/roles.spec.ts
@@ -1,5 +1,5 @@
 import { checkForRoleInheritanceLoops } from "../roles"
-import { Role } from "@budibase/types"
+import { BuiltinPermissionID, Role } from "@budibase/types"
 
 /**
  * This unit test exists as this utility will be used in the frontend and backend, confirmation
@@ -19,7 +19,7 @@ function role(id: string, inherits: string | string[]): TestRole {
     _id: id,
     inherits: inherits,
     name: "ROLE",
-    permissionId: "PERMISSION",
+    permissionId: BuiltinPermissionID.WRITE,
     permissions: {}, // not needed for this test
   }
   allRoles.push(role)
diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts
index aa5dd2f31ba..f1c573c8494 100644
--- a/packages/types/src/api/web/role.ts
+++ b/packages/types/src/api/web/role.ts
@@ -1,12 +1,12 @@
 import { Role, RoleUIMetadata } from "../../documents"
-import { PermissionLevel } from "../../sdk"
+import { PermissionLevel, BuiltinPermissionID } from "../../sdk"
 
 export interface SaveRoleRequest {
   _id?: string
   _rev?: string
   name: string
   inherits?: string | string[]
-  permissionId?: string
+  permissionId?: BuiltinPermissionID
   permissions?: Record<string, PermissionLevel[]>
   version?: string
   uiMetadata?: RoleUIMetadata
diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts
index 22f4ab9cd38..b624a5c6175 100644
--- a/packages/types/src/documents/app/role.ts
+++ b/packages/types/src/documents/app/role.ts
@@ -1,5 +1,5 @@
 import { Document } from "../document"
-import { PermissionLevel } from "../../sdk"
+import { PermissionLevel, BuiltinPermissionID } from "../../sdk"
 
 export interface RoleUIMetadata {
   displayName?: string
@@ -8,7 +8,7 @@ export interface RoleUIMetadata {
 }
 
 export interface Role extends Document {
-  permissionId: string
+  permissionId: BuiltinPermissionID
   inherits?: string | string[]
   permissions: Record<string, PermissionLevel[]>
   version?: string
diff --git a/packages/types/src/sdk/permissions.ts b/packages/types/src/sdk/permissions.ts
index 638de97f483..d27876156ea 100644
--- a/packages/types/src/sdk/permissions.ts
+++ b/packages/types/src/sdk/permissions.ts
@@ -5,6 +5,15 @@ export enum PermissionLevel {
   ADMIN = "admin",
 }
 
+// used within the role, specifies base permissions
+export enum BuiltinPermissionID {
+  PUBLIC = "public",
+  READ_ONLY = "read_only",
+  WRITE = "write",
+  ADMIN = "admin",
+  POWER = "power",
+}
+
 // these are the global types, that govern the underlying default behaviour
 export enum PermissionType {
   APP = "app",
diff --git a/packages/worker/src/api/routes/global/tests/roles.spec.ts b/packages/worker/src/api/routes/global/tests/roles.spec.ts
index 28977d85215..4e7acb2741d 100644
--- a/packages/worker/src/api/routes/global/tests/roles.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts
@@ -1,6 +1,6 @@
 import { structures, TestConfiguration } from "../../../../tests"
-import { context, db, permissions, roles } from "@budibase/backend-core"
-import { App, Database } from "@budibase/types"
+import { context, db, roles } from "@budibase/backend-core"
+import { App, Database, BuiltinPermissionID } from "@budibase/types"
 
 jest.mock("@budibase/backend-core", () => {
   const core = jest.requireActual("@budibase/backend-core")
@@ -44,7 +44,7 @@ describe("/api/global/roles", () => {
   const role = new roles.Role(
     db.generateRoleID(ROLE_NAME),
     ROLE_NAME,
-    permissions.BuiltinPermissionID.READ_ONLY,
+    BuiltinPermissionID.READ_ONLY,
     { displayName: roles.BUILTIN_ROLE_IDS.BASIC }
   )
 

From ca0dc030ade5f1c941a66437e2f228bdda73d546 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 23 Oct 2024 15:08:59 +0100
Subject: [PATCH 360/455] Make sure processSearchFilters handles an undefined
 input.

---
 packages/shared-core/src/utils.ts | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 5db4ead1dc7..8e22c10c15d 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -138,9 +138,16 @@ export function isSupportedUserSearch(query: SearchFilters) {
   return true
 }
 
-export const processSearchFilters = (
+export function processSearchFilters(filterArray: undefined): undefined
+export function processSearchFilters(
   filterArray: LegacyFilter[]
-): Required<UISearchFilter> => {
+): Required<UISearchFilter>
+export function processSearchFilters(
+  filterArray?: LegacyFilter[]
+): Required<UISearchFilter> | undefined {
+  if (!filterArray) {
+    return undefined
+  }
   const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray)
   return {
     logicalOperator: UILogicalOperator.ALL,

From 3ddd8e10cc4e026710b5f1d6ab2b1272d46f9d79 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 23 Oct 2024 15:10:53 +0100
Subject: [PATCH 361/455] Comment and test case.

---
 packages/server/src/api/routes/tests/role.spec.ts | 2 +-
 packages/types/src/sdk/permissions.ts             | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 1eb59c5a0c8..a4e672e961d 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -200,7 +200,7 @@ describe("/roles", () => {
         ...res,
         permissionId: undefined,
       })
-      expect(updatedRes.permissionId).toEqual(PermissionLevel.READ)
+      expect(updatedRes.permissionId).toEqual(BuiltinPermissionID.READ_ONLY)
     })
   })
 
diff --git a/packages/types/src/sdk/permissions.ts b/packages/types/src/sdk/permissions.ts
index d27876156ea..bb36af4170c 100644
--- a/packages/types/src/sdk/permissions.ts
+++ b/packages/types/src/sdk/permissions.ts
@@ -1,3 +1,5 @@
+// used in resource permissions - permissions can be at one of these levels
+// endpoints will set what type of permission they require (e.g. searching requires READ)
 export enum PermissionLevel {
   READ = "read",
   WRITE = "write",

From 9ebd890c04902dcb2d810c307e46cef004ef8980 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 23 Oct 2024 15:13:58 +0100
Subject: [PATCH 362/455] Handle empty arrays too.

---
 packages/shared-core/src/utils.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 8e22c10c15d..01afbc25391 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -135,17 +135,19 @@ export function isSupportedUserSearch(query: SearchFilters) {
       return false
     }
   }
+
   return true
 }
 
 export function processSearchFilters(filterArray: undefined): undefined
+export function processSearchFilters(filterArray: []): undefined
 export function processSearchFilters(
   filterArray: LegacyFilter[]
 ): Required<UISearchFilter>
 export function processSearchFilters(
   filterArray?: LegacyFilter[]
 ): Required<UISearchFilter> | undefined {
-  if (!filterArray) {
+  if (!filterArray || filterArray.length === 0) {
     return undefined
   }
   const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray)

From e28357c45120c01025da3fb2ab0b381f76ad4a1c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 23 Oct 2024 15:21:37 +0100
Subject: [PATCH 363/455] Linting.

---
 packages/server/src/api/controllers/role.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index f8af71fe5cf..7f5ac6b92f8 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -18,7 +18,6 @@ import {
   UserCtx,
   UserMetadata,
   DocumentType,
-  PermissionLevel,
   BuiltinPermissionID,
 } from "@budibase/types"
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"

From f2a844c3b5606e39257f64f86ce37e0e3ed20626 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 23 Oct 2024 15:24:32 +0100
Subject: [PATCH 364/455] Fix test.

---
 packages/shared-core/src/filters.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index e0703321e82..e6cab22911e 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -467,7 +467,7 @@ export function buildQuery(
   }
 
   if (Array.isArray(filter)) {
-    filter = processSearchFilters(filter)
+    filter = processSearchFilters(filter) ?? []
   }
 
   const operator = logicalOperatorFromUI(

From a24728394f7b5e4ce1a2a514c0403eb2b4790df5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 16:26:14 +0200
Subject: [PATCH 365/455] Trigger info data in the frontend

---
 .../AutomationBuilder/FlowChart/FlowItem.svelte  | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
index aca3e950c37..873e769e217 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte
@@ -3,7 +3,7 @@
     automationStore,
     selectedAutomation,
     permissions,
-    selectedAutomationDisplayData,
+    tables,
   } from "stores/builder"
   import {
     Icon,
@@ -17,6 +17,7 @@
     AbsTooltip,
     InlineAlert,
   } from "@budibase/bbui"
+  import { sdk } from "@budibase/shared-core"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import ActionModal from "./ActionModal.svelte"
@@ -51,7 +52,12 @@
   $: isAppAction && setPermissions(role)
   $: isAppAction && getPermissions(automationId)
 
-  $: triggerInfo = $selectedAutomationDisplayData?.triggerInfo
+  $: triggerInfo = sdk.automations.isRowAction($selectedAutomation) && {
+    title: "Automation trigger",
+    tableName: $tables.list.find(
+      x => x._id === $selectedAutomation.definition.trigger.inputs?.tableId
+    )?.name,
+  }
 
   async function setPermissions(role) {
     if (!role || !automationId) {
@@ -187,10 +193,10 @@
           {block}
           {webhookModal}
         />
-        {#if isTrigger && triggerInfo}
+        {#if triggerInfo}
           <InlineAlert
-            header={triggerInfo.type}
-            message={`This trigger is tied to the "${triggerInfo.rowAction.name}" row action in your ${triggerInfo.table.name} table`}
+            header={triggerInfo.title}
+            message={`This trigger is tied to your "${triggerInfo.tableName}" table`}
           />
         {/if}
         {#if lastStep}

From 8ebb02eb7976b0a88a847d28509b32a782888473 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 23 Oct 2024 15:30:46 +0100
Subject: [PATCH 366/455] Test fix.

---
 packages/server/src/api/routes/tests/role.spec.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index a4e672e961d..ee5ba7840d8 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -232,9 +232,7 @@ describe("/roles", () => {
       const customRoleFetched = res.find(r => r._id === customRole.name)
       expect(customRoleFetched).toBeDefined()
       expect(customRoleFetched!.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC)
-      expect(customRoleFetched!.permissionId).toEqual(
-        BuiltinPermissionID.READ_ONLY
-      )
+      expect(customRoleFetched!.permissionId).toEqual(BuiltinPermissionID.WRITE)
     })
 
     it("should be able to get the role with a permission added", async () => {

From f7b84ca7eab2d9be82ff84b9e620fc8318b96aec Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 23 Oct 2024 15:32:07 +0100
Subject: [PATCH 367/455] Fix some type skullduggery.

---
 packages/shared-core/src/filters.ts | 13 ++++++-------
 packages/shared-core/src/utils.ts   |  5 -----
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index e6cab22911e..003e6b5082e 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -455,19 +455,18 @@ export function splitFiltersArray(filters: LegacyFilter[]) {
  * Legacy support remains for the old **SearchFilter[]** format.
  * These will be migrated to an appropriate **SearchFilters** object, if encountered
  */
-export function buildQuery(filter: undefined): undefined
-export function buildQuery(
-  filter: UISearchFilter | LegacyFilter[]
-): SearchFilters
 export function buildQuery(
   filter?: UISearchFilter | LegacyFilter[]
-): SearchFilters | undefined {
+): SearchFilters {
   if (!filter) {
-    return
+    return {}
   }
 
   if (Array.isArray(filter)) {
-    filter = processSearchFilters(filter) ?? []
+    filter = processSearchFilters(filter)
+    if (!filter) {
+      return {}
+    }
   }
 
   const operator = logicalOperatorFromUI(
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 01afbc25391..0e49db9c7c2 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -139,11 +139,6 @@ export function isSupportedUserSearch(query: SearchFilters) {
   return true
 }
 
-export function processSearchFilters(filterArray: undefined): undefined
-export function processSearchFilters(filterArray: []): undefined
-export function processSearchFilters(
-  filterArray: LegacyFilter[]
-): Required<UISearchFilter>
 export function processSearchFilters(
   filterArray?: LegacyFilter[]
 ): Required<UISearchFilter> | undefined {

From 7901c2d4ac9cad004b812d57652b35d44ea4ae81 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 16:37:46 +0200
Subject: [PATCH 368/455] Remove builderData usage

---
 .../AutomationPanel/AutomationNavItem.svelte       |  2 +-
 .../AutomationPanel/AutomationPanel.svelte         | 10 ++--------
 packages/builder/src/stores/builder/automations.js | 14 --------------
 packages/builder/src/stores/builder/index.js       |  2 --
 4 files changed, 3 insertions(+), 25 deletions(-)

diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
index 50814897b48..4e9ca5fd53e 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte
@@ -112,7 +112,7 @@
   iconColor={automation.disabled
     ? "var(--spectrum-global-color-gray-600)"
     : "var(--spectrum-global-color-gray-900)"}
-  text={automation.displayName}
+  text={automation.name}
   selected={automation._id === $selectedAutomation?._id}
   hovering={automation._id === $contextMenuStore.id}
   on:click={() => automationStore.actions.select(automation._id)}
diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
index 6b96c4ebf5b..a26efdf2430 100644
--- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
@@ -25,15 +25,9 @@
         automation.name.toLowerCase().includes(searchString.toLowerCase())
       )
     })
-    .map(automation => ({
-      ...automation,
-      displayName:
-        $automationStore.automationDisplayData[automation._id]?.displayName ||
-        automation.name,
-    }))
     .sort((a, b) => {
-      const lowerA = a.displayName.toLowerCase()
-      const lowerB = b.displayName.toLowerCase()
+      const lowerA = a.name.toLowerCase()
+      const lowerB = b.name.toLowerCase()
       return lowerA > lowerB ? 1 : -1
     })
 
diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js
index b8765ad1e79..bb0f5d20857 100644
--- a/packages/builder/src/stores/builder/automations.js
+++ b/packages/builder/src/stores/builder/automations.js
@@ -23,7 +23,6 @@ const initialAutomationState = {
     ACTION: {},
   },
   selectedAutomationId: null,
-  automationDisplayData: {},
 }
 
 // If this functions, remove the actions elements
@@ -91,7 +90,6 @@ const automationActions = store => ({
       state.automations.sort((a, b) => {
         return a.name < b.name ? -1 : 1
       })
-      state.automationDisplayData = automationResponse.builderData
       state.blockDefinitions = getFinalDefinitions(
         definitions.trigger,
         definitions.action
@@ -153,8 +151,6 @@ const automationActions = store => ({
         state.selectedAutomationId = state.automations[0]?._id || null
       }
 
-      // Clear out automationDisplayData for the automation
-      delete state.automationDisplayData[automation._id]
       return state
     })
   },
@@ -432,13 +428,3 @@ export const selectedAutomation = derived(automationStore, $automationStore => {
     x => x._id === $automationStore.selectedAutomationId
   )
 })
-
-export const selectedAutomationDisplayData = derived(
-  [automationStore, selectedAutomation],
-  ([$automationStore, $selectedAutomation]) => {
-    if (!$selectedAutomation?._id) {
-      return null
-    }
-    return $automationStore.automationDisplayData[$selectedAutomation._id]
-  }
-)
diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js
index dbde7399518..158cd29973c 100644
--- a/packages/builder/src/stores/builder/index.js
+++ b/packages/builder/src/stores/builder/index.js
@@ -11,7 +11,6 @@ import {
   automationStore,
   selectedAutomation,
   automationHistoryStore,
-  selectedAutomationDisplayData,
 } from "./automations.js"
 import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
 import { deploymentStore } from "./deployments.js"
@@ -46,7 +45,6 @@ export {
   previewStore,
   automationStore,
   selectedAutomation,
-  selectedAutomationDisplayData,
   automationHistoryStore,
   sortedScreens,
   userStore,

From 11b2e40836a77a8d2f76a3945a35984c37734a4b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 16:37:55 +0200
Subject: [PATCH 369/455] Remove

---
 .../server/src/api/controllers/automation.ts     |  8 --------
 packages/types/src/api/web/automation.ts         | 16 ----------------
 2 files changed, 24 deletions(-)

diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts
index b19218647b4..d8bc9d6b21f 100644
--- a/packages/server/src/api/controllers/automation.ts
+++ b/packages/server/src/api/controllers/automation.ts
@@ -76,16 +76,8 @@ export async function update(ctx: UserCtx) {
 }
 
 export async function fetch(ctx: UserCtx<void, FetchAutomationResponse>) {
-  const query: { enrich?: string } = ctx.request.query || {}
-  const enrich = query.enrich === "true"
-
   const automations = await sdk.automations.fetch()
   ctx.body = { automations }
-  if (enrich) {
-    ctx.body.builderData = await sdk.automations.utils.getBuilderData(
-      automations
-    )
-  }
 }
 
 export async function find(ctx: UserCtx) {
diff --git a/packages/types/src/api/web/automation.ts b/packages/types/src/api/web/automation.ts
index 6c810d5e781..06080fc6678 100644
--- a/packages/types/src/api/web/automation.ts
+++ b/packages/types/src/api/web/automation.ts
@@ -3,22 +3,6 @@ import { Automation } from "../../documents"
 
 export interface DeleteAutomationResponse extends DocumentDestroyResponse {}
 
-export interface AutomationBuilderData {
-  displayName: string
-  triggerInfo?: {
-    type: string
-    table: {
-      id: string
-      name: string
-    }
-    rowAction: {
-      id: string
-      name: string
-    }
-  }
-}
-
 export interface FetchAutomationResponse {
   automations: Automation[]
-  builderData?: Record<string, AutomationBuilderData> // The key will be the automationId
 }

From e67352d51bf0f65600964393c6625d3e6eeaee25 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 16:38:07 +0200
Subject: [PATCH 370/455] Remove api flag

---
 packages/builder/src/stores/builder/automations.js | 2 +-
 packages/frontend-core/src/api/automations.js      | 9 ++-------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js
index bb0f5d20857..3ab81ccce6f 100644
--- a/packages/builder/src/stores/builder/automations.js
+++ b/packages/builder/src/stores/builder/automations.js
@@ -82,7 +82,7 @@ const automationActions = store => ({
   },
   fetch: async () => {
     const [automationResponse, definitions] = await Promise.all([
-      API.getAutomations({ enrich: true }),
+      API.getAutomations(),
       API.getAutomationDefinitions(),
     ])
     store.update(state => {
diff --git a/packages/frontend-core/src/api/automations.js b/packages/frontend-core/src/api/automations.js
index ce5f7d3aba1..37a834cf04b 100644
--- a/packages/frontend-core/src/api/automations.js
+++ b/packages/frontend-core/src/api/automations.js
@@ -26,14 +26,9 @@ export const buildAutomationEndpoints = API => ({
   /**
    * Gets a list of all automations.
    */
-  getAutomations: async ({ enrich }) => {
-    const params = new URLSearchParams()
-    if (enrich) {
-      params.set("enrich", true)
-    }
-
+  getAutomations: async () => {
     return await API.get({
-      url: `/api/automations?${params.toString()}`,
+      url: "/api/automations",
     })
   },
 

From 998d0002b8edf8abdcfdad4a9eeaca06665ec871 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 23 Oct 2024 16:48:22 +0200
Subject: [PATCH 371/455] Cleanup dead code

---
 .../server/src/sdk/app/automations/utils.ts   | 51 +------------------
 1 file changed, 1 insertion(+), 50 deletions(-)

diff --git a/packages/server/src/sdk/app/automations/utils.ts b/packages/server/src/sdk/app/automations/utils.ts
index f5ae8bf9484..79e70923f12 100644
--- a/packages/server/src/sdk/app/automations/utils.ts
+++ b/packages/server/src/sdk/app/automations/utils.ts
@@ -1,56 +1,7 @@
-import {
-  Automation,
-  AutomationActionStepId,
-  AutomationBuilderData,
-} from "@budibase/types"
-import { sdk as coreSdk } from "@budibase/shared-core"
-import sdk from "../../../sdk"
+import { Automation, AutomationActionStepId } from "@budibase/types"
 
 export function checkForCollectStep(automation: Automation) {
   return automation.definition.steps.some(
     (step: any) => step.stepId === AutomationActionStepId.COLLECT
   )
 }
-
-export async function getBuilderData(
-  automations: Automation[]
-): Promise<Record<string, AutomationBuilderData>> {
-  const tableNameCache: Record<string, string> = {}
-  async function getTableName(tableId: string) {
-    if (!tableNameCache[tableId]) {
-      const table = await sdk.tables.getTable(tableId)
-      tableNameCache[tableId] = table.name
-    }
-
-    return tableNameCache[tableId]
-  }
-
-  const result: Record<string, AutomationBuilderData> = {}
-  for (const automation of automations) {
-    const isRowAction = coreSdk.automations.isRowAction(automation)
-    if (!isRowAction) {
-      result[automation._id!] = { displayName: automation.name }
-      continue
-    }
-
-    const { tableId, rowActionId } = automation.definition.trigger.inputs
-    if (!tableId || !rowActionId) {
-      continue
-    }
-
-    const tableName = await getTableName(tableId)
-    const rowActionName = automation.name
-    result[automation._id!] = {
-      displayName: rowActionName,
-      triggerInfo: {
-        type: "Automation trigger",
-        table: { id: tableId, name: tableName },
-        rowAction: {
-          id: rowActionId,
-          name: rowActionName,
-        },
-      },
-    }
-  }
-  return result
-}

From 6ac6db706dff2378ae13067a936db62e2780ea9c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 16:10:57 +0100
Subject: [PATCH 372/455] Revert default param

---
 .../automation/SetupPanel/AutomationBlockSetup.svelte         | 2 +-
 .../backend/DataTable/buttons/TableFilterButton.svelte        | 4 ++--
 packages/builder/src/stores/builder/components.js             | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index 28872dcaced..cbac1bf2ffc 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -798,7 +798,7 @@
         break
       }
     }
-    return utils.processSearchFilters(filters || [])
+    return utils.processSearchFilters(filters)
   }
 
   function saveFilters(key) {
diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index 5bd1a675b53..74b7d06ea59 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -17,7 +17,7 @@
 
   let drawer
 
-  $: localFilters = utils.processSearchFilters(filters || [])
+  $: localFilters = utils.processSearchFilters(filters)
   $: schemaFields = search.getFields(
     $tables.list,
     Object.values(schema || {}),
@@ -58,7 +58,7 @@
   title="Filtering"
   on:drawerHide
   on:drawerShow={() => {
-    localFilters = utils.processSearchFilters(filters || [])
+    localFilters = utils.processSearchFilters(filters)
   }}
   forceModal
 >
diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 166122c8855..6c2c438f0c7 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -210,7 +210,7 @@ export class ComponentStore extends BudiStore {
 
       if (isLegacy) {
         const processedSetting = utils.processSearchFilters(
-          enrichedComponent[setting.key] || []
+          enrichedComponent[setting.key]
         )
         enrichedComponent[setting.key] = processedSetting
         migrated = true

From b6dbe542361874ebdefa3bc6687b2fc74d0b3ac3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 16:12:21 +0100
Subject: [PATCH 373/455] Remove old reference to deleted code

---
 packages/builder/src/stores/builder/viewsV2.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index 70af03e75ff..43c94770ab3 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -52,7 +52,6 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
-    view = convertToUISearchFilter(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId

From c8c5cbe16ccbac9123009c656a8bbc4ed3cedada Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 16:24:02 +0100
Subject: [PATCH 374/455] Update more usages of processing search filters

---
 .../SetupPanel/AutomationBlockSetup.svelte         |  4 +++-
 .../DataTable/buttons/TableFilterButton.svelte     | 13 ++-----------
 packages/builder/src/stores/builder/components.js  |  1 -
 .../src/components/CoreFilterBuilder.svelte        | 14 ++++++++++++--
 .../components/grid/stores/datasources/viewV2.js   |  1 +
 5 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index cbac1bf2ffc..b422154290d 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -798,7 +798,9 @@
         break
       }
     }
-    return utils.processSearchFilters(filters)
+    return Array.isArray(filters)
+      ? utils.processSearchFilters(filters)
+      : filters
   }
 
   function saveFilters(key) {
diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index 74b7d06ea59..ca4c76c9688 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -5,7 +5,6 @@
   import { getUserBindings } from "dataBinding"
   import { makePropSafe } from "@budibase/string-templates"
   import { search } from "@budibase/frontend-core"
-  import { utils } from "@budibase/shared-core"
   import { tables } from "stores/builder"
 
   export let schema
@@ -17,7 +16,7 @@
 
   let drawer
 
-  $: localFilters = utils.processSearchFilters(filters)
+  $: localFilters = filters
   $: schemaFields = search.getFields(
     $tables.list,
     Object.values(schema || {}),
@@ -53,15 +52,7 @@
   {filterCount ? `Filter: ${filterCount}` : "Filter"}
 </ActionButton>
 
-<Drawer
-  bind:this={drawer}
-  title="Filtering"
-  on:drawerHide
-  on:drawerShow={() => {
-    localFilters = utils.processSearchFilters(filters)
-  }}
-  forceModal
->
+<Drawer bind:this={drawer} title="Filtering" on:drawerHide forceModal>
   <Button
     cta
     slot="buttons"
diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index 6c2c438f0c7..faa6a086cad 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -207,7 +207,6 @@ export class ComponentStore extends BudiStore {
     )
     for (let setting of filterableTypes || []) {
       const isLegacy = Array.isArray(enrichedComponent[setting.key])
-
       if (isLegacy) {
         const processedSetting = utils.processSearchFilters(
           enrichedComponent[setting.key]
diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index 002d6fadb84..739426150cf 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -16,6 +16,7 @@
   import { QueryUtils, Constants } from "@budibase/frontend-core"
   import { getContext, createEventDispatcher } from "svelte"
   import FilterField from "./FilterField.svelte"
+  import { utils } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
   const {
@@ -39,8 +40,7 @@
   export let toReadable
   export let toRuntime
 
-  $: editableFilters = filters ? Helpers.cloneDeep(filters) : null
-
+  $: editableFilters = migrateFilters(filters)
   $: {
     if (
       tables.find(
@@ -54,6 +54,16 @@
     }
   }
 
+  // We still may need to migrate this even though the backend does it automatically nmow
+  // for query definitions. This is because we might be editing saved filter definitions
+  // from old screens, which will still be of type LegacyFilter[]
+  const migrateFilters = filters => {
+    if (Array.isArray(filters)) {
+      return utils.processSearchFilters(filters)
+    }
+    return Helpers.cloneDeep(filters)
+  }
+
   const filterOperatorOptions = Object.values(FilterOperator).map(entry => {
     return { value: entry, label: Helpers.capitalise(entry) }
   })
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 281443cbc32..3047b474987 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -126,6 +126,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
+          console.log("setting to", $definition.queryUI)
           filter.set($definition.queryUI)
         }
       })

From 13d0c0f7341fb94c3c4a1b7bc9a9207d1907dfcd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 23 Oct 2024 16:24:24 +0100
Subject: [PATCH 375/455] Remove log

---
 .../src/components/grid/stores/datasources/viewV2.js             | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 3047b474987..281443cbc32 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -126,7 +126,6 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          console.log("setting to", $definition.queryUI)
           filter.set($definition.queryUI)
         }
       })

From 9c7e3cd95e05ceb735c70bc21c82c2c698d7d144 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 07:24:45 +0100
Subject: [PATCH 376/455] Reset filters when reopening table filter button

---
 .../backend/DataTable/buttons/TableFilterButton.svelte | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index ca4c76c9688..2071ab8b860 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -52,7 +52,15 @@
   {filterCount ? `Filter: ${filterCount}` : "Filter"}
 </ActionButton>
 
-<Drawer bind:this={drawer} title="Filtering" on:drawerHide forceModal>
+<Drawer
+  bind:this={drawer}
+  title="Filtering"
+  on:drawerHide
+  on:drawerShow={() => {
+    localFilters = filters
+  }}
+  forceModal
+>
   <Button
     cta
     slot="buttons"

From 8a0193cfb716f8a765dffc8813f2e1d3554e3e6d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 07:25:45 +0100
Subject: [PATCH 377/455] Fix typo

---
 .../frontend-core/src/components/CoreFilterBuilder.svelte     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
index 739426150cf..9401d8e3841 100644
--- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte
+++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte
@@ -54,9 +54,9 @@
     }
   }
 
-  // We still may need to migrate this even though the backend does it automatically nmow
+  // We still may need to migrate this even though the backend does it automatically now
   // for query definitions. This is because we might be editing saved filter definitions
-  // from old screens, which will still be of type LegacyFilter[]
+  // from old screens, which will still be of type LegacyFilter[].
   const migrateFilters = filters => {
     if (Array.isArray(filters)) {
       return utils.processSearchFilters(filters)

From 247d57887a694af1dd2fe1825cf387a2eee7b7bb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 09:07:36 +0100
Subject: [PATCH 378/455] Remove lightest and dark themes, use enums for
 themes, standardise naming

---
 .../commandPalette/CommandPalette.svelte      |  4 +-
 .../src/components/settings/ThemeModal.svelte |  4 +-
 .../_components/Screen/AppThemeSelect.svelte  | 15 +++---
 .../builder/portal/apps/[appId]/index.svelte  |  9 ++--
 packages/builder/src/stores/builder/theme.js  | 14 +++---
 packages/builder/src/stores/portal/theme.js   | 46 +++++++++----------
 .../client/src/components/ClientApp.svelte    |  8 +++-
 packages/client/src/stores/theme.js           |  7 ++-
 packages/frontend-core/src/constants.js       | 32 +++++++------
 packages/frontend-core/src/utils/theme.js     | 43 ++++++++++++++---
 10 files changed, 106 insertions(+), 76 deletions(-)

diff --git a/packages/builder/src/components/commandPalette/CommandPalette.svelte b/packages/builder/src/components/commandPalette/CommandPalette.svelte
index f48fdf49293..a8b4dedbdb2 100644
--- a/packages/builder/src/components/commandPalette/CommandPalette.svelte
+++ b/packages/builder/src/components/commandPalette/CommandPalette.svelte
@@ -141,13 +141,13 @@
       icon: "ShareAndroid",
       action: () => $goto(`./automation/${automation._id}`),
     })) ?? []),
-    ...Constants.Themes.map(theme => ({
+    ...Constants.ThemeOptions.map(theme => ({
       type: "Change Builder Theme",
       name: theme.name,
       icon: "ColorPalette",
       action: () =>
         themeStore.update(state => {
-          state.theme = theme.class
+          state.theme = theme.id
           return state
         }),
     })),
diff --git a/packages/builder/src/components/settings/ThemeModal.svelte b/packages/builder/src/components/settings/ThemeModal.svelte
index cd1649fcb4b..eb63a49b02c 100644
--- a/packages/builder/src/components/settings/ThemeModal.svelte
+++ b/packages/builder/src/components/settings/ThemeModal.svelte
@@ -6,10 +6,10 @@
 
 <ModalContent title="Theme">
   <Select
-    options={Constants.Themes}
+    options={Constants.ThemeOptions}
     bind:value={$themeStore.theme}
     placeholder={null}
     getOptionLabel={x => x.name}
-    getOptionValue={x => x.class}
+    getOptionValue={x => x.id}
   />
 </ModalContent>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte
index 75a7598a60b..11852369244 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte
@@ -1,11 +1,11 @@
 <script>
   import { notifications } from "@budibase/bbui"
   import { themeStore, appStore } from "stores/builder"
-  import { Constants } from "@budibase/frontend-core"
+  import { Constants, getThemeClassNames } from "@budibase/frontend-core"
 
   const onChangeTheme = async theme => {
     try {
-      await themeStore.save(`spectrum--${theme}`, $appStore.appId)
+      await themeStore.save(theme, $appStore.appId)
     } catch (error) {
       notifications.error("Error updating theme")
     }
@@ -15,16 +15,13 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div class="container">
-  {#each Constants.Themes as theme}
+  {#each Constants.ThemeOptions as theme}
     <div
       class="theme"
-      class:selected={`spectrum--${theme.class}` === $themeStore.theme}
-      on:click={() => onChangeTheme(theme.class)}
+      class:selected={theme.id === $themeStore.theme}
+      on:click={() => onChangeTheme(theme.id)}
     >
-      <div
-        style="background: {theme.preview}"
-        class="color spectrum--{theme.class}"
-      />
+      <div class="color {getThemeClassNames(theme.id)}" />
       {theme.name}
     </div>
   {/each}
diff --git a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
index f5f410f6d95..f836944acba 100644
--- a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
@@ -21,7 +21,10 @@
   import { sdk } from "@budibase/shared-core"
   import { API } from "api"
   import ErrorSVG from "./ErrorSVG.svelte"
-  import { getBaseTheme, ClientAppSkeleton } from "@budibase/frontend-core"
+  import {
+    ClientAppSkeleton,
+    getThemeClassNames,
+  } from "@budibase/frontend-core"
   import { contextMenuStore } from "stores/builder"
 
   $: app = $enrichedApps.find(app => app.appId === $params.appId)
@@ -163,9 +166,7 @@
       class:hide={!loading || !app?.features?.skeletonLoader}
       class="loading"
     >
-      <div
-        class={`loadingThemeWrapper ${getBaseTheme(app.theme)} ${app.theme}`}
-      >
+      <div class="loadingThemeWrapper {getThemeClassNames(app.theme)}">
         <ClientAppSkeleton
           noAnimation
           hideDevTools={app?.status === "published"}
diff --git a/packages/builder/src/stores/builder/theme.js b/packages/builder/src/stores/builder/theme.js
index efa2609b974..6a06eda1ce3 100644
--- a/packages/builder/src/stores/builder/theme.js
+++ b/packages/builder/src/stores/builder/theme.js
@@ -1,24 +1,24 @@
 import { writable, get } from "svelte/store"
 import { API } from "api"
-import { getBaseTheme } from "@budibase/frontend-core"
+import { Constants, ensureValidTheme } from "@budibase/frontend-core"
 
+const DefaultAppTheme = Constants.Themes.Light
 const INITIAL_THEMES_STATE = {
-  theme: "",
+  theme: DefaultAppTheme,
   customTheme: {},
 }
 
-export const themes = () => {
+export const createThemeStore = () => {
   const store = writable({
     ...INITIAL_THEMES_STATE,
   })
 
   const syncAppTheme = app => {
     store.update(state => {
-      const theme = app.theme || "spectrum--light"
+      const theme = ensureValidTheme(app.theme, DefaultAppTheme)
       return {
         ...state,
         theme,
-        baseTheme: getBaseTheme(theme),
         customTheme: app.customTheme,
       }
     })
@@ -51,7 +51,7 @@ export const themes = () => {
     const { theme, customTheme } = metadata
     store.update(state => ({
       ...state,
-      theme,
+      theme: ensureValidTheme(theme, DefaultAppTheme),
       customTheme,
     }))
   }
@@ -66,4 +66,4 @@ export const themes = () => {
   }
 }
 
-export const themeStore = themes()
+export const themeStore = createThemeStore()
diff --git a/packages/builder/src/stores/portal/theme.js b/packages/builder/src/stores/portal/theme.js
index 619eb74f6a7..6f204849d06 100644
--- a/packages/builder/src/stores/portal/theme.js
+++ b/packages/builder/src/stores/portal/theme.js
@@ -1,38 +1,36 @@
-import { Constants, createLocalStorageStore } from "@budibase/frontend-core"
+import {
+  Constants,
+  createLocalStorageStore,
+  ensureValidTheme,
+  getThemeClassNames,
+} from "@budibase/frontend-core"
+import { derived } from "svelte/store"
 
 export const getThemeStore = () => {
+  const defaultBuilderTheme = Constants.Themes.Darkest
   const themeElement = document.documentElement
-
   const initialValue = {
-    theme: "darkest",
+    theme: defaultBuilderTheme,
   }
   const store = createLocalStorageStore("bb-theme", initialValue)
+  const derivedStore = derived(store, $store => ({
+    ...$store,
+    theme: ensureValidTheme($store.theme, defaultBuilderTheme),
+  }))
 
   // Update theme class when store changes
-  store.subscribe(state => {
-    // Handle any old local storage values - this can be removed after the update
-    if (state.darkMode !== undefined) {
-      store.set(initialValue)
-      return
-    }
-
-    // Update global class names to use the new theme and remove others
-    Constants.Themes.forEach(option => {
-      themeElement.classList.toggle(
-        `spectrum--${option.class}`,
-        option.class === state.theme
-      )
+  derivedStore.subscribe(({ theme }) => {
+    const classNames = getThemeClassNames(theme).split(" ")
+    Constants.ThemeOptions.forEach(option => {
+      const className = `${Constants.ThemeClassPrefix}${option.id}`
+      themeElement.classList.toggle(className, classNames.includes(className))
     })
-
-    // Add base theme if required
-    const selectedTheme = Constants.Themes.find(x => x.class === state.theme)
-    if (selectedTheme?.base) {
-      themeElement.classList.add(`spectrum--${selectedTheme.base}`)
-    }
   })
 
-  return store
+  return {
+    ...store,
+    subscribe: derivedStore.subscribe,
+  }
 }
 
-// ?? confusion
 export const themeStore = getThemeStore()
diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte
index f1e4f10403e..654e9510618 100644
--- a/packages/client/src/components/ClientApp.svelte
+++ b/packages/client/src/components/ClientApp.svelte
@@ -3,7 +3,11 @@
   import { setContext, onMount } from "svelte"
   import { Layout, Heading, Body } from "@budibase/bbui"
   import ErrorSVG from "@budibase/frontend-core/assets/error.svg"
-  import { Constants, CookieUtils } from "@budibase/frontend-core"
+  import {
+    Constants,
+    CookieUtils,
+    getThemeClassNames,
+  } from "@budibase/frontend-core"
   import Component from "./Component.svelte"
   import SDK from "sdk"
   import {
@@ -154,7 +158,7 @@
     id="spectrum-root"
     lang="en"
     dir="ltr"
-    class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
+    class="spectrum spectrum--medium {getThemeClassNames($themeStore.theme)}"
     class:builder={$builderStore.inBuilder}
     class:show={fontsLoaded && dataLoaded}
   >
diff --git a/packages/client/src/stores/theme.js b/packages/client/src/stores/theme.js
index 50dad12fa4d..dde387ba9aa 100644
--- a/packages/client/src/stores/theme.js
+++ b/packages/client/src/stores/theme.js
@@ -1,12 +1,12 @@
 import { derived } from "svelte/store"
 import { appStore } from "./app"
 import { builderStore } from "./builder"
-import { getBaseTheme } from "@budibase/frontend-core"
+import { Constants, ensureValidTheme } from "@budibase/frontend-core"
 
 // This is the good old acorn bug where having the word "g l o b a l" makes it
 // think that this is not ES6 compatible and starts throwing errors when using
 // optional chaining. Piss off acorn.
-const defaultTheme = "spectrum--light"
+const defaultTheme = Constants.Themes.Light
 const defaultCustomTheme = {
   primaryColor: "var(--spectrum-glo" + "bal-color-blue-600)",
   primaryColorHover: "var(--spectrum-glo" + "bal-color-blue-500)",
@@ -27,7 +27,7 @@ const createThemeStore = () => {
       }
 
       // Ensure theme is set
-      theme = theme || defaultTheme
+      theme = ensureValidTheme(theme, defaultTheme)
 
       // Delete and nullish keys from the custom theme
       if (customTheme) {
@@ -52,7 +52,6 @@ const createThemeStore = () => {
 
       return {
         theme,
-        baseTheme: getBaseTheme(theme),
         customTheme,
         customThemeCss,
       }
diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index 014b6b95797..976eab0534a 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -108,32 +108,34 @@ export const Roles = {
   CREATOR: "CREATOR",
 }
 
-export const Themes = [
-  {
-    class: "lightest",
-    name: "Lightest",
-  },
+// Theming
+export const ThemeClassPrefix = "spectrum--"
+export const Themes = {
+  Lightest: "lightest",
+  Light: "light",
+  Dark: "dark",
+  Darkest: "darkest",
+  Nord: "nord",
+  Midnight: "midnight",
+}
+export const ThemeOptions = [
   {
-    class: "light",
+    id: Themes.Light,
     name: "Light",
   },
   {
-    class: "dark",
+    id: Themes.Darkest,
     name: "Dark",
   },
   {
-    class: "darkest",
-    name: "Darkest",
-  },
-  {
-    class: "nord",
+    id: Themes.Nord,
     name: "Nord",
-    base: "darkest",
+    base: Themes.Darkest,
   },
   {
-    class: "midnight",
+    id: Themes.Midnight,
     name: "Midnight",
-    base: "darkest",
+    base: Themes.Darkest,
   },
 ]
 
diff --git a/packages/frontend-core/src/utils/theme.js b/packages/frontend-core/src/utils/theme.js
index 165f7c97822..309e6ab4a85 100644
--- a/packages/frontend-core/src/utils/theme.js
+++ b/packages/frontend-core/src/utils/theme.js
@@ -1,12 +1,41 @@
-import { Themes } from "../constants.js"
+import { Themes, ThemeOptions, ThemeClassPrefix } from "../constants.js"
 
-export const getBaseTheme = theme => {
+// Gets the CSS class names for the specified theme
+export const getThemeClassNames = theme => {
+  theme = ensureValidTheme(theme)
+  let classNames = `${ThemeClassPrefix}${theme}`
+
+  // Prefix with base class if required
+  const base = ThemeOptions.find(x => x.id === theme)?.base
+  if (base) {
+    classNames = `${ThemeClassPrefix}${base} ${classNames}`
+  }
+
+  return classNames
+}
+
+// Ensures a theme value is a valid option
+export const ensureValidTheme = (theme, fallback = Themes.Darkest) => {
+  // Default to darkest
   if (!theme) {
-    return ""
+    return fallback
   }
-  let base = Themes.find(x => `spectrum--${x.class}` === theme)?.base || ""
-  if (base) {
-    base = `spectrum--${base}`
+
+  // Ensure we aren't using the spectrum prefix
+  if (theme.startsWith(ThemeClassPrefix)) {
+    theme = theme.split(ThemeClassPrefix)[1]
+  }
+
+  // Check we aren't using a deprecated theme, and migrate
+  // to the nearest valid theme if we are
+  if (!ThemeOptions.some(x => x.id === theme)) {
+    if (theme === Themes.Lightest) {
+      return Themes.Light
+    } else if (theme === Themes.Dark) {
+      return Themes.Darkest
+    } else {
+      return fallback
+    }
   }
-  return base
+  return theme
 }

From 4c112d51e6de83c12ca23767fce0ebd819e99745 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 09:24:11 +0100
Subject: [PATCH 379/455] Move themes to shared core, add enum and type

---
 packages/frontend-core/src/constants.js       | 31 -------------------
 .../server/src/api/controllers/application.ts |  2 +-
 packages/server/src/constants/themes.ts       | 19 +++++++-----
 packages/shared-core/src/constants/themes.ts  | 25 +++++++++++++++
 packages/shared-core/src/index.ts             |  1 +
 .../theme.js => shared-core/src/themes.ts}    | 21 +++++++------
 packages/types/src/documents/app/index.ts     |  1 +
 packages/types/src/documents/app/theme.ts     | 14 +++++++++
 8 files changed, 65 insertions(+), 49 deletions(-)
 create mode 100644 packages/shared-core/src/constants/themes.ts
 rename packages/{frontend-core/src/utils/theme.js => shared-core/src/themes.ts} (61%)
 create mode 100644 packages/types/src/documents/app/theme.ts

diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index 976eab0534a..662b086915f 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -108,37 +108,6 @@ export const Roles = {
   CREATOR: "CREATOR",
 }
 
-// Theming
-export const ThemeClassPrefix = "spectrum--"
-export const Themes = {
-  Lightest: "lightest",
-  Light: "light",
-  Dark: "dark",
-  Darkest: "darkest",
-  Nord: "nord",
-  Midnight: "midnight",
-}
-export const ThemeOptions = [
-  {
-    id: Themes.Light,
-    name: "Light",
-  },
-  {
-    id: Themes.Darkest,
-    name: "Dark",
-  },
-  {
-    id: Themes.Nord,
-    name: "Nord",
-    base: Themes.Darkest,
-  },
-  {
-    id: Themes.Midnight,
-    name: "Midnight",
-    base: Themes.Darkest,
-  },
-]
-
 export const EventPublishType = {
   ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
 }
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index d8d9d75336b..0d811208cdb 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -302,7 +302,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
         navBackground: "var(--spectrum-global-color-gray-100)",
         links: [],
       },
-      theme: "spectrum--light",
+      theme: "light",
       customTheme: {
         buttonBorderRadius: "16px",
       },
diff --git a/packages/server/src/constants/themes.ts b/packages/server/src/constants/themes.ts
index 942d67f1005..cc73e856210 100644
--- a/packages/server/src/constants/themes.ts
+++ b/packages/server/src/constants/themes.ts
@@ -1,5 +1,9 @@
-export const getThemeVariables = (theme: string) => {
-  if (theme === "spectrum--lightest") {
+import { ensureValidTheme } from "@budibase/shared-core"
+import { Theme } from "@budibase/types"
+
+export const getThemeVariables = (theme: Theme) => {
+  theme = ensureValidTheme(theme, Theme.LIGHT)
+  if (theme === Theme.LIGHTEST) {
     return `
       --spectrum-global-color-gray-50: rgb(255, 255, 255);
       --spectrum-global-color-gray-200: rgb(244, 244, 244);
@@ -7,16 +11,15 @@ export const getThemeVariables = (theme: string) => {
       --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-50);
     `
   }
-  if (theme === "spectrum--light") {
+  if (theme === Theme.LIGHT) {
     return `
     --spectrum-global-color-gray-50: rgb(255, 255, 255);
     --spectrum-global-color-gray-200: rgb(234, 234, 234);
     --spectrum-global-color-gray-300: rgb(225, 225, 225);
     --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-50);
-
     `
   }
-  if (theme === "spectrum--dark") {
+  if (theme === Theme.DARK) {
     return `
     --spectrum-global-color-gray-100: rgb(50, 50, 50);
     --spectrum-global-color-gray-200: rgb(62, 62, 62);
@@ -24,7 +27,7 @@ export const getThemeVariables = (theme: string) => {
     --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
     `
   }
-  if (theme === "spectrum--darkest") {
+  if (theme === Theme.DARKEST) {
     return `
   --spectrum-global-color-gray-100: rgb(30, 30, 30);
   --spectrum-global-color-gray-200: rgb(44, 44, 44);
@@ -32,7 +35,7 @@ export const getThemeVariables = (theme: string) => {
   --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
     `
   }
-  if (theme === "spectrum--nord") {
+  if (theme === Theme.NORD) {
     return `
     --spectrum-global-color-gray-100: #3b4252;
 
@@ -41,7 +44,7 @@ export const getThemeVariables = (theme: string) => {
   --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
     `
   }
-  if (theme === "spectrum--midnight") {
+  if (theme === Theme.MIDNIGHT) {
     return `
     --hue: 220;
     --sat: 10%;
diff --git a/packages/shared-core/src/constants/themes.ts b/packages/shared-core/src/constants/themes.ts
new file mode 100644
index 00000000000..208700f1afe
--- /dev/null
+++ b/packages/shared-core/src/constants/themes.ts
@@ -0,0 +1,25 @@
+import { ThemeMeta, Theme } from "@budibase/types"
+
+export const ThemeOptions: ThemeMeta[] = [
+  {
+    id: Theme.LIGHT,
+    name: "Light",
+  },
+  {
+    // We call this dark for simplicity, but we want to use the spectrum darkest style
+    id: Theme.DARKEST,
+    name: "Dark",
+  },
+  {
+    id: Theme.NORD,
+    name: "Nord",
+    base: Theme.DARKEST,
+  },
+  {
+    id: Theme.MIDNIGHT,
+    name: "Midnight",
+    base: Theme.DARKEST,
+  },
+]
+
+export const ThemeClassPrefix = "spectrum--"
diff --git a/packages/shared-core/src/index.ts b/packages/shared-core/src/index.ts
index a26c76cd967..0833f999ee5 100644
--- a/packages/shared-core/src/index.ts
+++ b/packages/shared-core/src/index.ts
@@ -4,3 +4,4 @@ export * as helpers from "./helpers"
 export * as utils from "./utils"
 export * as sdk from "./sdk"
 export * from "./table"
+export * from "./themes"
diff --git a/packages/frontend-core/src/utils/theme.js b/packages/shared-core/src/themes.ts
similarity index 61%
rename from packages/frontend-core/src/utils/theme.js
rename to packages/shared-core/src/themes.ts
index 309e6ab4a85..1d1875d4e94 100644
--- a/packages/frontend-core/src/utils/theme.js
+++ b/packages/shared-core/src/themes.ts
@@ -1,7 +1,8 @@
-import { Themes, ThemeOptions, ThemeClassPrefix } from "../constants.js"
+import { ThemeOptions, ThemeClassPrefix } from "./constants/themes"
+import { Theme } from "@budibase/types"
 
 // Gets the CSS class names for the specified theme
-export const getThemeClassNames = theme => {
+export const getThemeClassNames = (theme: Theme): string => {
   theme = ensureValidTheme(theme)
   let classNames = `${ThemeClassPrefix}${theme}`
 
@@ -15,24 +16,26 @@ export const getThemeClassNames = theme => {
 }
 
 // Ensures a theme value is a valid option
-export const ensureValidTheme = (theme, fallback = Themes.Darkest) => {
-  // Default to darkest
+export const ensureValidTheme = (
+  theme: Theme,
+  fallback: Theme = Theme.DARKEST
+): Theme => {
   if (!theme) {
     return fallback
   }
 
   // Ensure we aren't using the spectrum prefix
   if (theme.startsWith(ThemeClassPrefix)) {
-    theme = theme.split(ThemeClassPrefix)[1]
+    theme = theme.split(ThemeClassPrefix)[1] as Theme
   }
 
   // Check we aren't using a deprecated theme, and migrate
   // to the nearest valid theme if we are
   if (!ThemeOptions.some(x => x.id === theme)) {
-    if (theme === Themes.Lightest) {
-      return Themes.Light
-    } else if (theme === Themes.Dark) {
-      return Themes.Darkest
+    if (theme === Theme.LIGHTEST) {
+      return Theme.LIGHT
+    } else if (theme === Theme.DARK) {
+      return Theme.DARKEST
     } else {
       return fallback
     }
diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts
index 2b13676ba13..bb94c3b4da7 100644
--- a/packages/types/src/documents/app/index.ts
+++ b/packages/types/src/documents/app/index.ts
@@ -17,3 +17,4 @@ export * from "./component"
 export * from "./sqlite"
 export * from "./snippet"
 export * from "./rowAction"
+export * from "./theme"
diff --git a/packages/types/src/documents/app/theme.ts b/packages/types/src/documents/app/theme.ts
new file mode 100644
index 00000000000..ca5d429b5bf
--- /dev/null
+++ b/packages/types/src/documents/app/theme.ts
@@ -0,0 +1,14 @@
+export enum Theme {
+  LIGHTEST = "lightest",
+  LIGHT = "light",
+  DARK = "dark",
+  DARKEST = "darkest",
+  NORD = "nord",
+  MIDNIGHT = "midnight",
+}
+
+export type ThemeMeta = {
+  id: string
+  name: string
+  base?: Theme
+}

From ab0f81824d040f4848d5bb17fd1b46f8ecd2f425 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 09:35:55 +0100
Subject: [PATCH 380/455] Update to use new shared-core location for theme
 utils

---
 .../commandPalette/CommandPalette.svelte      |  7 ++++---
 .../components/integration/QueryEditor.svelte |  3 ++-
 .../src/components/settings/ThemeModal.svelte |  4 ++--
 .../_components/Screen/AppThemeSelect.svelte  | 12 ++++++------
 .../builder/portal/apps/[appId]/index.svelte  |  7 ++-----
 packages/builder/src/stores/builder/theme.js  | 11 +++--------
 packages/builder/src/stores/portal/theme.js   | 19 ++++++++++---------
 .../client/src/components/ClientApp.svelte    |  7 ++-----
 packages/client/src/stores/theme.js           |  5 ++---
 packages/frontend-core/src/utils/index.js     |  1 -
 packages/shared-core/src/constants/index.ts   |  1 +
 packages/shared-core/src/constants/themes.ts  |  7 +++++--
 12 files changed, 39 insertions(+), 45 deletions(-)

diff --git a/packages/builder/src/components/commandPalette/CommandPalette.svelte b/packages/builder/src/components/commandPalette/CommandPalette.svelte
index a8b4dedbdb2..1e9acbb7e2a 100644
--- a/packages/builder/src/components/commandPalette/CommandPalette.svelte
+++ b/packages/builder/src/components/commandPalette/CommandPalette.svelte
@@ -23,6 +23,7 @@
   import { themeStore } from "stores/portal"
   import { getContext } from "svelte"
   import { Constants } from "@budibase/frontend-core"
+  import { ThemeOptions } from "@budibase/shared-core"
 
   const modalContext = getContext(Context.Modal)
   const commands = [
@@ -141,13 +142,13 @@
       icon: "ShareAndroid",
       action: () => $goto(`./automation/${automation._id}`),
     })) ?? []),
-    ...Constants.ThemeOptions.map(theme => ({
+    ...ThemeOptions.map(themeMeta => ({
       type: "Change Builder Theme",
-      name: theme.name,
+      name: themeMeta.name,
       icon: "ColorPalette",
       action: () =>
         themeStore.update(state => {
-          state.theme = theme.id
+          state.theme = themeMeta.id
           return state
         }),
     })),
diff --git a/packages/builder/src/components/integration/QueryEditor.svelte b/packages/builder/src/components/integration/QueryEditor.svelte
index 97998c3d229..995451b7e35 100644
--- a/packages/builder/src/components/integration/QueryEditor.svelte
+++ b/packages/builder/src/components/integration/QueryEditor.svelte
@@ -3,6 +3,7 @@
   import { Label } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { themeStore } from "stores/portal"
+  import { Theme } from "@budibase/types"
 
   const dispatch = createEventDispatcher()
 
@@ -116,7 +117,7 @@
       readOnly,
       autoCloseBrackets: true,
       autoCloseTags: true,
-      theme: $themeStore.theme.includes("light") ? THEMES.LIGHT : THEMES.DARK,
+      theme: $themeStore.theme === Theme.LIGHT ? THEMES.LIGHT : THEMES.DARK,
     }
 
     if (!tab)
diff --git a/packages/builder/src/components/settings/ThemeModal.svelte b/packages/builder/src/components/settings/ThemeModal.svelte
index eb63a49b02c..7bc3c1efa37 100644
--- a/packages/builder/src/components/settings/ThemeModal.svelte
+++ b/packages/builder/src/components/settings/ThemeModal.svelte
@@ -1,12 +1,12 @@
 <script>
   import { ModalContent, Select } from "@budibase/bbui"
   import { themeStore } from "stores/portal"
-  import { Constants } from "@budibase/frontend-core"
+  import { ThemeOptions } from "@budibase/shared-core"
 </script>
 
 <ModalContent title="Theme">
   <Select
-    options={Constants.ThemeOptions}
+    options={ThemeOptions}
     bind:value={$themeStore.theme}
     placeholder={null}
     getOptionLabel={x => x.name}
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte
index 11852369244..eb7a839b227 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/AppThemeSelect.svelte
@@ -1,7 +1,7 @@
 <script>
   import { notifications } from "@budibase/bbui"
   import { themeStore, appStore } from "stores/builder"
-  import { Constants, getThemeClassNames } from "@budibase/frontend-core"
+  import { ThemeOptions, getThemeClassNames } from "@budibase/shared-core"
 
   const onChangeTheme = async theme => {
     try {
@@ -15,14 +15,14 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div class="container">
-  {#each Constants.ThemeOptions as theme}
+  {#each ThemeOptions as themeMeta}
     <div
       class="theme"
-      class:selected={theme.id === $themeStore.theme}
-      on:click={() => onChangeTheme(theme.id)}
+      class:selected={themeMeta.id === $themeStore.theme}
+      on:click={() => onChangeTheme(themeMeta.id)}
     >
-      <div class="color {getThemeClassNames(theme.id)}" />
-      {theme.name}
+      <div class="color {getThemeClassNames(themeMeta.id)}" />
+      {themeMeta.name}
     </div>
   {/each}
 </div>
diff --git a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
index f836944acba..cc03ec01dc0 100644
--- a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
@@ -18,13 +18,10 @@
     TooltipPosition,
     TooltipType,
   } from "@budibase/bbui"
-  import { sdk } from "@budibase/shared-core"
+  import { sdk, getThemeClassNames } from "@budibase/shared-core"
   import { API } from "api"
   import ErrorSVG from "./ErrorSVG.svelte"
-  import {
-    ClientAppSkeleton,
-    getThemeClassNames,
-  } from "@budibase/frontend-core"
+  import { ClientAppSkeleton } from "@budibase/frontend-core"
   import { contextMenuStore } from "stores/builder"
 
   $: app = $enrichedApps.find(app => app.appId === $params.appId)
diff --git a/packages/builder/src/stores/builder/theme.js b/packages/builder/src/stores/builder/theme.js
index 6a06eda1ce3..8e03e856b35 100644
--- a/packages/builder/src/stores/builder/theme.js
+++ b/packages/builder/src/stores/builder/theme.js
@@ -1,16 +1,11 @@
 import { writable, get } from "svelte/store"
 import { API } from "api"
-import { Constants, ensureValidTheme } from "@budibase/frontend-core"
-
-const DefaultAppTheme = Constants.Themes.Light
-const INITIAL_THEMES_STATE = {
-  theme: DefaultAppTheme,
-  customTheme: {},
-}
+import { ensureValidTheme, DefaultAppTheme } from "@budibase/shared-core"
 
 export const createThemeStore = () => {
   const store = writable({
-    ...INITIAL_THEMES_STATE,
+    theme: DefaultAppTheme,
+    customTheme: {},
   })
 
   const syncAppTheme = app => {
diff --git a/packages/builder/src/stores/portal/theme.js b/packages/builder/src/stores/portal/theme.js
index 6f204849d06..11a87845e12 100644
--- a/packages/builder/src/stores/portal/theme.js
+++ b/packages/builder/src/stores/portal/theme.js
@@ -1,28 +1,29 @@
+import { createLocalStorageStore } from "@budibase/frontend-core"
+import { derived } from "svelte/store"
 import {
-  Constants,
-  createLocalStorageStore,
+  DefaultBuilderTheme,
   ensureValidTheme,
   getThemeClassNames,
-} from "@budibase/frontend-core"
-import { derived } from "svelte/store"
+  ThemeOptions,
+  ThemeClassPrefix,
+} from "@budibase/shared-core"
 
 export const getThemeStore = () => {
-  const defaultBuilderTheme = Constants.Themes.Darkest
   const themeElement = document.documentElement
   const initialValue = {
-    theme: defaultBuilderTheme,
+    theme: DefaultBuilderTheme,
   }
   const store = createLocalStorageStore("bb-theme", initialValue)
   const derivedStore = derived(store, $store => ({
     ...$store,
-    theme: ensureValidTheme($store.theme, defaultBuilderTheme),
+    theme: ensureValidTheme($store.theme, DefaultBuilderTheme),
   }))
 
   // Update theme class when store changes
   derivedStore.subscribe(({ theme }) => {
     const classNames = getThemeClassNames(theme).split(" ")
-    Constants.ThemeOptions.forEach(option => {
-      const className = `${Constants.ThemeClassPrefix}${option.id}`
+    ThemeOptions.forEach(option => {
+      const className = `${ThemeClassPrefix}${option.id}`
       themeElement.classList.toggle(className, classNames.includes(className))
     })
   })
diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte
index 654e9510618..1610f36bdc7 100644
--- a/packages/client/src/components/ClientApp.svelte
+++ b/packages/client/src/components/ClientApp.svelte
@@ -3,11 +3,8 @@
   import { setContext, onMount } from "svelte"
   import { Layout, Heading, Body } from "@budibase/bbui"
   import ErrorSVG from "@budibase/frontend-core/assets/error.svg"
-  import {
-    Constants,
-    CookieUtils,
-    getThemeClassNames,
-  } from "@budibase/frontend-core"
+  import { Constants, CookieUtils } from "@budibase/frontend-core"
+  import { getThemeClassNames } from "@budibase/shared-core"
   import Component from "./Component.svelte"
   import SDK from "sdk"
   import {
diff --git a/packages/client/src/stores/theme.js b/packages/client/src/stores/theme.js
index dde387ba9aa..523d34cdc59 100644
--- a/packages/client/src/stores/theme.js
+++ b/packages/client/src/stores/theme.js
@@ -1,12 +1,11 @@
 import { derived } from "svelte/store"
 import { appStore } from "./app"
 import { builderStore } from "./builder"
-import { Constants, ensureValidTheme } from "@budibase/frontend-core"
+import { ensureValidTheme, DefaultAppTheme } from "@budibase/shared-core"
 
 // This is the good old acorn bug where having the word "g l o b a l" makes it
 // think that this is not ES6 compatible and starts throwing errors when using
 // optional chaining. Piss off acorn.
-const defaultTheme = Constants.Themes.Light
 const defaultCustomTheme = {
   primaryColor: "var(--spectrum-glo" + "bal-color-blue-600)",
   primaryColorHover: "var(--spectrum-glo" + "bal-color-blue-500)",
@@ -27,7 +26,7 @@ const createThemeStore = () => {
       }
 
       // Ensure theme is set
-      theme = ensureValidTheme(theme, defaultTheme)
+      theme = ensureValidTheme(theme, DefaultAppTheme)
 
       // Delete and nullish keys from the custom theme
       if (customTheme) {
diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js
index 00f98fcc77f..cc44eec74c2 100644
--- a/packages/frontend-core/src/utils/index.js
+++ b/packages/frontend-core/src/utils/index.js
@@ -9,7 +9,6 @@ export * as SchemaUtils from "./schema"
 export { memo, derivedMemo } from "./memo"
 export { createWebsocket } from "./websocket"
 export * from "./download"
-export * from "./theme"
 export * from "./settings"
 export * from "./relatedColumns"
 export * from "./table"
diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts
index 11bb79c8d3d..47b4e8ab77a 100644
--- a/packages/shared-core/src/constants/index.ts
+++ b/packages/shared-core/src/constants/index.ts
@@ -3,6 +3,7 @@ export * from "./api"
 export * from "./fields"
 export * from "./rows"
 export * from "./colors"
+export * from "./themes"
 
 export const OperatorOptions = {
   Equals: {
diff --git a/packages/shared-core/src/constants/themes.ts b/packages/shared-core/src/constants/themes.ts
index 208700f1afe..be34611009f 100644
--- a/packages/shared-core/src/constants/themes.ts
+++ b/packages/shared-core/src/constants/themes.ts
@@ -1,5 +1,10 @@
 import { ThemeMeta, Theme } from "@budibase/types"
 
+export const ThemeClassPrefix = "spectrum--"
+export const DefaultBuilderTheme = Theme.DARKEST
+export const DefaultAppTheme = Theme.LIGHT
+
+// Currently available theme options for builder and apps
 export const ThemeOptions: ThemeMeta[] = [
   {
     id: Theme.LIGHT,
@@ -21,5 +26,3 @@ export const ThemeOptions: ThemeMeta[] = [
     base: Theme.DARKEST,
   },
 ]
-
-export const ThemeClassPrefix = "spectrum--"

From d7ad035e0bd61cd710ebf8713cd7613a7e3ac526 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 09:36:51 +0100
Subject: [PATCH 381/455] Lint

---
 packages/builder/src/stores/builder/viewsV2.js                   | 1 -
 .../src/components/grid/stores/datasources/viewV2.js             | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js
index 43c94770ab3..9bd32f4a247 100644
--- a/packages/builder/src/stores/builder/viewsV2.js
+++ b/packages/builder/src/stores/builder/viewsV2.js
@@ -1,7 +1,6 @@
 import { writable, derived, get } from "svelte/store"
 import { tables } from "./tables"
 import { API } from "api"
-import { dataFilters } from "@budibase/shared-core"
 
 export function createViewsV2Store() {
   const store = writable({
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 281443cbc32..24170a59b35 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -1,5 +1,4 @@
 import { get } from "svelte/store"
-import { dataFilters } from "@budibase/shared-core"
 
 const SuppressErrors = true
 

From 27d42a9bfe7bfb496f43772fb214db553d653cc5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 09:43:30 +0100
Subject: [PATCH 382/455] Fix theme of in builder preview

---
 .../design/[screenId]/_components/AppPreview.svelte            | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
index 055ecd88e08..ccea636328d 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
@@ -19,6 +19,7 @@
   import { findComponent, findComponentPath } from "helpers/components"
   import { isActive, goto } from "@roxi/routify"
   import { ClientAppSkeleton } from "@budibase/frontend-core"
+  import { getThemeClassNames } from "@budibase/shared-core"
 
   let iframe
   let layout
@@ -257,7 +258,7 @@
   class:mobile={$previewStore.previewDevice === "mobile"}
 >
   {#if loading}
-    <div class={`loading ${$themeStore.baseTheme} ${$themeStore.theme}`}>
+    <div class={`loading ${getThemeClassNames($themeStore.theme)}`}>
       <ClientAppSkeleton
         sideNav={$navigationStore?.navigation === "Left"}
         hideFooter

From 90fb36315f70bf162252182f16b2796e26b5b4c3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 10:06:04 +0100
Subject: [PATCH 383/455] Add client feature to ensure full backwards
 compatibility with themes

---
 .../design/[screenId]/_components/AppPreview.svelte         | 6 ++++--
 packages/client/manifest.json                               | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
index ccea636328d..1e8868f03ca 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
@@ -19,7 +19,7 @@
   import { findComponent, findComponentPath } from "helpers/components"
   import { isActive, goto } from "@roxi/routify"
   import { ClientAppSkeleton } from "@budibase/frontend-core"
-  import { getThemeClassNames } from "@budibase/shared-core"
+  import { getThemeClassNames, ThemeClassPrefix } from "@budibase/shared-core"
 
   let iframe
   let layout
@@ -48,7 +48,9 @@
     layout,
     screen,
     selectedComponentId,
-    theme: $themeStore.theme,
+    theme: $appStore.clientFeatures.unifiedThemes
+      ? $themeStore.theme
+      : `${ThemeClassPrefix}${$themeStore.theme}`,
     customTheme: $themeStore.customTheme,
     previewDevice: $previewStore.previewDevice,
     messagePassing: $appStore.clientFeatures.messagePassing,
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 17aebc5225f..393c08a2824 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -1,6 +1,7 @@
 {
   "features": {
     "spectrumThemes": true,
+    "unifiedTheme": true,
     "intelligentLoading": true,
     "deviceAwareness": true,
     "state": true,

From a38ae930970cfea7adf932f0a93597a22ebec34b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 10:10:14 +0100
Subject: [PATCH 384/455] Lint

---
 .../builder/src/components/commandPalette/CommandPalette.svelte  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/commandPalette/CommandPalette.svelte b/packages/builder/src/components/commandPalette/CommandPalette.svelte
index 1e9acbb7e2a..51a688fc724 100644
--- a/packages/builder/src/components/commandPalette/CommandPalette.svelte
+++ b/packages/builder/src/components/commandPalette/CommandPalette.svelte
@@ -22,7 +22,6 @@
   } from "stores/builder"
   import { themeStore } from "stores/portal"
   import { getContext } from "svelte"
-  import { Constants } from "@budibase/frontend-core"
   import { ThemeOptions } from "@budibase/shared-core"
 
   const modalContext = getContext(Context.Modal)

From 249f44dc110a059b5d20404e39df01811d1421fb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 10:13:42 +0100
Subject: [PATCH 385/455] Use new default app theme constant when creating apps

---
 packages/server/src/api/controllers/application.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 0d811208cdb..e7d0ed7ba74 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -63,7 +63,7 @@ import {
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { sdk as sharedCoreSDK } from "@budibase/shared-core"
+import { DefaultAppTheme, sdk as sharedCoreSDK } from "@budibase/shared-core"
 import * as appMigrations from "../../appMigrations"
 
 // utility function, need to do away with this
@@ -302,7 +302,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
         navBackground: "var(--spectrum-global-color-gray-100)",
         links: [],
       },
-      theme: "light",
+      theme: DefaultAppTheme,
       customTheme: {
         buttonBorderRadius: "16px",
       },

From 9c9d2560b2f8ef6ebc95af1bd8c12a35561cadd4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 11:17:31 +0200
Subject: [PATCH 386/455] Remove fetchEnriched

---
 .../server/src/api/routes/tests/rowAction.spec.ts     |  2 +-
 packages/server/src/tests/utilities/api/automation.ts | 11 -----------
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index 5cec83fe23d..2b6b0f01c9c 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -958,7 +958,7 @@ describe("/rowsActions", () => {
       // document was not being cleaned up. This meant there existed code paths
       // that would find it and try to reference the tables within it, resulting
       // in errors.
-      await config.api.automation.fetchEnriched({
+      await config.api.automation.fetch({
         status: 200,
       })
     })
diff --git a/packages/server/src/tests/utilities/api/automation.ts b/packages/server/src/tests/utilities/api/automation.ts
index 11c041d52b7..9d9a27e891a 100644
--- a/packages/server/src/tests/utilities/api/automation.ts
+++ b/packages/server/src/tests/utilities/api/automation.ts
@@ -23,17 +23,6 @@ export class AutomationAPI extends TestAPI {
     })
   }
 
-  fetchEnriched = async (
-    expectations?: Expectations
-  ): Promise<FetchAutomationResponse> => {
-    return await this._get<FetchAutomationResponse>(
-      `/api/automations?enrich=true`,
-      {
-        expectations,
-      }
-    )
-  }
-
   post = async (
     body: Automation,
     expectations?: Expectations

From 889b216a93e63374b431c9cfcf78e11aab3180cb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 10:22:43 +0100
Subject: [PATCH 387/455] Add theme tests

---
 packages/shared-core/src/tests/themes.ts | 28 ++++++++++++++++++++++++
 packages/shared-core/src/themes.ts       |  2 +-
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 packages/shared-core/src/tests/themes.ts

diff --git a/packages/shared-core/src/tests/themes.ts b/packages/shared-core/src/tests/themes.ts
new file mode 100644
index 00000000000..89c6910ad5b
--- /dev/null
+++ b/packages/shared-core/src/tests/themes.ts
@@ -0,0 +1,28 @@
+import { getThemeClassNames, ensureValidTheme } from "../themes"
+import { Theme } from "@budibase/types"
+
+describe("theme class names", () => {
+  it("generates class names for a theme without base theme", () => {
+    expect(getThemeClassNames(Theme.LIGHT)).toStrictEqual("spectrum--light")
+  })
+  it("generates class names for a theme with base theme", () => {
+    expect(getThemeClassNames(Theme.NORD)).toStrictEqual(
+      "spectrum--darkest spectrum--nord"
+    )
+  })
+})
+
+describe("theme validity checking", () => {
+  it("handles no theme", () => {
+    expect(ensureValidTheme(undefined)).toStrictEqual(Theme.DARKEST)
+  })
+  it("allows specifiying a fallback", () => {
+    expect(ensureValidTheme(undefined, Theme.NORD)).toStrictEqual(Theme.NORD)
+  })
+  it("migrates lightest to light", () => {
+    expect(ensureValidTheme(Theme.LIGHTEST)).toStrictEqual(Theme.LIGHT)
+  })
+  it("migrates dark to darkest", () => {
+    expect(ensureValidTheme(Theme.DARK)).toStrictEqual(Theme.DARKEST)
+  })
+})
diff --git a/packages/shared-core/src/themes.ts b/packages/shared-core/src/themes.ts
index 1d1875d4e94..d70f1884265 100644
--- a/packages/shared-core/src/themes.ts
+++ b/packages/shared-core/src/themes.ts
@@ -17,7 +17,7 @@ export const getThemeClassNames = (theme: Theme): string => {
 
 // Ensures a theme value is a valid option
 export const ensureValidTheme = (
-  theme: Theme,
+  theme?: Theme,
   fallback: Theme = Theme.DARKEST
 ): Theme => {
   if (!theme) {

From 687996257f6f2109b7a088031b1708d7c2709263 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 10:27:41 +0100
Subject: [PATCH 388/455] Revert change to view error message

---
 packages/server/src/sdk/app/views/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts
index a0194b6d1bf..8ca5fd1098f 100644
--- a/packages/server/src/sdk/app/views/utils.ts
+++ b/packages/server/src/sdk/app/views/utils.ts
@@ -23,7 +23,7 @@ export function ensureQueryUISet(viewArg: Readonly<ViewV2>): ViewV2 {
       // So despite the type saying that `view.query` is a LegacyFilter[] |
       // SearchFilters, it will never be a SearchFilters when a `view.queryUI`
       // is specified, making it "safe" to throw an error here.
-      throw new HTTPError(`View ${view.name} is missing queryUI field`, 400)
+      throw new HTTPError("view is missing queryUI field", 400)
     }
 
     view.queryUI = utils.processSearchFilters(view.query)

From ccdb03244966022e25d7a50b5f786dced0e5660b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 11:30:59 +0200
Subject: [PATCH 389/455] Move sdk rowaction code

---
 .../src/sdk/app/{rowActions.ts => rowActions/crud.ts}  | 10 +++++-----
 packages/server/src/sdk/app/rowActions/index.ts        |  1 +
 2 files changed, 6 insertions(+), 5 deletions(-)
 rename packages/server/src/sdk/app/{rowActions.ts => rowActions/crud.ts} (96%)
 create mode 100644 packages/server/src/sdk/app/rowActions/index.ts

diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions/crud.ts
similarity index 96%
rename from packages/server/src/sdk/app/rowActions.ts
rename to packages/server/src/sdk/app/rowActions/crud.ts
index de73969696a..11536776858 100644
--- a/packages/server/src/sdk/app/rowActions.ts
+++ b/packages/server/src/sdk/app/rowActions/crud.ts
@@ -7,11 +7,11 @@ import {
   User,
   VirtualDocumentType,
 } from "@budibase/types"
-import { generateRowActionsID } from "../../db/utils"
-import automations from "./automations"
-import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo"
-import * as triggers from "../../automations/triggers"
-import sdk from ".."
+import { generateRowActionsID } from "../../../db/utils"
+import automations from "../automations"
+import { definitions as TRIGGER_DEFINITIONS } from "../../../automations/triggerInfo"
+import * as triggers from "../../../automations/triggers"
+import sdk from "../.."
 
 async function ensureUniqueAndThrow(
   doc: TableRowActions,
diff --git a/packages/server/src/sdk/app/rowActions/index.ts b/packages/server/src/sdk/app/rowActions/index.ts
new file mode 100644
index 00000000000..8baf08477f7
--- /dev/null
+++ b/packages/server/src/sdk/app/rowActions/index.ts
@@ -0,0 +1 @@
+export * from "./crud"

From f2e46863f63befe90ea4f0d64c744b63786fa29f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 11:53:59 +0200
Subject: [PATCH 390/455] Add test deleting external ds

---
 .../src/api/routes/tests/rowAction.spec.ts    | 59 ++++++++++++++++++-
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index 2b6b0f01c9c..5246304906c 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -1,17 +1,20 @@
 import _ from "lodash"
 import tk from "timekeeper"
 
+import { context, roles } from "@budibase/backend-core"
+import { automations } from "@budibase/pro"
 import {
   CreateRowActionRequest,
   DocumentType,
   PermissionLevel,
   RowActionResponse,
+  TableRowActions,
 } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { Expectations } from "../../../tests/utilities/api/base"
-import { roles } from "@budibase/backend-core"
-import { automations } from "@budibase/pro"
+import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import { generateRowActionsID } from "../../../db/utils"
 
 const expectAutomationId = () =>
   expect.stringMatching(`^${DocumentType.AUTOMATION}_.+`)
@@ -962,5 +965,57 @@ describe("/rowsActions", () => {
         status: 200,
       })
     })
+
+    it.each([
+      [
+        "external",
+        async () => {
+          const ds = await config.createDatasource({
+            datasource: await getDatasource(DatabaseName.POSTGRES),
+          })
+          const table = await config.api.table.save(
+            setup.structures.tableForDatasource(ds)
+          )
+          return table
+        },
+      ],
+    ])(
+      "should delete all the row actions (and automations) for its tables when a datasource is deleted",
+      async (_, getTable) => {
+        async function getRowActionsFromDb(tableId: string) {
+          return await context.doInAppContext(config.getAppId(), async () => {
+            const db = context.getAppDB()
+            const tableDoc = await db.tryGet<TableRowActions>(
+              generateRowActionsID(tableId)
+            )
+            return tableDoc
+          })
+        }
+
+        const table = await getTable()
+        const tableId = table._id!
+
+        await config.api.rowAction.save(tableId, {
+          name: generator.guid(),
+        })
+        await config.api.rowAction.save(tableId, {
+          name: generator.guid(),
+        })
+
+        const { actions } = (await getRowActionsFromDb(tableId))!
+        expect(Object.entries(actions)).toHaveLength(2)
+
+        const { automations } = await config.api.automation.fetch()
+        expect(automations).toHaveLength(2)
+
+        const datasource = await config.api.datasource.get(table.sourceId)
+        await config.api.datasource.delete(datasource)
+
+        const automationsResp = await config.api.automation.fetch()
+        expect(automationsResp.automations).toHaveLength(0)
+
+        expect(await getRowActionsFromDb(tableId)).toBeUndefined()
+      }
+    )
   })
 })

From a1fbd3ef9aab56da55d62dd607abc6188c89b3f1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 10:56:52 +0100
Subject: [PATCH 391/455] Try to fix pro

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 297fdc937e9..f6aebba9445 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 297fdc937e9c650b4964fc1a942b60022b195865
+Subproject commit f6aebba94451ce47bba551926e5ad72bd75f71c6

From 7b0419773cd9f07af246d1d1584dafa695f34c1b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 12:08:17 +0200
Subject: [PATCH 392/455] Implement deletion

---
 packages/server/src/api/controllers/datasource.ts | 2 ++
 packages/server/src/sdk/app/rowActions/index.ts   | 1 +
 packages/server/src/sdk/app/rowActions/utils.ts   | 9 +++++++++
 3 files changed, 12 insertions(+)
 create mode 100644 packages/server/src/sdk/app/rowActions/utils.ts

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index baab008da87..794e2dfddd1 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -267,6 +267,8 @@ export async function destroy(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
+  await sdk.rowActions.deleteAllForDatasource(datasourceId)
+
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
     await destroyInternalTablesBySourceId(datasourceId)
   } else {
diff --git a/packages/server/src/sdk/app/rowActions/index.ts b/packages/server/src/sdk/app/rowActions/index.ts
index 8baf08477f7..b0dbd426df7 100644
--- a/packages/server/src/sdk/app/rowActions/index.ts
+++ b/packages/server/src/sdk/app/rowActions/index.ts
@@ -1 +1,2 @@
 export * from "./crud"
+export * from "./utils"
diff --git a/packages/server/src/sdk/app/rowActions/utils.ts b/packages/server/src/sdk/app/rowActions/utils.ts
new file mode 100644
index 00000000000..d4e5a4610d6
--- /dev/null
+++ b/packages/server/src/sdk/app/rowActions/utils.ts
@@ -0,0 +1,9 @@
+import sdk from "../../../sdk"
+
+export async function deleteAllForDatasource(datasourceId: string) {
+  const allTables = await sdk.tables.getAllTables()
+  const tables = allTables.filter(t => t.sourceId === datasourceId)
+  for (const table of Object.values(tables)) {
+    await sdk.rowActions.deleteAll(table._id!)
+  }
+}

From bbc9a2798de2f307f6aca87bf0fd4c35b7964939 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 12:08:44 +0200
Subject: [PATCH 393/455] Add test for internal

---
 .../src/api/routes/tests/rowAction.spec.ts      | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index 5246304906c..513953cbfde 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -1,7 +1,11 @@
 import _ from "lodash"
 import tk from "timekeeper"
 
-import { context, roles } from "@budibase/backend-core"
+import {
+  context,
+  DEFAULT_BB_DATASOURCE_ID,
+  roles,
+} from "@budibase/backend-core"
 import { automations } from "@budibase/pro"
 import {
   CreateRowActionRequest,
@@ -967,6 +971,17 @@ describe("/rowsActions", () => {
     })
 
     it.each([
+      [
+        "internal",
+        async () => {
+          await config.api.application.addSampleData(config.getAppId())
+          const tables = await config.api.table.fetch()
+          const table = tables.find(
+            t => t.sourceId === DEFAULT_BB_DATASOURCE_ID
+          )!
+          return table
+        },
+      ],
       [
         "external",
         async () => {

From ae6f2f93820f44bca9e6cd797b9ffa65efa8d1da Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 12:22:51 +0200
Subject: [PATCH 394/455] Fix test

---
 packages/server/src/api/routes/tests/rowAction.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index 513953cbfde..14a1812195b 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -974,6 +974,7 @@ describe("/rowsActions", () => {
       [
         "internal",
         async () => {
+          await config.newTenant()
           await config.api.application.addSampleData(config.getAppId())
           const tables = await config.api.table.fetch()
           const table = tables.find(
@@ -985,6 +986,7 @@ describe("/rowsActions", () => {
       [
         "external",
         async () => {
+          await config.newTenant()
           const ds = await config.createDatasource({
             datasource: await getDatasource(DatabaseName.POSTGRES),
           })

From 8dda1a51cfed47c2c56fdd6a75582c22f609a21d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 12:39:20 +0200
Subject: [PATCH 395/455] Remove row actions when unfetching tables

---
 packages/server/src/sdk/app/datasources/plus.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
index 31ec51c7289..a8a21e69060 100644
--- a/packages/server/src/sdk/app/datasources/plus.ts
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -9,6 +9,7 @@ import * as datasources from "./datasources"
 import tableSdk from "../tables"
 import { getIntegration } from "../../../integrations"
 import { context } from "@budibase/backend-core"
+import sdk from "../.."
 
 function checkForSchemaErrors(schema: Record<string, Table>) {
   const errors: Record<string, string> = {}
@@ -96,6 +97,15 @@ export async function buildSchemaFromSource(
   const datasource = await datasources.get(datasourceId)
 
   const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+
+  const oldTables = datasource.entities || {}
+  const tablesToRemove = Object.keys(oldTables).filter(
+    t => !Object.keys(tables).includes(t)
+  )
+  for (const table of tablesToRemove) {
+    await sdk.rowActions.deleteAll(oldTables[table]._id!)
+  }
+
   datasource.entities = tables
 
   datasources.setDefaultDisplayColumns(datasource)

From 47e16dd84467d3f9b72fad11d6e4563ce8d1e7b0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 13:07:13 +0200
Subject: [PATCH 396/455] Typing emitter

---
 .../server/src/api/controllers/row/index.ts   | 19 +++++++++--------
 .../server/src/events/AutomationEmitter.ts    |  4 ++--
 packages/types/src/sdk/koa.ts                 | 21 ++++++++++++++++++-
 3 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 4d40476b7c0..d3b1d3d5b29 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -7,6 +7,7 @@ import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTableID } from "../../../integrations/utils"
 import {
+  AutomationEventType,
   Ctx,
   DeleteRow,
   DeleteRowRequest,
@@ -64,15 +65,15 @@ export async function patch(
       ctx.throw(404, "Row not found")
     }
     ctx.status = 200
-    ctx.eventEmitter &&
-      ctx.eventEmitter.emitRow({
-        eventName: `row:update`,
-        appId,
-        row,
-        table,
-        oldRow,
-        user: sdk.users.getUserContextBindings(ctx.user),
-      })
+
+    ctx.eventEmitter?.emitRow({
+      eventName: AutomationEventType.ROW_UPDATE,
+      appId,
+      row,
+      table,
+      oldRow,
+      user: sdk.users.getUserContextBindings(ctx.user),
+    })
     ctx.message = `${table.name} updated successfully.`
     ctx.body = row
     gridSocket?.emitRowUpdate(ctx, row)
diff --git a/packages/server/src/events/AutomationEmitter.ts b/packages/server/src/events/AutomationEmitter.ts
index a63273bdc04..ec1ea7f28c6 100644
--- a/packages/server/src/events/AutomationEmitter.ts
+++ b/packages/server/src/events/AutomationEmitter.ts
@@ -1,12 +1,12 @@
 import { rowEmission, tableEmission } from "./utils"
 import mainEmitter from "./index"
 import env from "../environment"
-import { Table, Row, DocumentType, App } from "@budibase/types"
+import { Table, Row, DocumentType, App, ContextEmitter } from "@budibase/types"
 import { context } from "@budibase/backend-core"
 
 const MAX_AUTOMATIONS_ALLOWED = 5
 
-class AutomationEmitter {
+class AutomationEmitter implements ContextEmitter {
   chainCount: number
   metadata: { automationChainCount: number }
 
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index a7df7011719..8add047c0bc 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -1,5 +1,14 @@
 import { Context, Request } from "koa"
-import { User, Role, UserRoles, Account, ConfigType } from "../documents"
+import {
+  User,
+  Role,
+  UserRoles,
+  Account,
+  ConfigType,
+  Row,
+  Table,
+  AutomationEventType,
+} from "../documents"
 import { FeatureFlag, License } from "../sdk"
 import { Files } from "formidable"
 
@@ -40,6 +49,7 @@ export interface UserCtx<RequestBody = any, ResponseBody = any>
   extends Ctx<RequestBody, ResponseBody> {
   user: ContextUser
   roleId?: string
+  eventEmitter?: ContextEmitter
 }
 
 /**
@@ -49,3 +59,12 @@ export interface UserCtx<RequestBody = any, ResponseBody = any>
 export interface BBContext extends Ctx {
   user?: ContextUser
 }
+
+export interface ContextEmitter {
+  emitRow: (params: {
+    eventName: AutomationEventType
+    appId: string
+    row: Row
+    table?: Table
+  }) => Promise<void>
+}

From 342d70b3265a559f2d20337471a7ddc2dc4ee9c4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 13:34:47 +0200
Subject: [PATCH 397/455] Proper typing

---
 .../server/src/api/controllers/row/index.ts   | 46 +++++++++----------
 packages/server/src/db/linkedRows/index.ts    | 11 +----
 packages/server/src/events/BudibaseEmitter.ts | 16 +++++--
 packages/server/src/events/utils.ts           |  6 +--
 packages/types/src/core/events.ts             |  8 ++++
 packages/types/src/core/index.ts              |  1 +
 packages/types/src/sdk/koa.ts                 | 26 +++++++++--
 7 files changed, 68 insertions(+), 46 deletions(-)
 create mode 100644 packages/types/src/core/events.ts

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index d3b1d3d5b29..fe29d467001 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -7,11 +7,11 @@ import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTableID } from "../../../integrations/utils"
 import {
-  AutomationEventType,
   Ctx,
   DeleteRow,
   DeleteRowRequest,
   DeleteRows,
+  EventType,
   ExportRowsRequest,
   ExportRowsResponse,
   FieldType,
@@ -67,7 +67,7 @@ export async function patch(
     ctx.status = 200
 
     ctx.eventEmitter?.emitRow({
-      eventName: AutomationEventType.ROW_UPDATE,
+      eventName: EventType.ROW_UPDATE,
       appId,
       row,
       table,
@@ -104,14 +104,14 @@ export const save = async (ctx: UserCtx<Row, Row>) => {
         sdk.rows.save(sourceId, ctx.request.body, ctx.user?._id)
       )
   ctx.status = 200
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitRow({
-      eventName: `row:save`,
-      appId,
-      row,
-      table,
-      user: sdk.users.getUserContextBindings(ctx.user),
-    })
+
+  ctx.eventEmitter?.emitRow({
+    eventName: EventType.ROW_SAVE,
+    appId,
+    row,
+    table,
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   ctx.message = `${table.name} saved successfully`
   // prefer squashed for response
   ctx.body = row || squashed
@@ -183,13 +183,12 @@ async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
   }
 
   for (let row of rows) {
-    ctx.eventEmitter &&
-      ctx.eventEmitter.emitRow({
-        eventName: `row:delete`,
-        appId,
-        row,
-        user: sdk.users.getUserContextBindings(ctx.user),
-      })
+    ctx.eventEmitter?.emitRow({
+      eventName: EventType.ROW_DELETE,
+      appId,
+      row,
+      user: sdk.users.getUserContextBindings(ctx.user),
+    })
     gridSocket?.emitRowDeletion(ctx, row)
   }
   return rows
@@ -204,13 +203,12 @@ async function deleteRow(ctx: UserCtx<DeleteRowRequest>) {
     await quotas.removeRow()
   }
 
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitRow({
-      eventName: `row:delete`,
-      appId,
-      row: resp.row,
-      user: sdk.users.getUserContextBindings(ctx.user),
-    })
+  ctx.eventEmitter?.emitRow({
+    eventName: EventType.ROW_DELETE,
+    appId,
+    row: resp.row,
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   gridSocket?.emitRowDeletion(ctx, resp.row)
 
   return resp
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index fbb83b49a55..73ac6958788 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -17,6 +17,7 @@ import {
 import { context, features } from "@budibase/backend-core"
 import {
   ContextUser,
+  EventType,
   FeatureFlag,
   FieldType,
   LinkDocumentValue,
@@ -44,15 +45,7 @@ const INVALID_DISPLAY_COLUMN_TYPE = [
  * This functionality makes sure that when rows with links are created, updated or deleted they are processed
  * correctly - making sure that no stale links are left around and that all links have been made successfully.
  */
-
-export const EventType = {
-  ROW_SAVE: "row:save",
-  ROW_UPDATE: "row:update",
-  ROW_DELETE: "row:delete",
-  TABLE_SAVE: "table:save",
-  TABLE_UPDATED: "table:updated",
-  TABLE_DELETE: "table:delete",
-}
+export { EventType } from "@budibase/types"
 
 function clearRelationshipFields(schema: TableSchema, rows: Row[]) {
   for (let [key, field] of Object.entries(schema)) {
diff --git a/packages/server/src/events/BudibaseEmitter.ts b/packages/server/src/events/BudibaseEmitter.ts
index c8983096d0d..9fc5c2f9063 100644
--- a/packages/server/src/events/BudibaseEmitter.ts
+++ b/packages/server/src/events/BudibaseEmitter.ts
@@ -1,6 +1,12 @@
 import { EventEmitter } from "events"
 import { rowEmission, tableEmission } from "./utils"
-import { Table, Row, User } from "@budibase/types"
+import {
+  Table,
+  Row,
+  UserBindings,
+  EventType,
+  ContextEmitter,
+} from "@budibase/types"
 
 /**
  * keeping event emitter in one central location as it might be used for things other than
@@ -12,7 +18,7 @@ import { Table, Row, User } from "@budibase/types"
  * Extending the standard emitter to some syntactic sugar and standardisation to the emitted event.
  * This is specifically quite important for template strings used in automations.
  */
-class BudibaseEmitter extends EventEmitter {
+class BudibaseEmitter extends EventEmitter implements ContextEmitter {
   emitRow({
     eventName,
     appId,
@@ -21,17 +27,17 @@ class BudibaseEmitter extends EventEmitter {
     oldRow,
     user,
   }: {
-    eventName: string
+    eventName: EventType.ROW_SAVE | EventType.ROW_DELETE | EventType.ROW_UPDATE
     appId: string
     row: Row
     table?: Table
     oldRow?: Row
-    user: User
+    user: UserBindings
   }) {
     rowEmission({ emitter: this, eventName, appId, row, table, oldRow, user })
   }
 
-  emitTable(eventName: string, appId: string, table?: Table) {
+  emitTable(eventName: EventType, appId: string, table?: Table) {
     tableEmission({ emitter: this, eventName, appId, table })
   }
 
diff --git a/packages/server/src/events/utils.ts b/packages/server/src/events/utils.ts
index 5e4a1bebbfd..9cb5eef1875 100644
--- a/packages/server/src/events/utils.ts
+++ b/packages/server/src/events/utils.ts
@@ -1,4 +1,4 @@
-import { Table, Row, User } from "@budibase/types"
+import { Table, Row, UserBindings } from "@budibase/types"
 import BudibaseEmitter from "./BudibaseEmitter"
 
 type BBEventOpts = {
@@ -9,7 +9,7 @@ type BBEventOpts = {
   row?: Row
   oldRow?: Row
   metadata?: any
-  user?: User
+  user?: UserBindings
 }
 
 interface BBEventTable extends Table {
@@ -25,7 +25,7 @@ type BBEvent = {
   id?: string
   revision?: string
   metadata?: any
-  user?: User
+  user?: UserBindings
 }
 
 export function rowEmission({
diff --git a/packages/types/src/core/events.ts b/packages/types/src/core/events.ts
new file mode 100644
index 00000000000..3ce948b3b50
--- /dev/null
+++ b/packages/types/src/core/events.ts
@@ -0,0 +1,8 @@
+export const enum EventType {
+  ROW_SAVE = "row:save",
+  ROW_UPDATE = "row:update",
+  ROW_DELETE = "row:delete",
+  TABLE_SAVE = "table:save",
+  TABLE_UPDATED = "table:updated",
+  TABLE_DELETE = "table:delete",
+}
diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts
index b5cbd7affa3..73cc7d35e06 100644
--- a/packages/types/src/core/index.ts
+++ b/packages/types/src/core/index.ts
@@ -1 +1,2 @@
 export * from "./installation"
+export * from "./events"
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index 8add047c0bc..f0a81a78864 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -7,10 +7,11 @@ import {
   ConfigType,
   Row,
   Table,
-  AutomationEventType,
+  UserBindings,
 } from "../documents"
 import { FeatureFlag, License } from "../sdk"
 import { Files } from "formidable"
+import { EventType } from "../core"
 
 export interface ContextUser extends Omit<User, "roles"> {
   globalId?: string
@@ -61,10 +62,25 @@ export interface BBContext extends Ctx {
 }
 
 export interface ContextEmitter {
-  emitRow: (params: {
-    eventName: AutomationEventType
+  emitRow(values: {
+    eventName: EventType.ROW_SAVE
     appId: string
     row: Row
-    table?: Table
-  }) => Promise<void>
+    table: Table
+    user: UserBindings
+  }): void
+  emitRow(values: {
+    eventName: EventType.ROW_UPDATE
+    appId: string
+    row: Row
+    table: Table
+    oldRow: Row
+    user: UserBindings
+  }): void
+  emitRow(values: {
+    eventName: EventType.ROW_DELETE
+    appId: string
+    row: Row
+    user: UserBindings
+  }): void
 }

From 637c13c1af339313f99763ff10d1e1c6cb861581 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 13:44:20 +0200
Subject: [PATCH 398/455] Proper typing for emitTable

---
 packages/server/src/api/controllers/table/index.ts | 8 ++++----
 packages/types/src/sdk/koa.ts                      | 5 +++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 2f2f93bffe4..77c1f3923ae 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -16,6 +16,7 @@ import {
   BulkImportResponse,
   CsvToJsonRequest,
   CsvToJsonResponse,
+  EventType,
   FetchTablesResponse,
   FieldType,
   MigrateRequest,
@@ -129,8 +130,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   }
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
+  ctx.eventEmitter?.emitTable(EventType.TABLE_SAVE, appId, { ...savedTable })
   ctx.body = savedTable
 
   savedTable = await processTable(savedTable)
@@ -143,8 +143,8 @@ export async function destroy(ctx: UserCtx) {
   await sdk.rowActions.deleteAll(tableId)
   const deletedTable = await pickApi({ tableId }).destroy(ctx)
   await events.table.deleted(deletedTable)
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:delete`, appId, deletedTable)
+
+  ctx.eventEmitter?.emitTable(EventType.TABLE_DELETE, appId, deletedTable)
   ctx.status = 200
   ctx.table = deletedTable
   ctx.body = { message: `Table ${tableId} deleted.` }
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index f0a81a78864..95ea2b652f6 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -83,4 +83,9 @@ export interface ContextEmitter {
     row: Row
     user: UserBindings
   }): void
+  emitTable(
+    eventName: EventType.TABLE_SAVE | EventType.TABLE_DELETE,
+    appId: string,
+    table?: Table
+  ): void
 }

From 4dedde5165b9c876480e8077f5bbe63ad55d9ee3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 13:51:44 +0200
Subject: [PATCH 399/455] Add missing rowaction

---
 .../server/src/events/AutomationEmitter.ts     | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/events/AutomationEmitter.ts b/packages/server/src/events/AutomationEmitter.ts
index ec1ea7f28c6..a95acd0877e 100644
--- a/packages/server/src/events/AutomationEmitter.ts
+++ b/packages/server/src/events/AutomationEmitter.ts
@@ -1,7 +1,15 @@
 import { rowEmission, tableEmission } from "./utils"
 import mainEmitter from "./index"
 import env from "../environment"
-import { Table, Row, DocumentType, App, ContextEmitter } from "@budibase/types"
+import {
+  Table,
+  Row,
+  DocumentType,
+  App,
+  ContextEmitter,
+  EventType,
+  UserBindings,
+} from "@budibase/types"
 import { context } from "@budibase/backend-core"
 
 const MAX_AUTOMATIONS_ALLOWED = 5
@@ -36,11 +44,15 @@ class AutomationEmitter implements ContextEmitter {
     appId,
     row,
     table,
+    oldRow,
+    user,
   }: {
-    eventName: string
+    eventName: EventType.ROW_SAVE | EventType.ROW_DELETE | EventType.ROW_UPDATE
     appId: string
     row: Row
     table?: Table
+    oldRow?: Row
+    user: UserBindings
   }) {
     let MAX_AUTOMATION_CHAIN = await this.getMaxAutomationChain()
 
@@ -54,7 +66,9 @@ class AutomationEmitter implements ContextEmitter {
       appId,
       row,
       table,
+      oldRow,
       metadata: this.metadata,
+      user,
     })
   }
 

From f279a1716df4d9ec1aad8211d9a79c1d918854a1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 15:52:52 +0200
Subject: [PATCH 400/455] Allow overriding the min version where we stop
 serving the power role

---
 packages/backend-core/src/environment.ts    | 2 ++
 packages/backend-core/src/security/roles.ts | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 7d3a9f18f56..4cb0a9c7315 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -223,6 +223,8 @@ const environment = {
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  MIN_VERSION_WITHOUT_POWER_ROLE:
+    process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
 }
 
 export function setEnv(newEnvVars: Partial<typeof environment>): () => void {
diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index df7f41e6cea..76c4f08fbea 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -19,6 +19,7 @@ import {
 import cloneDeep from "lodash/fp/cloneDeep"
 import { RoleColor, helpers } from "@budibase/shared-core"
 import { uniqBy } from "lodash"
+import { default as env } from "../environment"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",
@@ -545,7 +546,10 @@ async function shouldIncludePowerRole(db: Database) {
     return true
   }
 
-  const isGreaterThan3x = semver.gte(creationVersion, "3.0.0")
+  const isGreaterThan3x = semver.gte(
+    creationVersion,
+    env.MIN_VERSION_WITHOUT_POWER_ROLE
+  )
   return !isGreaterThan3x
 }
 

From 288517df96688b04b57c51d71da1d00fb163e971 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 24 Oct 2024 16:08:04 +0200
Subject: [PATCH 401/455] Fix frontend issues due power role missing

---
 .../backend/DataTable/buttons/ManageAccessButton.svelte       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 2826d8d9860..17c26847b36 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -34,7 +34,9 @@
   $: buttonLabel = readableRole ? `Access: ${readableRole}` : "Access"
   $: highlight = roleMismatch || selectedRole === Roles.PUBLIC
 
-  $: builtInRoles = builtins.map(roleId => $roles.find(x => x._id === roleId))
+  $: builtInRoles = builtins
+    .map(roleId => $roles.find(x => x._id === roleId))
+    .filter(r => !!r)
   $: customRoles = $roles
     .filter(x => !builtins.includes(x._id))
     .slice()

From c8a783786834e39b6c653dd93ca27d1b76f0efb7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 15:30:05 +0100
Subject: [PATCH 402/455] Skip access API request if unchanged

---
 .../backend/DataTable/buttons/ManageAccessButton.svelte        | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
index 17c26847b36..85227b91404 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte
@@ -102,6 +102,9 @@
   }
 
   const changePermission = async role => {
+    if (role === selectedRole) {
+      return
+    }
     try {
       await permissionsStore.save({
         level: "read",

From be5564621c0339aa17e52436a511ae232914e7af Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 16:19:43 +0100
Subject: [PATCH 403/455] Add info display in user side panel to warn builders
 if they haven't published their app

---
 .../src/components/deploy/AppActions.svelte   | 20 +++++++---------
 .../_components/BuilderSidePanel.svelte       | 23 +++++++++++++++----
 .../_components/Component/InfoDisplay.svelte  | 23 +++++++++++--------
 packages/builder/src/stores/builder/index.js  |  2 ++
 .../builder/src/stores/builder/published.js   | 13 +++++++++++
 5 files changed, 56 insertions(+), 25 deletions(-)
 create mode 100644 packages/builder/src/stores/builder/published.js

diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte
index bb950983a6d..9ba46832f4b 100644
--- a/packages/builder/src/components/deploy/AppActions.svelte
+++ b/packages/builder/src/components/deploy/AppActions.svelte
@@ -25,6 +25,7 @@
     appStore,
     deploymentStore,
     sortedScreens,
+    appPublished,
   } from "stores/builder"
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
@@ -45,11 +46,6 @@
 
   $: filteredApps = $appsStore.apps.filter(app => app.devId === application)
   $: selectedApp = filteredApps?.length ? filteredApps[0] : null
-  $: latestDeployments = $deploymentStore
-    .filter(deployment => deployment.status === "SUCCESS")
-    .sort((a, b) => a.updatedAt > b.updatedAt)
-  $: isPublished =
-    selectedApp?.status === "published" && latestDeployments?.length > 0
   $: updateAvailable =
     $appStore.upgradableVersion &&
     $appStore.version &&
@@ -117,7 +113,7 @@
   }
 
   const confirmUnpublishApp = async () => {
-    if (!application || !isPublished) {
+    if (!application || !$appPublished) {
       //confirm the app has loaded.
       return
     }
@@ -204,7 +200,7 @@
     >
       <div bind:this={appActionPopoverAnchor}>
         <div class="app-action">
-          <Icon name={isPublished ? "GlobeCheck" : "GlobeStrike"} />
+          <Icon name={$appPublished ? "GlobeCheck" : "GlobeStrike"} />
           <TourWrap stepKeys={[TOUR_STEP_KEYS.BUILDER_APP_PUBLISH]}>
             <span class="publish-open" id="builder-app-publish-button">
               Publish
@@ -219,7 +215,7 @@
       <Popover
         bind:this={appActionPopover}
         align="right"
-        disabled={!isPublished}
+        disabled={!$appPublished}
         anchor={appActionPopoverAnchor}
         offset={35}
         on:close={() => {
@@ -236,13 +232,13 @@
                 class="app-link"
                 on:click={() => {
                   appActionPopover.hide()
-                  if (isPublished) {
+                  if ($appPublished) {
                     viewApp()
                   }
                 }}
               >
                 {$appStore.url}
-                {#if isPublished}
+                {#if $appPublished}
                   <Icon size="S" name="LinkOut" />
                 {/if}
               </span>
@@ -250,7 +246,7 @@
 
             <Body size="S">
               <span class="publish-popover-status">
-                {#if isPublished}
+                {#if $appPublished}
                   <span class="status-text">
                     {lastDeployed}
                   </span>
@@ -279,7 +275,7 @@
               </span>
             </Body>
             <div class="action-buttons">
-              {#if isPublished}
+              {#if $appPublished}
                 <ActionButton
                   quiet
                   icon="Code"
diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
index 035dc5a2ef7..687678c08ac 100644
--- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
@@ -14,7 +14,7 @@
     Button,
     FancySelect,
   } from "@budibase/bbui"
-  import { builderStore, appStore, roles } from "stores/builder"
+  import { builderStore, appStore, roles, appPublished } from "stores/builder"
   import {
     groups,
     licensing,
@@ -36,6 +36,7 @@
   import UpgradeModal from "components/common/users/UpgradeModal.svelte"
   import { emailValidator } from "helpers/validation"
   import { fly } from "svelte/transition"
+  import InfoDisplay from "../design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
 
   let query = null
   let loaded = false
@@ -608,6 +609,17 @@
     </div>
 
     <div class="body">
+      {#if !$appPublished}
+        <div class="alert">
+          <InfoDisplay
+            icon="AlertCircleFilled"
+            warning
+            title="App unpublished"
+            body="Users won't be able to access your app until you've published it"
+          />
+        </div>
+      {/if}
+
       {#if promptInvite && !userOnboardResponse}
         <Layout gap="S" paddingX="XL">
           <div class="invite-header">
@@ -623,7 +635,7 @@
       {/if}
 
       {#if !promptInvite}
-        <Layout gap="L" noPadding>
+        <Layout gap="M" noPadding>
           {#if filteredInvites?.length}
             <Layout noPadding gap="XS">
               <div class="auth-entity-header">
@@ -926,7 +938,7 @@
   .auth-entity,
   .auth-entity-header {
     display: grid;
-    grid-template-columns: 1fr 180px;
+    grid-template-columns: 1fr 220px;
     align-items: center;
     gap: var(--spacing-xl);
   }
@@ -957,7 +969,7 @@
     overflow-y: auto;
     overflow-x: hidden;
     position: absolute;
-    width: 440px;
+    width: 480px;
     right: 0;
     height: 100%;
     box-shadow: 0 0 40px 10px rgba(0, 0, 0, 0.1);
@@ -1034,4 +1046,7 @@
     gap: var(--spacing-xl);
     padding: var(--spacing-xl) 0;
   }
+  .alert {
+    padding: 0 var(--spacing-xl);
+  }
 </style>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
index 88ddf6f9a53..5543a4c7fd0 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
@@ -7,9 +7,17 @@
   export let quiet = false
   export let warning = false
   export let error = false
+  export let info = false
 </script>
 
-<div class="info" class:noTitle={!title} class:warning class:error class:quiet>
+<div
+  class="container"
+  class:info
+  class:noTitle={!title}
+  class:warning
+  class:error
+  class:quiet
+>
   {#if title}
     <div class="title">
       <Icon name={icon} />
@@ -27,7 +35,7 @@
 </div>
 
 <style>
-  .info {
+  .container {
     padding: var(--spacing-m) var(--spacing-l);
     background-color: var(--spectrum-global-color-gray-200);
     border-radius: var(--border-radius-s);
@@ -39,6 +47,9 @@
   .error {
     background: rgba(255, 0, 0, 0.2);
   }
+  .info {
+    background: var(--spectrum-global-color-blue-100);
+  }
   .noTitle {
     display: flex;
     align-items: center;
@@ -57,13 +68,7 @@
   }
   .title,
   .icon {
-    color: var(--spectrum-global-color-gray-600);
-  }
-  .info {
-    background-color: var(--background-alt);
-    padding: var(--spacing-m) var(--spacing-l) var(--spacing-m) var(--spacing-l);
-    border-radius: var(--border-radius-s);
-    font-size: 13px;
+    color: var(--spectrum-global-color-gray-700);
   }
   .quiet {
     background: none;
diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js
index 158cd29973c..08d87bebf5b 100644
--- a/packages/builder/src/stores/builder/index.js
+++ b/packages/builder/src/stores/builder/index.js
@@ -30,6 +30,7 @@ import { queries } from "./queries"
 import { flags } from "./flags"
 import { rowActions } from "./rowActions"
 import componentTreeNodesStore from "./componentTreeNodes"
+import { appPublished } from "./published"
 
 export {
   componentTreeNodesStore,
@@ -65,6 +66,7 @@ export {
   hoverStore,
   snippets,
   rowActions,
+  appPublished,
 }
 
 export const reset = () => {
diff --git a/packages/builder/src/stores/builder/published.js b/packages/builder/src/stores/builder/published.js
new file mode 100644
index 00000000000..6ae9392f40c
--- /dev/null
+++ b/packages/builder/src/stores/builder/published.js
@@ -0,0 +1,13 @@
+import { appStore } from "./app"
+import { appsStore } from "stores/portal/apps"
+import { deploymentStore } from "./deployments"
+import { derived } from "svelte/store"
+
+export const appPublished = derived(
+  [appStore, appsStore, deploymentStore],
+  ([$appStore, $appsStore, $deploymentStore]) => {
+    const app = $appsStore.apps.find(app => app.devId === $appStore.appId)
+    const deployments = $deploymentStore.filter(x => x.status === "SUCCESS")
+    return app?.status === "published" && deployments.length > 0
+  }
+)

From 2dec33c12d57d00a7c838d6fcd95d37b3ba30380 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 24 Oct 2024 16:27:30 +0100
Subject: [PATCH 404/455] Revert info display changes

---
 .../_components/Component/InfoDisplay.svelte      | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
index 5543a4c7fd0..93044cdb9af 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte
@@ -7,17 +7,9 @@
   export let quiet = false
   export let warning = false
   export let error = false
-  export let info = false
 </script>
 
-<div
-  class="container"
-  class:info
-  class:noTitle={!title}
-  class:warning
-  class:error
-  class:quiet
->
+<div class="info" class:noTitle={!title} class:warning class:error class:quiet>
   {#if title}
     <div class="title">
       <Icon name={icon} />
@@ -35,7 +27,7 @@
 </div>
 
 <style>
-  .container {
+  .info {
     padding: var(--spacing-m) var(--spacing-l);
     background-color: var(--spectrum-global-color-gray-200);
     border-radius: var(--border-radius-s);
@@ -47,9 +39,6 @@
   .error {
     background: rgba(255, 0, 0, 0.2);
   }
-  .info {
-    background: var(--spectrum-global-color-blue-100);
-  }
   .noTitle {
     display: flex;
     align-items: center;

From 5e4cfafa9620b83c7f1b3bb33b8655d497a9a044 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 25 Oct 2024 11:53:27 +0100
Subject: [PATCH 405/455] Expose bigints as a column to do calculations on.

---
 .../buttons/grid/GridViewCalculationButton.svelte    | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 72216f3b3be..db58ba4a8f2 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -7,7 +7,12 @@
     Icon,
     Multiselect,
   } from "@budibase/bbui"
-  import { CalculationType, canGroupBy, FieldType } from "@budibase/types"
+  import {
+    CalculationType,
+    canGroupBy,
+    FieldType,
+    isNumeric,
+  } from "@budibase/types"
   import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
   import { getContext } from "svelte"
 
@@ -90,10 +95,7 @@
     return Object.entries(schema)
       .filter(([field, fieldSchema]) => {
         // Only allow numeric fields that are not calculations themselves
-        if (
-          fieldSchema.calculationType ||
-          fieldSchema.type !== FieldType.NUMBER
-        ) {
+        if (fieldSchema.calculationType || !isNumeric(fieldSchema.type)) {
           return false
         }
         // Don't allow duplicates

From f46a6684657e043fb6bac68d83f04603e7215cd5 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 25 Oct 2024 15:55:07 +0100
Subject: [PATCH 406/455] Allow bigints to be used in view calculations, and
 make sure they're accurate up to 64bit signed int via tests.

---
 packages/backend-core/src/sql/sql.ts          |  65 +++++---
 .../src/api/routes/tests/viewV2.spec.ts       | 140 +++++++++++++-----
 .../src/utilities/rowProcessor/index.ts       |  27 ++--
 3 files changed, 160 insertions(+), 72 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 88953bbf990..91dceaf5a66 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -179,12 +179,6 @@ class InternalBuilder {
     return this.table.schema[column]
   }
 
-  private supportsILike(): boolean {
-    return !(
-      this.client === SqlClient.ORACLE || this.client === SqlClient.SQL_LITE
-    )
-  }
-
   private quoteChars(): [string, string] {
     const wrapped = this.knexClient.wrapIdentifier("foo", {})
     return [wrapped[0], wrapped[wrapped.length - 1]]
@@ -216,8 +210,30 @@ class InternalBuilder {
     return formatter.wrap(value, false)
   }
 
-  private rawQuotedValue(value: string): Knex.Raw {
-    return this.knex.raw(this.quotedValue(value))
+  private castIntToString(identifier: string | Knex.Raw): Knex.Raw {
+    switch (this.client) {
+      case SqlClient.ORACLE: {
+        return this.knex.raw("to_char(??)", [identifier])
+      }
+      case SqlClient.POSTGRES: {
+        return this.knex.raw("??::TEXT", [identifier])
+      }
+      case SqlClient.MY_SQL:
+      case SqlClient.MARIADB: {
+        return this.knex.raw("CAST(?? AS CHAR)", [identifier])
+      }
+      case SqlClient.SQL_LITE: {
+        // Technically sqlite can actually represent numbers larger than a 64bit
+        // int as a string, but it does it using scientific notation (e.g.
+        // "1e+20") which is not what we want. Given that the external SQL
+        // databases are limited to supporting only 64bit ints, we settle for
+        // that here.
+        return this.knex.raw("printf('%d', ??)", [identifier])
+      }
+      case SqlClient.MS_SQL: {
+        return this.knex.raw("CONVERT(NVARCHAR, ??)", [identifier])
+      }
+    }
   }
 
   // Unfortuantely we cannot rely on knex's identifier escaping because it trims
@@ -1078,21 +1094,26 @@ class InternalBuilder {
           query = query.count(`* as ${aggregation.name}`)
         }
       } else {
-        const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
-        switch (op) {
-          case CalculationType.SUM:
-            query = query.sum(field)
-            break
-          case CalculationType.AVG:
-            query = query.avg(field)
-            break
-          case CalculationType.MIN:
-            query = query.min(field)
-            break
-          case CalculationType.MAX:
-            query = query.max(field)
-            break
+        const fieldSchema = this.getFieldSchema(aggregation.field)
+        if (!fieldSchema) {
+          // This should not happen in practice.
+          throw new Error(
+            `field schema missing for aggregation target: ${aggregation.field}`
+          )
+        }
+
+        let aggregate = this.knex.raw("??(??)", [
+          this.knex.raw(op),
+          this.rawQuotedIdentifier(`${tableName}.${aggregation.field}`),
+        ])
+
+        if (fieldSchema.type === FieldType.BIGINT) {
+          aggregate = this.castIntToString(aggregate)
         }
+
+        query = query.select(
+          this.knex.raw("?? as ??", [aggregate, aggregation.name])
+        )
       }
     }
     return query
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 2af11b513bc..44e5875b7e8 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -2268,58 +2268,118 @@ describe.each([
       })
     })
 
-    describe("calculation views", () => {
-      it("should not remove calculation columns when modifying table schema", async () => {
-        let table = await config.api.table.save(
-          saveTableRequest({
+    !isLucene &&
+      describe("calculation views", () => {
+        it("should not remove calculation columns when modifying table schema", async () => {
+          let table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                name: {
+                  name: "name",
+                  type: FieldType.STRING,
+                },
+                age: {
+                  name: "age",
+                  type: FieldType.NUMBER,
+                },
+              },
+            })
+          )
+
+          let view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            type: ViewV2Type.CALCULATION,
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "age",
+              },
+            },
+          })
+
+          table = await config.api.table.get(table._id!)
+          await config.api.table.save({
+            ...table,
             schema: {
+              ...table.schema,
               name: {
                 name: "name",
                 type: FieldType.STRING,
-              },
-              age: {
-                name: "age",
-                type: FieldType.NUMBER,
+                constraints: { presence: true },
               },
             },
           })
-        )
 
-        let view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          type: ViewV2Type.CALCULATION,
-          schema: {
-            sum: {
-              visible: true,
-              calculationType: CalculationType.SUM,
-              field: "age",
-            },
-          },
+          view = await config.api.viewV2.get(view.id)
+          expect(Object.keys(view.schema!).sort()).toEqual([
+            "age",
+            "id",
+            "name",
+            "sum",
+          ])
         })
 
-        table = await config.api.table.get(table._id!)
-        await config.api.table.save({
-          ...table,
-          schema: {
-            ...table.schema,
-            name: {
-              name: "name",
-              type: FieldType.STRING,
-              constraints: { presence: true },
-            },
-          },
-        })
+        describe("bigints", () => {
+          let table: Table
+          let view: ViewV2
 
-        view = await config.api.viewV2.get(view.id)
-        expect(Object.keys(view.schema!).sort()).toEqual([
-          "age",
-          "id",
-          "name",
-          "sum",
-        ])
+          beforeEach(async () => {
+            table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  bigint: {
+                    name: "bigint",
+                    type: FieldType.BIGINT,
+                  },
+                },
+              })
+            )
+
+            view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "bigint",
+                },
+              },
+            })
+          })
+
+          it("should not lose precision handling ints larger than JSs int53", async () => {
+            // The sum of the following 3 numbers cannot be represented by
+            // JavaScripts default int53 datatype for numbers, so this is a test
+            // that makes sure we aren't losing precision between the DB and the
+            // user.
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                { bigint: "1000000000000000000" },
+                { bigint: "123" },
+                { bigint: "321" },
+              ],
+            })
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual("1000000000000000444")
+          })
+
+          it("should be able to handle up to 2**63 - 1 bigints", async () => {
+            await config.api.row.bulkImport(table._id!, {
+              rows: [{ bigint: "9223372036854775806" }, { bigint: "1" }],
+            })
+
+            const { rows } = await config.api.row.search(view.id)
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual("9223372036854775807")
+          })
+        })
       })
-    })
   })
 
   describe("row operations", () => {
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index b9f4d0db8b7..0ffa7600aad 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -440,19 +440,26 @@ export async function coreOutputProcessing(
     }
 
     if (sdk.views.isView(source)) {
-      const calculationFields = Object.keys(
-        helpers.views.calculationFields(source)
-      )
-
-      // We ensure all calculation fields are returned as numbers.  During the
+      // We ensure calculation fields are returned as numbers.  During the
       // testing of this feature it was discovered that the COUNT operation
       // returns a string for MySQL, MariaDB, and Postgres. But given that all
-      // calculation fields should be numbers, we blanket make sure of that
-      // here.
-      for (const key of calculationFields) {
+      // calculation fields (except ones operating on BIGINTs) should be
+      // numbers, we blanket make sure of that here.
+      for (const [name, field] of Object.entries(
+        helpers.views.calculationFields(source)
+      )) {
+        if ("field" in field) {
+          const targetSchema = table.schema[field.field]
+          // We don't convert BIGINT fields to floats because we could lose
+          // precision.
+          if (targetSchema.type === FieldType.BIGINT) {
+            continue
+          }
+        }
+
         for (const row of rows) {
-          if (typeof row[key] === "string") {
-            row[key] = parseFloat(row[key])
+          if (typeof row[name] === "string") {
+            row[name] = parseFloat(row[name])
           }
         }
       }

From dae550c21e00b4a97b74a137b743dc9c2fecc558 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 25 Oct 2024 16:24:40 +0100
Subject: [PATCH 407/455] Tidy up import data modal and change to be a detail
 popover

---
 .../DataTable/buttons/ImportButton.svelte     |  92 ++++++++++--
 .../DataTable/modals/ImportModal.svelte       |  61 --------
 .../TableNavigator/TableDataImport.svelte     | 134 ++++++++----------
 3 files changed, 145 insertions(+), 142 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
index 88c57daa211..5e9850b290f 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
@@ -1,17 +1,91 @@
 <script>
-  import { ActionButton, Modal } from "@budibase/bbui"
-  import ImportModal from "../modals/ImportModal.svelte"
+  import {
+    ActionButton,
+    Label,
+    Button,
+    Body,
+    Layout,
+    notifications,
+  } from "@budibase/bbui"
+  import DetailPopover from "components/common/DetailPopover.svelte"
+  import TableDataImport from "components/backend/TableNavigator/TableDataImport.svelte"
+  import { createEventDispatcher } from "svelte"
+  import { API } from "api"
 
   export let tableId
   export let tableType
   export let disabled
 
-  let modal
+  const dispatch = createEventDispatcher()
+
+  let popover
+  let rows = []
+  let allValid = false
+  let displayColumn = null
+  let identifierFields = []
+  let loading = false
+
+  const openPopover = () => {
+    rows = []
+    allValid = false
+    displayColumn = null
+    identifierFields = []
+    loading = false
+    popover.show()
+  }
+
+  const importData = async () => {
+    try {
+      loading = true
+      await API.importTableData({
+        tableId,
+        rows,
+        identifierFields,
+      })
+      notifications.success("Rows successfully imported")
+      popover.hide()
+    } catch (error) {
+      console.error(error)
+      notifications.error("Unable to import data")
+    } finally {
+      loading = false
+    }
+
+    // Always refresh rows just to be sure
+    dispatch("importrows")
+  }
 </script>
 
-<ActionButton icon="DataUpload" quiet on:click={modal.show} {disabled}>
-  Import
-</ActionButton>
-<Modal bind:this={modal}>
-  <ImportModal {tableId} {tableType} on:importrows />
-</Modal>
+<DetailPopover title="Import data" bind:this={popover}>
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton
+      icon="DataUpload"
+      quiet
+      on:click={openPopover}
+      {disabled}
+      selected={open}
+    >
+      Import
+    </ActionButton>
+  </svelte:fragment>
+  <Body size="S">
+    Import rows to an existing table from a CSV or JSON file. Only columns from
+    the file which exist in the table will be imported.
+  </Body>
+  <Layout gap="XS" noPadding>
+    <Label grey extraSmall>CSV or JSON file to import</Label>
+    <TableDataImport
+      {tableId}
+      {tableType}
+      bind:rows
+      bind:allValid
+      bind:displayColumn
+      bind:identifierFields
+    />
+  </Layout>
+  <div>
+    <Button cta disabled={loading || !allValid} on:click={importData}>
+      Import
+    </Button>
+  </div>
+</DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
deleted file mode 100644
index 1696c6ba03d..00000000000
--- a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
+++ /dev/null
@@ -1,61 +0,0 @@
-<script>
-  import {
-    ModalContent,
-    Label,
-    notifications,
-    Body,
-    Layout,
-  } from "@budibase/bbui"
-  import TableDataImport from "../../TableNavigator/ExistingTableDataImport.svelte"
-  import { API } from "api"
-  import { createEventDispatcher } from "svelte"
-
-  const dispatch = createEventDispatcher()
-
-  export let tableId
-  export let tableType
-
-  let rows = []
-  let allValid = false
-  let displayColumn = null
-  let identifierFields = []
-
-  async function importData() {
-    try {
-      await API.importTableData({
-        tableId,
-        rows,
-        identifierFields,
-      })
-      notifications.success("Rows successfully imported")
-    } catch (error) {
-      notifications.error("Unable to import data")
-    }
-
-    // Always refresh rows just to be sure
-    dispatch("importrows")
-  }
-</script>
-
-<ModalContent
-  title="Import Data"
-  confirmText="Import"
-  onConfirm={importData}
-  disabled={!allValid}
->
-  <Body size="S">
-    Import rows to an existing table from a CSV or JSON file. Only columns from
-    the file which exist in the table will be imported.
-  </Body>
-  <Layout gap="XS" noPadding>
-    <Label grey extraSmall>CSV or JSON file to import</Label>
-    <TableDataImport
-      {tableId}
-      {tableType}
-      bind:rows
-      bind:allValid
-      bind:displayColumn
-      bind:identifierFields
-    />
-  </Layout>
-</ModalContent>
diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 5804dc3172a..77437628774 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Select, Icon } from "@budibase/bbui"
+  import { Select, Icon, Layout } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { utils } from "@budibase/shared-core"
   import { canBeDisplayColumn } from "@budibase/frontend-core"
@@ -184,70 +184,71 @@
   }
 </script>
 
-<div class="dropzone">
-  <input
-    bind:this={fileInput}
-    disabled={loading}
-    id="file-upload"
-    accept="text/csv,application/json"
-    type="file"
-    on:change={handleFile}
-  />
-  <label for="file-upload" class:uploaded={rawRows.length > 0}>
-    {#if error}
-      Error: {error}
-    {:else if fileName}
-      {fileName}
-    {:else}
-      Upload
-    {/if}
-  </label>
-</div>
-{#if rawRows.length > 0 && !error}
-  <div class="schema-fields">
-    {#each Object.entries(schema) as [name, column]}
-      <div class="field">
-        <span>{column.name}</span>
-        <Select
-          bind:value={selectedColumnTypes[column.name]}
-          on:change={e => handleChange(name, e)}
-          options={Object.values(typeOptions)}
-          placeholder={null}
-          getOptionLabel={option => option.label}
-          getOptionValue={option => option.value}
-        />
-        <span
-          class={validation[column.name]
-            ? "fieldStatusSuccess"
-            : "fieldStatusFailure"}
-        >
-          {#if validation[column.name]}
-            Success
-          {:else}
-            Failure
-            {#if errors[column.name]}
-              <Icon name="Help" tooltip={errors[column.name]} />
-            {/if}
-          {/if}
-        </span>
-        <Icon
-          size="S"
-          name="Close"
-          hoverable
-          on:click={() => deleteColumn(column.name)}
-        />
-      </div>
-    {/each}
+<Layout noPadding gap="S">
+  <div class="dropzone">
+    <input
+      bind:this={fileInput}
+      disabled={loading}
+      id="file-upload"
+      accept="text/csv,application/json"
+      type="file"
+      on:change={handleFile}
+    />
+    <label for="file-upload" class:uploaded={rawRows.length > 0}>
+      {#if error}
+        Error: {error}
+      {:else if fileName}
+        {fileName}
+      {:else}
+        Upload
+      {/if}
+    </label>
   </div>
-  <div class="display-column">
+
+  {#if rawRows.length > 0 && !error}
+    <div>
+      {#each Object.entries(schema) as [name, column]}
+        <div class="field">
+          <span>{column.name}</span>
+          <Select
+            bind:value={selectedColumnTypes[column.name]}
+            on:change={e => handleChange(name, e)}
+            options={Object.values(typeOptions)}
+            placeholder={null}
+            getOptionLabel={option => option.label}
+            getOptionValue={option => option.value}
+          />
+          <span
+            class={validation[column.name]
+              ? "fieldStatusSuccess"
+              : "fieldStatusFailure"}
+          >
+            {#if validation[column.name]}
+              Success
+            {:else}
+              Failure
+              {#if errors[column.name]}
+                <Icon name="Help" tooltip={errors[column.name]} />
+              {/if}
+            {/if}
+          </span>
+          <Icon
+            size="S"
+            name="Close"
+            hoverable
+            on:click={() => deleteColumn(column.name)}
+          />
+        </div>
+      {/each}
+    </div>
     <Select
       label="Display Column"
       bind:value={displayColumn}
       options={displayColumnOptions}
       sort
     />
-  </div>
-{/if}
+  {/if}
+</Layout>
 
 <style>
   .dropzone {
@@ -269,7 +270,6 @@
     box-sizing: border-box;
     overflow: hidden;
     border-radius: var(--border-radius-s);
-    color: var(--ink);
     padding: var(--spacing-m) var(--spacing-l);
     transition: all 0.2s ease 0s;
     display: inline-flex;
@@ -283,20 +283,14 @@
     align-items: center;
     justify-content: center;
     width: 100%;
-    background-color: var(--grey-2);
-    font-size: var(--font-size-xs);
+    background-color: var(--spectrum-global-color-gray-300);
+    font-size: var(--font-size-s);
     line-height: normal;
     border: var(--border-transparent);
   }
-
   .uploaded {
-    color: var(--blue);
+    color: var(--spectrum-global-color-blue-600);
   }
-
-  .schema-fields {
-    margin-top: var(--spacing-xl);
-  }
-
   .field {
     display: grid;
     grid-template-columns: 2fr 2fr 1fr auto;
@@ -322,8 +316,4 @@
   .fieldStatusFailure :global(.spectrum-Icon) {
     width: 12px;
   }
-
-  .display-column {
-    margin-top: var(--spacing-xl);
-  }
 </style>

From 9ab266a96368fbd68666695931ad20517d780263 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 25 Oct 2024 16:38:24 +0100
Subject: [PATCH 408/455] Fix lint.

---
 .../buttons/grid/GridViewCalculationButton.svelte          | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index db58ba4a8f2..87c77a1457a 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -7,12 +7,7 @@
     Icon,
     Multiselect,
   } from "@budibase/bbui"
-  import {
-    CalculationType,
-    canGroupBy,
-    FieldType,
-    isNumeric,
-  } from "@budibase/types"
+  import { CalculationType, canGroupBy, isNumeric } from "@budibase/types"
   import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
   import { getContext } from "svelte"
 

From 271023823b351a9184dff90665a1773ba17277ac Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 25 Oct 2024 16:49:45 +0100
Subject: [PATCH 409/455] Tidy up export modal and update to be a detail
 popover

---
 .../DataTable/buttons/ExportButton.svelte     | 144 +++++++++++-
 .../buttons/grid/GridExportButton.svelte      |  28 +--
 .../DataTable/modals/ExportModal.svelte       | 217 ------------------
 3 files changed, 144 insertions(+), 245 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
index 4fa1d07abd3..e9352045ea1 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
@@ -1,20 +1,144 @@
 <script>
-  import { ActionButton, Modal } from "@budibase/bbui"
-  import ExportModal from "../modals/ExportModal.svelte"
+  import {
+    ActionButton,
+    Select,
+    notifications,
+    Body,
+    Button,
+  } from "@budibase/bbui"
+  import download from "downloadjs"
+  import { API } from "api"
+  import { ROW_EXPORT_FORMATS } from "constants/backend"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   export let view
-  export let filters
   export let sorting
   export let disabled = false
   export let selectedRows
   export let formats
 
-  let modal
+  const FORMATS = [
+    {
+      name: "CSV",
+      key: ROW_EXPORT_FORMATS.CSV,
+    },
+    {
+      name: "JSON",
+      key: ROW_EXPORT_FORMATS.JSON,
+    },
+    {
+      name: "JSON with Schema",
+      key: ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA,
+    },
+  ]
+
+  let popover
+  let exportFormat
+  let loading = false
+
+  $: options = FORMATS.filter(format => {
+    if (formats && !formats.includes(format.key)) {
+      return false
+    }
+    return true
+  })
+  $: if (options && !exportFormat) {
+    exportFormat = Array.isArray(options) ? options[0]?.key : []
+  }
+
+  const openPopover = () => {
+    loading = false
+    popover.show()
+  }
+
+  function downloadWithBlob(data, filename) {
+    download(new Blob([data], { type: "text/plain" }), filename)
+  }
+
+  const exportAllData = async () => {
+    return await API.exportView({
+      viewName: view,
+      format: exportFormat,
+    })
+  }
+
+  const exportFilteredData = async () => {
+    let payload = {
+      tableId: view,
+      format: exportFormat,
+      search: {
+        paginate: false,
+      },
+    }
+    if (selectedRows?.length) {
+      payload.rows = selectedRows.map(row => row._id)
+    }
+    if (sorting) {
+      payload.search.sort = sorting.sortColumn
+      payload.search.sortOrder = sorting.sortOrder
+    }
+    return await API.exportRows(payload)
+  }
+
+  const exportData = async () => {
+    try {
+      loading = true
+      let data
+      if (selectedRows?.length || sorting) {
+        data = await exportFilteredData()
+      } else {
+        data = await exportAllData()
+      }
+      notifications.success("Export successful")
+      downloadWithBlob(data, `export.${exportFormat}`)
+      popover.hide()
+    } catch (error) {
+      console.error(error)
+      notifications.error("Error exporting data")
+    } finally {
+      loading = false
+    }
+  }
 </script>
 
-<ActionButton {disabled} icon="DataDownload" quiet on:click={modal.show}>
-  Export
-</ActionButton>
-<Modal bind:this={modal}>
-  <ExportModal {view} {filters} {sorting} {selectedRows} {formats} />
-</Modal>
+<DetailPopover title="Export data" bind:this={popover}>
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton
+      icon="DataDownload"
+      quiet
+      on:click={openPopover}
+      {disabled}
+      selected={open}
+    >
+      Export
+    </ActionButton>
+  </svelte:fragment>
+
+  {#if selectedRows?.length}
+    <Body size="S">
+      <span data-testid="exporting-n-rows">
+        <strong>{selectedRows?.length}</strong>
+        {`row${selectedRows?.length > 1 ? "s" : ""} will be exported.`}
+      </span>
+    </Body>
+  {:else}
+    <Body size="S">
+      <span data-testid="export-all-rows">
+        Exporting <strong>all</strong> rows.
+      </span>
+    </Body>
+  {/if}
+  <span data-testid="format-select">
+    <Select
+      label="Format"
+      bind:value={exportFormat}
+      {options}
+      placeholder={null}
+      getOptionLabel={x => x.name}
+      getOptionValue={x => x.key}
+    />
+  </span>
+  <div>
+    <Button cta disabled={loading} on:click={exportData}>Export</Button>
+  </div>
+</DetailPopover>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridExportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridExportButton.svelte
index f1bbc04328c..674ff0015b4 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridExportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridExportButton.svelte
@@ -9,21 +9,13 @@
   $: selectedRowArray = Object.keys($selectedRows).map(id => ({ _id: id }))
 </script>
 
-<span data-ignore-click-outside="true">
-  <ExportButton
-    {disabled}
-    view={$datasource.tableId}
-    filters={$filter}
-    sorting={{
-      sortColumn: $sort.column,
-      sortOrder: $sort.order,
-    }}
-    selectedRows={selectedRowArray}
-  />
-</span>
-
-<style>
-  span {
-    display: contents;
-  }
-</style>
+<ExportButton
+  {disabled}
+  view={$datasource.tableId}
+  filters={$filter}
+  sorting={{
+    sortColumn: $sort.column,
+    sortOrder: $sort.order,
+  }}
+  selectedRows={selectedRowArray}
+/>
diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
index 2a31f5c4529..12065412e78 100644
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
@@ -1,224 +1,7 @@
 <script>
-  import {
-    Select,
-    ModalContent,
-    notifications,
-    Body,
-    Table,
-  } from "@budibase/bbui"
-  import download from "downloadjs"
-  import { API } from "api"
-  import { QueryUtils } from "@budibase/frontend-core"
-  import { utils } from "@budibase/shared-core"
-  import { ROW_EXPORT_FORMATS } from "constants/backend"
-
   export let view
   export let filters
   export let sorting
   export let selectedRows = []
   export let formats
-
-  const FORMATS = [
-    {
-      name: "CSV",
-      key: ROW_EXPORT_FORMATS.CSV,
-    },
-    {
-      name: "JSON",
-      key: ROW_EXPORT_FORMATS.JSON,
-    },
-    {
-      name: "JSON with Schema",
-      key: ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA,
-    },
-  ]
-
-  $: appliedFilters = filters?.filter(filter => !filter.onEmptyFilter)
-
-  $: options = FORMATS.filter(format => {
-    if (formats && !formats.includes(format.key)) {
-      return false
-    }
-    return true
-  })
-
-  let exportFormat
-  let filterLookup
-
-  $: if (options && !exportFormat) {
-    exportFormat = Array.isArray(options) ? options[0]?.key : []
-  }
-
-  $: query = QueryUtils.buildQuery(appliedFilters)
-  $: exportOpDisplay = buildExportOpDisplay(
-    sorting,
-    filterDisplay,
-    appliedFilters
-  )
-
-  filterLookup = utils.filterValueToLabel()
-
-  const filterDisplay = () => {
-    if (!appliedFilters) {
-      return []
-    }
-    return appliedFilters.map(filter => {
-      let newFieldName = filter.field + ""
-      const parts = newFieldName.split(":")
-      parts.shift()
-      newFieldName = parts.join(":")
-      return {
-        Field: newFieldName,
-        Operation: filterLookup[filter.operator],
-        "Field Value": filter.value || "",
-      }
-    })
-  }
-
-  const buildExportOpDisplay = (sorting, filterDisplay) => {
-    let filterDisplayConfig = filterDisplay()
-    if (sorting?.sortColumn) {
-      filterDisplayConfig = [
-        ...filterDisplayConfig,
-        {
-          Field: sorting.sortColumn,
-          Operation: "Order By",
-          "Field Value": sorting.sortOrder,
-        },
-      ]
-    }
-    return filterDisplayConfig
-  }
-
-  const displaySchema = {
-    Field: {
-      type: "string",
-      fieldName: "Field",
-    },
-    Operation: {
-      type: "string",
-      fieldName: "Operation",
-    },
-    "Field Value": {
-      type: "string",
-      fieldName: "Value",
-    },
-  }
-
-  function downloadWithBlob(data, filename) {
-    download(new Blob([data], { type: "text/plain" }), filename)
-  }
-
-  async function exportView() {
-    try {
-      const data = await API.exportView({
-        viewName: view,
-        format: exportFormat,
-      })
-      downloadWithBlob(
-        data,
-        `export.${exportFormat === "csv" ? "csv" : "json"}`
-      )
-    } catch (error) {
-      notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
-    }
-  }
-
-  async function exportRows() {
-    if (selectedRows?.length) {
-      const data = await API.exportRows({
-        tableId: view,
-        rows: selectedRows.map(row => row._id),
-        format: exportFormat,
-      })
-      downloadWithBlob(data, `export.${exportFormat}`)
-    } else if (appliedFilters || sorting) {
-      let response
-      try {
-        response = await API.exportRows({
-          tableId: view,
-          format: exportFormat,
-          search: {
-            query,
-            sort: sorting?.sortColumn,
-            sortOrder: sorting?.sortOrder,
-            paginate: false,
-          },
-        })
-      } catch (e) {
-        console.error("Failed to export", e)
-        notifications.error("Export Failed")
-      }
-      if (response) {
-        downloadWithBlob(response, `export.${exportFormat}`)
-        notifications.success("Export Successful")
-      }
-    } else {
-      await exportView()
-    }
-  }
 </script>
-
-<ModalContent
-  title="Export Data"
-  confirmText="Export"
-  onConfirm={exportRows}
-  size={appliedFilters?.length || sorting ? "M" : "S"}
->
-  {#if selectedRows?.length}
-    <Body size="S">
-      <span data-testid="exporting-n-rows">
-        <strong>{selectedRows?.length}</strong>
-        {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
-      </span>
-    </Body>
-  {:else if appliedFilters?.length || (sorting?.sortOrder && sorting?.sortColumn)}
-    <Body size="S">
-      {#if !appliedFilters}
-        <span data-testid="exporting-rows">
-          Exporting <strong>all</strong> rows
-        </span>
-      {:else}
-        <span data-testid="filters-applied">Filters applied</span>
-      {/if}
-    </Body>
-
-    <div class="table-wrap" data-testid="export-config-table">
-      <Table
-        schema={displaySchema}
-        data={exportOpDisplay}
-        {appliedFilters}
-        loading={false}
-        rowCount={appliedFilters?.length + 1}
-        disableSorting={true}
-        allowSelectRows={false}
-        allowEditRows={false}
-        allowEditColumns={false}
-        quiet={true}
-        compact={true}
-      />
-    </div>
-  {:else}
-    <Body size="S">
-      <span data-testid="export-all-rows">
-        Exporting <strong>all</strong> rows
-      </span>
-    </Body>
-  {/if}
-  <span data-testid="format-select">
-    <Select
-      label="Format"
-      bind:value={exportFormat}
-      {options}
-      placeholder={null}
-      getOptionLabel={x => x.name}
-      getOptionValue={x => x.key}
-    />
-  </span>
-</ModalContent>
-
-<style>
-  .table-wrap :global(.wrapper) {
-    max-width: 400px;
-  }
-</style>

From 18130729b2e3fc75dd240ac3b4201c38ea878f58 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:04:50 +0000
Subject: [PATCH 410/455] Remove old files

---
 .../DataTable/modals/ExportModal.svelte       |   7 -
 .../DataTable/modals/ExportModal.test.js      | 243 ------------------
 2 files changed, 250 deletions(-)
 delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
 delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js

diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
deleted file mode 100644
index 12065412e78..00000000000
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
+++ /dev/null
@@ -1,7 +0,0 @@
-<script>
-  export let view
-  export let filters
-  export let sorting
-  export let selectedRows = []
-  export let formats
-</script>
diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
deleted file mode 100644
index 4927f51a8d5..00000000000
--- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js
+++ /dev/null
@@ -1,243 +0,0 @@
-import { it, expect, describe, vi } from "vitest"
-import { render, screen } from "@testing-library/svelte"
-import "@testing-library/jest-dom"
-
-import ExportModal from "./ExportModal.svelte"
-import { utils } from "@budibase/shared-core"
-
-const labelLookup = utils.filterValueToLabel()
-
-const rowText = filter => {
-  let readableField = filter.field.split(":")[1]
-  let rowLabel = labelLookup[filter.operator]
-  let value = Array.isArray(filter.value)
-    ? JSON.stringify(filter.value)
-    : filter.value
-  return `${readableField}${rowLabel}${value}`.trim()
-}
-
-const defaultFilters = [
-  {
-    onEmptyFilter: "all",
-  },
-]
-
-vi.mock("svelte", async () => {
-  return {
-    getContext: () => {
-      return {
-        hide: vi.fn(),
-        cancel: vi.fn(),
-      }
-    },
-    createEventDispatcher: vi.fn(),
-    onDestroy: vi.fn(),
-    tick: vi.fn(),
-  }
-})
-
-vi.mock("api", async () => {
-  return {
-    API: {
-      exportView: vi.fn(),
-      exportRows: vi.fn(),
-    },
-  }
-})
-
-describe("Export Modal", () => {
-  it("show default messaging with no export config specified", () => {
-    render(ExportModal, {
-      props: {},
-    })
-
-    expect(screen.getByTestId("export-all-rows")).toBeVisible()
-    expect(screen.getByTestId("export-all-rows")).toHaveTextContent(
-      "Exporting all rows"
-    )
-
-    expect(screen.queryByTestId("export-config-table")).toBe(null)
-  })
-
-  it("indicate that a filter is being applied to the export", () => {
-    const propsCfg = {
-      filters: [
-        {
-          id: "MOQkMx9p9",
-          field: "1:Cost",
-          operator: "rangeHigh",
-          value: "100",
-          valueType: "Value",
-          type: "number",
-          noValue: false,
-        },
-        ...defaultFilters,
-      ],
-    }
-
-    render(ExportModal, {
-      props: propsCfg,
-    })
-
-    expect(propsCfg.filters[0].field).toBe("1:Cost")
-
-    expect(screen.getByTestId("filters-applied")).toBeVisible()
-    expect(screen.getByTestId("filters-applied").textContent).toBe(
-      "Filters applied"
-    )
-
-    const ele = screen.queryByTestId("export-config-table")
-    expect(ele).toBeVisible()
-
-    const rows = ele.getElementsByClassName("spectrum-Table-row")
-
-    expect(rows.length).toBe(1)
-    let rowTextContent = rowText(propsCfg.filters[0])
-
-    //"CostLess than or equal to100"
-    expect(rows[0].textContent?.trim()).toEqual(rowTextContent)
-  })
-
-  it("Show only selected row messaging if rows are supplied", () => {
-    const propsCfg = {
-      filters: [
-        {
-          id: "MOQkMx9p9",
-          field: "1:Cost",
-          operator: "rangeHigh",
-          value: "100",
-          valueType: "Value",
-          type: "number",
-          noValue: false,
-        },
-        ...defaultFilters,
-      ],
-      sorting: {
-        sortColumn: "Cost",
-        sortOrder: "descending",
-      },
-      selectedRows: [
-        {
-          _id: "ro_ta_bb_expenses_57d5f6fe1b6640d8bb22b15f5eae62cd",
-        },
-        {
-          _id: "ro_ta_bb_expenses_99ce5760a53a430bab4349cd70335a07",
-        },
-      ],
-    }
-
-    render(ExportModal, {
-      props: propsCfg,
-    })
-
-    expect(screen.queryByTestId("export-config-table")).toBeNull()
-    expect(screen.queryByTestId("filters-applied")).toBeNull()
-
-    expect(screen.queryByTestId("exporting-n-rows")).toBeVisible()
-    expect(screen.queryByTestId("exporting-n-rows").textContent).toEqual(
-      "2 rows will be exported"
-    )
-  })
-
-  it("Show only the configured sort when no filters are specified", () => {
-    const propsCfg = {
-      filters: [...defaultFilters],
-      sorting: {
-        sortColumn: "Cost",
-        sortOrder: "descending",
-      },
-    }
-    render(ExportModal, {
-      props: propsCfg,
-    })
-
-    expect(screen.queryByTestId("export-config-table")).toBeVisible()
-    const ele = screen.queryByTestId("export-config-table")
-    const rows = ele.getElementsByClassName("spectrum-Table-row")
-
-    expect(rows.length).toBe(1)
-    expect(rows[0].textContent?.trim()).toEqual(
-      `${propsCfg.sorting.sortColumn}Order By${propsCfg.sorting.sortOrder}`
-    )
-  })
-
-  it("Display all currently configured filters and applied sort", () => {
-    const propsCfg = {
-      filters: [
-        {
-          id: "MOQkMx9p9",
-          field: "1:Cost",
-          operator: "rangeHigh",
-          value: "100",
-          valueType: "Value",
-          type: "number",
-          noValue: false,
-        },
-        {
-          id: "2ot-aB0gE",
-          field: "2:Expense Tags",
-          operator: "contains",
-          value: ["Equipment", "Services"],
-          valueType: "Value",
-          type: "array",
-          noValue: false,
-        },
-        ...defaultFilters,
-      ],
-      sorting: {
-        sortColumn: "Payment Due",
-        sortOrder: "ascending",
-      },
-    }
-
-    render(ExportModal, {
-      props: propsCfg,
-    })
-
-    const ele = screen.queryByTestId("export-config-table")
-    expect(ele).toBeVisible()
-
-    const rows = ele.getElementsByClassName("spectrum-Table-row")
-    expect(rows.length).toBe(3)
-
-    let rowTextContent1 = rowText(propsCfg.filters[0])
-    expect(rows[0].textContent?.trim()).toEqual(rowTextContent1)
-
-    let rowTextContent2 = rowText(propsCfg.filters[1])
-    expect(rows[1].textContent?.trim()).toEqual(rowTextContent2)
-
-    expect(rows[2].textContent?.trim()).toEqual(
-      `${propsCfg.sorting.sortColumn}Order By${propsCfg.sorting.sortOrder}`
-    )
-  })
-
-  it("show only the valid, configured download formats", () => {
-    const propsCfg = {
-      formats: ["badger", "json"],
-    }
-
-    render(ExportModal, {
-      props: propsCfg,
-    })
-
-    let ele = screen.getByTestId("format-select")
-    expect(ele).toBeVisible()
-
-    let formatDisplay = ele.getElementsByTagName("button")[0]
-
-    expect(formatDisplay.textContent.trim()).toBe("JSON")
-  })
-
-  it("Load the default format config when no explicit formats are configured", () => {
-    render(ExportModal, {
-      props: {},
-    })
-
-    let ele = screen.getByTestId("format-select")
-    expect(ele).toBeVisible()
-
-    let formatDisplay = ele.getElementsByTagName("button")[0]
-
-    expect(formatDisplay.textContent.trim()).toBe("CSV")
-  })
-})

From 4979cce1322c902fb2b24ca4bdcef0aa7abfcf45 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:05:02 +0000
Subject: [PATCH 411/455] Udpate sort button to be a detail popover

---
 .../buttons/grid/GridSortButton.svelte        | 79 ++++++++-----------
 1 file changed, 32 insertions(+), 47 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
index 4591b22b781..e86742e573c 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
@@ -1,12 +1,12 @@
 <script>
   import { getContext } from "svelte"
-  import { ActionButton, Popover, Select } from "@budibase/bbui"
+  import { ActionButton, Select } from "@budibase/bbui"
   import { canBeSortColumn } from "@budibase/frontend-core"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { sort, columns } = getContext("grid")
 
-  let open = false
-  let anchor
+  let popover
 
   $: columnOptions = $columns
     .filter(col => canBeSortColumn(col.schema))
@@ -45,50 +45,35 @@
   }
 </script>
 
-<div bind:this={anchor}>
-  <ActionButton
-    icon="SortOrderDown"
-    quiet
-    size="M"
-    on:click={() => (open = !open)}
-    selected={open}
-    disabled={!columnOptions.length}
-  >
-    Sort
-  </ActionButton>
-</div>
-
-<Popover bind:open {anchor} align="left">
-  <div class="content">
+<DetailPopover bind:this={popover} title="Sorting parameters">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton
+      icon="SortOrderDown"
+      quiet
+      size="M"
+      on:click={popover?.open}
+      selected={open}
+      disabled={!columnOptions.length}
+    >
+      Sort
+    </ActionButton>
+  </svelte:fragment>
+  <Select
+    placeholder="Default"
+    value={$sort.column}
+    options={columnOptions}
+    autoWidth
+    on:change={updateSortColumn}
+    label="Column"
+  />
+  {#if $sort.column}
     <Select
-      placeholder="Default"
-      value={$sort.column}
-      options={columnOptions}
+      placeholder={null}
+      value={$sort.order || "ascending"}
+      options={orderOptions}
       autoWidth
-      on:change={updateSortColumn}
-      label="Column"
+      on:change={updateSortOrder}
+      label="Order"
     />
-    {#if $sort.column}
-      <Select
-        placeholder={null}
-        value={$sort.order || "ascending"}
-        options={orderOptions}
-        autoWidth
-        on:change={updateSortOrder}
-        label="Order"
-      />
-    {/if}
-  </div>
-</Popover>
-
-<style>
-  .content {
-    padding: 6px 12px 12px 12px;
-    display: flex;
-    align-items: center;
-    gap: 8px;
-  }
-  .content :global(.spectrum-Picker) {
-    width: 140px;
-  }
-</style>
+  {/if}
+</DetailPopover>

From 83eed815c33a72bb6de5abaefca2872895922155 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:05:14 +0000
Subject: [PATCH 412/455] Update size button to be a detail popover

---
 .../buttons/grid/GridSizeButton.svelte        | 95 +++++++++----------
 1 file changed, 46 insertions(+), 49 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
index f9499e54b76..c72366c53ba 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
@@ -1,6 +1,7 @@
 <script>
   import { getContext } from "svelte"
-  import { ActionButton, Popover, Label } from "@budibase/bbui"
+  import { ActionButton, Label } from "@budibase/bbui"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   const {
     Constants,
@@ -32,8 +33,7 @@
     },
   ]
 
-  let open = false
-  let anchor
+  let popover
 
   // Column width sizes
   $: allSmall = $columns.every(col => col.width === smallColSize)
@@ -66,55 +66,52 @@
   }
 </script>
 
-<div bind:this={anchor}>
-  <ActionButton
-    icon="MoveUpDown"
-    quiet
-    size="M"
-    on:click={() => (open = !open)}
-    selected={open}
-    disabled={!$columns.length}
-  >
-    Size
-  </ActionButton>
-</div>
-
-<Popover bind:open {anchor} align="left">
-  <div class="content">
-    <div class="size">
-      <Label>Row height</Label>
-      <div class="options">
-        {#each rowSizeOptions as option}
-          <ActionButton
-            disabled={$fixedRowHeight}
-            quiet
-            selected={$rowHeight === option.size}
-            on:click={() => changeRowHeight(option.size)}
-          >
-            {option.label}
-          </ActionButton>
-        {/each}
-      </div>
+<DetailPopover bind:this={popover} title="Column and row size">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton
+      icon="MoveUpDown"
+      quiet
+      size="M"
+      on:click={popover?.open}
+      selected={open}
+      disabled={!$columns.length}
+    >
+      Size
+    </ActionButton>
+  </svelte:fragment>
+  <div class="size">
+    <Label>Row height</Label>
+    <div class="options">
+      {#each rowSizeOptions as option}
+        <ActionButton
+          disabled={$fixedRowHeight}
+          quiet
+          selected={$rowHeight === option.size}
+          on:click={() => changeRowHeight(option.size)}
+        >
+          {option.label}
+        </ActionButton>
+      {/each}
     </div>
-    <div class="size">
-      <Label>Column width</Label>
-      <div class="options">
-        {#each columnSizeOptions as option}
-          <ActionButton
-            quiet
-            on:click={() => columns.actions.changeAllColumnWidths(option.size)}
-            selected={option.selected}
-          >
-            {option.label}
-          </ActionButton>
-        {/each}
-        {#if custom}
-          <ActionButton selected={custom} quiet>Custom</ActionButton>
-        {/if}
-      </div>
+  </div>
+  <div class="size">
+    <Label>Column width</Label>
+    <div class="options">
+      {#each columnSizeOptions as option}
+        <ActionButton
+          quiet
+          on:click={() => columns.actions.changeAllColumnWidths(option.size)}
+          selected={option.selected}
+        >
+          {option.label}
+        </ActionButton>
+      {/each}
+      {#if custom}
+        <ActionButton selected={custom} quiet>Custom</ActionButton>
+      {/if}
     </div>
   </div>
-</Popover>
+</DetailPopover>
 
 <style>
   .content {

From 76ccdbaa12a2135bb0f376357e6dc8072520cc78 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:05:41 +0000
Subject: [PATCH 413/455] Remove styles

---
 .../backend/DataTable/buttons/grid/GridSizeButton.svelte    | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
index c72366c53ba..fa4d336ab3a 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
@@ -114,12 +114,6 @@
 </DetailPopover>
 
 <style>
-  .content {
-    padding: 12px;
-    display: flex;
-    flex-direction: column;
-    gap: 16px;
-  }
   .size {
     display: flex;
     flex-direction: column;

From 67c5cc4a45b814a02daaf70f685d50d9255e5cd5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:09:57 +0000
Subject: [PATCH 414/455] Update column settings button to be a detail popover

---
 .../buttons/grid/ColumnsSettingContent.svelte | 28 ++++++++-------
 .../grid/GridColumnsSettingButton.svelte      | 35 +++++++++----------
 2 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
index 9e9449976b6..85af8596a0b 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
@@ -210,16 +210,18 @@
     anchor={relationshipPanelAnchor}
     align="left"
   >
-    {#if relationshipPanelColumns.length}
-      <div class="relationship-header">
-        {relationshipFieldName} columns
-      </div>
-    {/if}
-    <svelte:self
-      columns={relationshipPanelColumns}
-      permissions={[FieldPermissions.READONLY, FieldPermissions.HIDDEN]}
-      fromRelationshipField={relationshipField}
-    />
+    <div class="nested">
+      {#if relationshipPanelColumns.length}
+        <div class="relationship-header">
+          {relationshipFieldName} columns
+        </div>
+      {/if}
+      <svelte:self
+        columns={relationshipPanelColumns}
+        permissions={[FieldPermissions.READONLY, FieldPermissions.HIDDEN]}
+        fromRelationshipField={relationshipField}
+      />
+    </div>
   </Popover>
 {/if}
 
@@ -230,11 +232,13 @@
   }
 
   .content {
-    padding: 12px 12px;
     display: flex;
     flex-direction: column;
     gap: 12px;
   }
+  .nested {
+    padding: 12px;
+  }
   .columns {
     display: grid;
     align-items: center;
@@ -262,6 +266,6 @@
   }
   .relationship-header {
     color: var(--spectrum-global-color-gray-600);
-    padding: 12px 12px 0 12px;
+    margin-bottom: 12px;
   }
 </style>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
index 83327a48c4a..0e47fe1bc05 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
@@ -12,11 +12,11 @@
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
   import { isEnabled } from "helpers/featureFlags"
   import { FeatureFlag } from "@budibase/types"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { tableColumns, datasource } = getContext("grid")
 
-  let open = false
-  let anchor
+  let popover
 
   $: anyRestricted = $tableColumns.filter(
     col => !col.visible || col.readonly
@@ -32,24 +32,23 @@
       : [FieldPermissions.WRITABLE, FieldPermissions.HIDDEN]
 </script>
 
-<div bind:this={anchor}>
-  <ActionButton
-    icon="ColumnSettings"
-    quiet
-    size="M"
-    on:click={() => (open = !open)}
-    selected={open || anyRestricted}
-    disabled={!$tableColumns.length}
-    accentColor="#674D00"
-  >
-    {text}
-  </ActionButton>
-</div>
-
-<Popover bind:open {anchor} align="left">
+<DetailPopover bind:this={popover} title="Column settings">
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton
+      icon="ColumnSettings"
+      quiet
+      size="M"
+      on:click={popover?.open}
+      selected={open || anyRestricted}
+      disabled={!$tableColumns.length}
+      accentColor="#674D00"
+    >
+      {text}
+    </ActionButton>
+  </svelte:fragment>
   <ColumnsSettingContent
     columns={$tableColumns}
     canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
     {permissions}
   />
-</Popover>
+</DetailPopover>

From 3ad913ae28022cacd01701b8d1ce5938e9249eb5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:11:13 +0000
Subject: [PATCH 415/455] Reorder buttons to access is always first

---
 .../[application]/data/table/[tableId]/[viewId]/index.svelte    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index 44fb70b9d02..bc21b04ad0b 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -60,10 +60,10 @@
   buttonsCollapsed
 >
   <svelte:fragment slot="controls">
+    <GridManageAccessButton />
     {#if calculation}
       <GridViewCalculationButton />
     {/if}
-    <GridManageAccessButton />
     <GridFilterButton />
     <GridSortButton />
     <GridSizeButton />

From 27c1788db28b06d41d15361fba5e513d784d3de3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:17:50 +0000
Subject: [PATCH 416/455] Update view calculation button to be a detail popover

---
 .../grid/GridViewCalculationButton.svelte     | 142 +++++++++---------
 1 file changed, 74 insertions(+), 68 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 72216f3b3be..10bf35d4dce 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -1,15 +1,15 @@
 <script>
   import {
     ActionButton,
-    Modal,
-    ModalContent,
     Select,
     Icon,
     Multiselect,
+    Button,
   } from "@budibase/bbui"
   import { CalculationType, canGroupBy, FieldType } from "@budibase/types"
   import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
   import { getContext } from "svelte"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { definition, datasource, rows } = getContext("grid")
   const calculationTypeOptions = [
@@ -35,19 +35,20 @@
     },
   ]
 
-  let modal
+  let popover
   let calculations = []
   let groupBy = []
   let schema = {}
+  let loading = false
 
   $: schema = $definition?.schema || {}
   $: count = extractCalculations($definition?.schema || {}).length
   $: groupByOptions = getGroupByOptions(schema)
 
-  const open = () => {
+  const openPopover = () => {
     calculations = extractCalculations(schema)
     groupBy = calculations.length ? extractGroupBy(schema) : []
-    modal?.show()
+    popover?.show()
   }
 
   const extractCalculations = schema => {
@@ -135,6 +136,7 @@
 
   const save = async () => {
     let newSchema = {}
+    loading = true
 
     // Add calculations
     for (let calc of calculations) {
@@ -168,76 +170,80 @@
     }
 
     // Save changes
-    await datasource.actions.saveDefinition({
-      ...$definition,
-      primaryDisplay,
-      schema: newSchema,
-    })
-    await rows.actions.refreshData()
+    try {
+      await datasource.actions.saveDefinition({
+        ...$definition,
+        primaryDisplay,
+        schema: newSchema,
+      })
+      await rows.actions.refreshData()
+    } finally {
+      loading = false
+      popover.hide()
+    }
   }
 </script>
 
-<ActionButton icon="WebPage" quiet on:click={open}>
-  Configure calculations{count ? `: ${count}` : ""}
-</ActionButton>
+<DetailPopover bind:this={popover} title="Calculations" width={480}>
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton icon="WebPage" quiet on:click={openPopover} selected={open}>
+      Configure calculations{count ? `: ${count}` : ""}
+    </ActionButton>
+  </svelte:fragment>
 
-<Modal bind:this={modal}>
-  <ModalContent
-    title="Calculations"
-    confirmText="Save"
-    size="M"
-    onConfirm={save}
-  >
-    {#if calculations.length}
-      <div class="calculations">
-        {#each calculations as calc, idx}
-          <span>{idx === 0 ? "Calculate" : "and"} the</span>
-          <Select
-            options={getTypeOptions(calc, calculations)}
-            bind:value={calc.type}
-            placeholder={false}
-          />
-          <span>of</span>
-          <Select
-            options={getFieldOptions(calc, calculations, schema)}
-            bind:value={calc.field}
-            placeholder="Column"
-          />
-          <Icon
-            hoverable
-            name="Delete"
-            size="S"
-            on:click={() => deleteCalc(idx)}
-            color="var(--spectrum-global-color-gray-700)"
-          />
-        {/each}
-        <span>Group by</span>
-        <div class="group-by">
-          <Multiselect
-            options={groupByOptions}
-            bind:value={groupBy}
-            placeholder="None"
-          />
-        </div>
+  {#if calculations.length}
+    <div class="calculations">
+      {#each calculations as calc, idx}
+        <span>{idx === 0 ? "Calculate" : "and"} the</span>
+        <Select
+          options={getTypeOptions(calc, calculations)}
+          bind:value={calc.type}
+          placeholder={false}
+        />
+        <span>of</span>
+        <Select
+          options={getFieldOptions(calc, calculations, schema)}
+          bind:value={calc.field}
+          placeholder="Column"
+        />
+        <Icon
+          hoverable
+          name="Delete"
+          size="S"
+          on:click={() => deleteCalc(idx)}
+          color="var(--spectrum-global-color-gray-700)"
+        />
+      {/each}
+      <span>Group by</span>
+      <div class="group-by">
+        <Multiselect
+          options={groupByOptions}
+          bind:value={groupBy}
+          placeholder="None"
+        />
       </div>
-    {/if}
-    <div class="buttons">
-      <ActionButton
-        quiet
-        icon="Add"
-        on:click={addCalc}
-        disabled={calculations.length >= 5}
-      >
-        Add calculation
-      </ActionButton>
     </div>
-    <InfoDisplay
-      icon="Help"
+  {/if}
+  <div class="buttons">
+    <ActionButton
       quiet
-      body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
-    />
-  </ModalContent>
-</Modal>
+      icon="Add"
+      on:click={addCalc}
+      disabled={calculations.length >= 5}
+    >
+      Add calculation
+    </ActionButton>
+  </div>
+  <InfoDisplay
+    icon="Help"
+    quiet
+    body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
+  />
+
+  <div>
+    <Button cta on:click={save} disabled={loading}>Save</Button>
+  </div>
+</DetailPopover>
 
 <style>
   .calculations {

From 5156749ac9f2706a742fbf471e29ff74aa7e5162 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:24:16 +0000
Subject: [PATCH 417/455] Update table filter button to be a detail popover

---
 .../buttons/TableFilterButton.svelte          | 82 +++++++++----------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
index 2071ab8b860..c6103b16974 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@@ -1,11 +1,12 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { ActionButton, Drawer, DrawerContent, Button } from "@budibase/bbui"
+  import { ActionButton, Button } from "@budibase/bbui"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
   import { getUserBindings } from "dataBinding"
   import { makePropSafe } from "@budibase/string-templates"
   import { search } from "@budibase/frontend-core"
   import { tables } from "stores/builder"
+  import DetailPopover from "components/common/DetailPopover.svelte"
 
   export let schema
   export let filters
@@ -14,7 +15,7 @@
 
   const dispatch = createEventDispatcher()
 
-  let drawer
+  let popover
 
   $: localFilters = filters
   $: schemaFields = search.getFields(
@@ -39,45 +40,44 @@
     },
     ...getUserBindings(),
   ]
+
+  const openPopover = () => {
+    localFilters = filters
+    popover.show()
+  }
 </script>
 
-<ActionButton
-  icon="Filter"
-  quiet
-  {disabled}
-  on:click={drawer.show}
-  selected={filterCount > 0}
-  accentColor="#004EA6"
->
-  {filterCount ? `Filter: ${filterCount}` : "Filter"}
-</ActionButton>
+<DetailPopover bind:this={popover} title="Configure filters" width={800}>
+  <svelte:fragment slot="anchor" let:open>
+    <ActionButton
+      icon="Filter"
+      quiet
+      {disabled}
+      on:click={openPopover}
+      selected={open || filterCount > 0}
+      accentColor="#004EA6"
+    >
+      {filterCount ? `Filter: ${filterCount}` : "Filter"}
+    </ActionButton>
+  </svelte:fragment>
 
-<Drawer
-  bind:this={drawer}
-  title="Filtering"
-  on:drawerHide
-  on:drawerShow={() => {
-    localFilters = filters
-  }}
-  forceModal
->
-  <Button
-    cta
-    slot="buttons"
-    on:click={() => {
-      dispatch("change", localFilters)
-      drawer.hide()
-    }}
-  >
-    Save
-  </Button>
-  <DrawerContent slot="body">
-    <FilterBuilder
-      filters={localFilters}
-      {schemaFields}
-      datasource={{ type: "table", tableId }}
-      on:change={e => (localFilters = e.detail)}
-      {bindings}
-    />
-  </DrawerContent>
-</Drawer>
+  <FilterBuilder
+    filters={localFilters}
+    {schemaFields}
+    datasource={{ type: "table", tableId }}
+    on:change={e => (localFilters = e.detail)}
+    {bindings}
+  />
+  <div>
+    <Button
+      cta
+      slot="buttons"
+      on:click={() => {
+        dispatch("change", localFilters)
+        popover.hide()
+      }}
+    >
+      Save
+    </Button>
+  </div>
+</DetailPopover>

From 293368e5510a47070fa11456dd184525c214a236 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:24:25 +0000
Subject: [PATCH 418/455] Update titles

---
 .../backend/DataTable/buttons/grid/GridSortButton.svelte        | 2 +-
 .../DataTable/buttons/grid/GridViewCalculationButton.svelte     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
index e86742e573c..3cb93ad243c 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
@@ -45,7 +45,7 @@
   }
 </script>
 
-<DetailPopover bind:this={popover} title="Sorting parameters">
+<DetailPopover bind:this={popover} title="Sorting">
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="SortOrderDown"
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 10bf35d4dce..f16d1254f3e 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -184,7 +184,7 @@
   }
 </script>
 
-<DetailPopover bind:this={popover} title="Calculations" width={480}>
+<DetailPopover bind:this={popover} title="Configure calculations" width={480}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton icon="WebPage" quiet on:click={openPopover} selected={open}>
       Configure calculations{count ? `: ${count}` : ""}

From 8f94e35d2981e35486eb2a57d7d4c626e2c3060e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:38:56 +0000
Subject: [PATCH 419/455] Shrink size button popover

---
 .../backend/DataTable/buttons/grid/GridSizeButton.svelte        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
index fa4d336ab3a..148d8b1d5ff 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte
@@ -66,7 +66,7 @@
   }
 </script>
 
-<DetailPopover bind:this={popover} title="Column and row size">
+<DetailPopover bind:this={popover} title="Column and row size" width={300}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="MoveUpDown"

From 6e021e6793f04fe57b1c30a4818968a284bbb015 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 28 Oct 2024 14:54:18 +0000
Subject: [PATCH 420/455] lint

---
 .../DataTable/buttons/grid/GridColumnsSettingButton.svelte      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
index 0e47fe1bc05..7ab84ba0397 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
@@ -8,7 +8,7 @@
 
 <script>
   import { getContext } from "svelte"
-  import { ActionButton, Popover } from "@budibase/bbui"
+  import { ActionButton } from "@budibase/bbui"
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
   import { isEnabled } from "helpers/featureFlags"
   import { FeatureFlag } from "@budibase/types"

From c5c17cd6bd4cfd79095fd7dde310091e56b125c2 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 08:48:07 +0000
Subject: [PATCH 421/455] Reduce size of sorting popover

---
 .../backend/DataTable/buttons/grid/GridSortButton.svelte        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
index 3cb93ad243c..244a1b15604 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
@@ -45,7 +45,7 @@
   }
 </script>
 
-<DetailPopover bind:this={popover} title="Sorting">
+<DetailPopover bind:this={popover} title="Sorting" width={300}>
   <svelte:fragment slot="anchor" let:open>
     <ActionButton
       icon="SortOrderDown"

From d5aed4361feebd38565f299fa27292efc3014f22 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 08:49:44 +0000
Subject: [PATCH 422/455] Fix z-index issue with detail popovers and bindings

---
 packages/builder/src/components/common/DetailPopover.svelte | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte
index c437804b332..f1e81a63409 100644
--- a/packages/builder/src/components/common/DetailPopover.svelte
+++ b/packages/builder/src/components/common/DetailPopover.svelte
@@ -30,6 +30,7 @@
   {showPopover}
   on:open
   on:close
+  customZindex={100}
 >
   <div class="detail-popover">
     <div class="detail-popover__header">

From b9b4f88e4a256032d21eed6e92239398b4263c1d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 29 Oct 2024 10:23:32 +0000
Subject: [PATCH 423/455] Fix default values when using multi-option column and
 supplying empty array.

---
 packages/server/src/api/routes/tests/row.spec.ts    | 13 +++++++++++++
 packages/server/src/utilities/rowProcessor/index.ts |  8 +++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 66e931d0ab0..42c3560b369 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -763,12 +763,25 @@ describe.each([
           expect(row.food).toEqual(["apple", "orange"])
         })
 
+        it("creates a new row with a default value when given an empty list", async () => {
+          const row = await config.api.row.save(table._id!, { food: [] })
+          expect(row.food).toEqual(["apple", "orange"])
+        })
+
         it("does not use default value if value specified", async () => {
           const row = await config.api.row.save(table._id!, {
             food: ["orange"],
           })
           expect(row.food).toEqual(["orange"])
         })
+
+        it("resets back to its default value when empty", async () => {
+          let row = await config.api.row.save(table._id!, {
+            food: ["orange"],
+          })
+          row = await config.api.row.save(table._id!, { ...row, food: [] })
+          expect(row.food).toEqual(["apple", "orange"])
+        })
       })
 
       describe("user column", () => {
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 0ffa7600aad..d5cbbd2b643 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -20,6 +20,7 @@ import {
   User,
   ViewV2,
   FeatureFlag,
+  Document,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
@@ -134,7 +135,12 @@ async function processDefaultValues(table: Table, row: Row) {
   }
 
   for (const [key, schema] of Object.entries(table.schema)) {
-    if ("default" in schema && schema.default != null && row[key] == null) {
+    const isEmpty =
+      row[key] == null ||
+      row[key] === "" ||
+      (Array.isArray(row[key]) && row[key].length === 0)
+
+    if ("default" in schema && schema.default != null && isEmpty) {
       let processed: string | string[]
       if (Array.isArray(schema.default)) {
         processed = schema.default.map(val => processStringSync(val, ctx))

From 67fea783bb46820dae8ab97bc9c3687d6fe72bc0 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 29 Oct 2024 10:29:41 +0000
Subject: [PATCH 424/455] Support boolean and bigint fields having default
 values.

---
 .../server/src/api/routes/tests/row.spec.ts   | 56 +++++++++++++++++++
 packages/shared-core/src/table.ts             |  4 +-
 .../types/src/documents/app/table/schema.ts   | 14 +++++
 3 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 2980803fbc0..1cb6afe58b5 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -835,6 +835,62 @@ describe.each([
         })
       })
 
+      describe("boolean column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                active: {
+                  name: "active",
+                  type: FieldType.BOOLEAN,
+                  default: "true",
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.active).toEqual(true)
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            active: false,
+          })
+          expect(row.active).toEqual(false)
+        })
+      })
+
+      describe("bigint column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                bigNumber: {
+                  name: "bigNumber",
+                  type: FieldType.BIGINT,
+                  default: "1234567890",
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.bigNumber).toEqual("1234567890")
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            bigNumber: "9876543210",
+          })
+          expect(row.bigNumber).toEqual("9876543210")
+        })
+      })
+
       describe("bindings", () => {
         describe("string column", () => {
           beforeAll(async () => {
diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts
index 73e3ad6cb78..5402439d9cb 100644
--- a/packages/shared-core/src/table.ts
+++ b/packages/shared-core/src/table.ts
@@ -57,12 +57,12 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
   [FieldType.OPTIONS]: true,
   [FieldType.ARRAY]: true,
+  [FieldType.BIGINT]: true,
+  [FieldType.BOOLEAN]: true,
 
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
   [FieldType.BARCODEQR]: false,
-  [FieldType.BIGINT]: false,
-  [FieldType.BOOLEAN]: false,
   [FieldType.FORMULA]: false,
   [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 45d0aa78d2a..9e4555e1a23 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -198,6 +198,16 @@ interface BaseFieldSchema extends UIFieldMetadata {
   subtype?: never
 }
 
+interface BooleanFieldMetadata extends BaseFieldSchema {
+  type: FieldType.BOOLEAN
+  default?: string
+}
+
+interface BigIntFieldMetadata extends BaseFieldSchema {
+  type: FieldType.BIGINT
+  default?: string
+}
+
 interface OtherFieldMetadata extends BaseFieldSchema {
   type: Exclude<
     FieldType,
@@ -214,6 +224,8 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.STRING
     | FieldType.ARRAY
     | FieldType.OPTIONS
+    | FieldType.BOOLEAN
+    | FieldType.BIGINT
   >
 }
 
@@ -233,6 +245,8 @@ export type FieldSchema =
   | BBReferenceSingleFieldMetadata
   | ArrayFieldMetadata
   | OptionsFieldMetadata
+  | BooleanFieldMetadata
+  | BigIntFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema

From 0c8590be9c43a4ecdda68bad35746378b9c1529b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 29 Oct 2024 10:32:49 +0000
Subject: [PATCH 425/455] Fix lint.

---
 packages/server/src/utilities/rowProcessor/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index d5cbbd2b643..ed74394a85a 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -20,7 +20,6 @@ import {
   User,
   ViewV2,
   FeatureFlag,
-  Document,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {

From b58fd7dc6b0e92976c330901b7b21b07b19163d1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 29 Oct 2024 10:34:14 +0000
Subject: [PATCH 426/455] Fix OSS build.

---
 .../types/src/documents/app/table/schema.ts   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 9e4555e1a23..7e79902a49b 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -186,6 +186,16 @@ export interface ArrayFieldMetadata extends BaseFieldSchema {
   default?: string[]
 }
 
+export interface BooleanFieldMetadata extends BaseFieldSchema {
+  type: FieldType.BOOLEAN
+  default?: string
+}
+
+export interface BigIntFieldMetadata extends BaseFieldSchema {
+  type: FieldType.BIGINT
+  default?: string
+}
+
 interface BaseFieldSchema extends UIFieldMetadata {
   type: FieldType
   name: string
@@ -198,16 +208,6 @@ interface BaseFieldSchema extends UIFieldMetadata {
   subtype?: never
 }
 
-interface BooleanFieldMetadata extends BaseFieldSchema {
-  type: FieldType.BOOLEAN
-  default?: string
-}
-
-interface BigIntFieldMetadata extends BaseFieldSchema {
-  type: FieldType.BIGINT
-  default?: string
-}
-
 interface OtherFieldMetadata extends BaseFieldSchema {
   type: Exclude<
     FieldType,

From aab5256bf256ddf2e45c6cf944afe7f74b87d4aa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 29 Oct 2024 12:13:54 +0100
Subject: [PATCH 427/455] Fix frontend filtering

---
 packages/frontend-core/src/fetch/DataFetch.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index fb1dbd58856..175365a442c 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -184,7 +184,7 @@ export default class DataFetch {
 
     // Build the query
     let query = this.options.query
-    if (!query && this.features.supportsSearch) {
+    if (!query) {
       query = buildQuery(filter)
     }
 

From 32ad21de4f37704440d03f3e8b812872cd39d0c9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 11:38:00 +0000
Subject: [PATCH 428/455] Fix forms always requesting table schema and cache
 view definitions

---
 .../src/components/app/forms/Form.svelte      | 17 ++++----
 .../src/components/app/forms/InnerForm.svelte |  6 +--
 .../src/components/app/forms/validation.js    |  8 ++--
 packages/client/src/utils/schema.js           | 39 ++++++++++++++-----
 packages/frontend-core/src/api/viewsV2.js     |  1 +
 5 files changed, 45 insertions(+), 26 deletions(-)

diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte
index fe02dc665db..88333f75656 100644
--- a/packages/client/src/components/app/forms/Form.svelte
+++ b/packages/client/src/components/app/forms/Form.svelte
@@ -3,6 +3,7 @@
   import InnerForm from "./InnerForm.svelte"
   import { Helpers } from "@budibase/bbui"
   import { writable } from "svelte/store"
+  import { fetchDatasourceDefinition } from "utils/schema"
 
   export let dataSource
   export let theme
@@ -32,9 +33,9 @@
     return parsedFormStep
   }
 
-  let loaded = false
+  let definition
   let schema
-  let table
+  let loaded = false
   let currentStep = getContext("current-step") || writable(getInitialFormStep())
 
   $: fetchSchema(dataSource)
@@ -84,12 +85,10 @@
 
   // Fetches the form schema from this form's dataSource
   const fetchSchema = async dataSource => {
-    if (dataSource?.tableId && !dataSource?.type?.startsWith("query")) {
-      try {
-        table = await API.fetchTableDefinition(dataSource.tableId)
-      } catch (error) {
-        table = null
-      }
+    try {
+      definition = await fetchDatasourceDefinition(dataSource)
+    } catch (error) {
+      definition = null
     }
     const res = await fetchDatasourceSchema(dataSource)
     schema = res || {}
@@ -121,7 +120,7 @@
       {readonly}
       {actionType}
       {schema}
-      {table}
+      {definition}
       {initialValues}
       {disableSchemaValidation}
       {editAutoColumns}
diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte
index bb2f51fc955..91a444aa6a6 100644
--- a/packages/client/src/components/app/forms/InnerForm.svelte
+++ b/packages/client/src/components/app/forms/InnerForm.svelte
@@ -10,7 +10,7 @@
   export let initialValues
   export let size
   export let schema
-  export let table
+  export let definition
   export let disableSchemaValidation = false
   export let editAutoColumns = false
 
@@ -164,7 +164,7 @@
         schemaConstraints,
         validationRules,
         field,
-        table
+        definition
       )
 
       // Sanitise the default value to ensure it doesn't contain invalid data
@@ -338,7 +338,7 @@
         schemaConstraints,
         validationRules,
         field,
-        table
+        definition
       )
 
       // Update validator
diff --git a/packages/client/src/components/app/forms/validation.js b/packages/client/src/components/app/forms/validation.js
index 46a5330cf3f..ad3e5ae6897 100644
--- a/packages/client/src/components/app/forms/validation.js
+++ b/packages/client/src/components/app/forms/validation.js
@@ -5,17 +5,17 @@ import { Helpers } from "@budibase/bbui"
 /**
  * Creates a validation function from a combination of schema-level constraints
  * and custom validation rules
- * @param schemaConstraints any schema level constraints from the table
+ * @param schemaConstraints any schema level constraints from the datasource
  * @param customRules any custom validation rules
  * @param field the field name we are evaluating
- * @param table the definition of the table we are evaluating
+ * @param definition the definition of the datasource we are evaluating
  * @returns {function} a validator function which accepts test values
  */
 export const createValidatorFromConstraints = (
   schemaConstraints,
   customRules,
   field,
-  table
+  definition
 ) => {
   let rules = []
 
@@ -23,7 +23,7 @@ export const createValidatorFromConstraints = (
   if (schemaConstraints) {
     // Required constraint
     if (
-      field === table?.primaryDisplay ||
+      field === definition?.primaryDisplay ||
       schemaConstraints.presence?.allowEmpty === false ||
       schemaConstraints.presence === true
     ) {
diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index e2399e87383..ec1cef53cee 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -10,16 +10,13 @@ import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
 import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
 
 /**
- * Fetches the schema of any kind of datasource.
+ * Constructs a fetch instance for a given datasource.
  * All datasource fetch classes implement their own functionality to get the
  * schema of a datasource of their respective types.
- * @param datasource the datasource to fetch the schema for
- * @param options options for enriching the schema
+ * @param datasource the datasource
+ * @returns
  */
-export const fetchDatasourceSchema = async (
-  datasource,
-  options = { enrichRelationships: false, formSchema: false }
-) => {
+const getDatasourceFetchInstance = datasource => {
   const handler = {
     table: TableFetch,
     view: ViewFetch,
@@ -34,10 +31,23 @@ export const fetchDatasourceSchema = async (
   if (!handler) {
     return null
   }
-  const instance = new handler({ API })
+  return new handler({ API })
+}
 
-  // Get the datasource definition and then schema
-  const definition = await instance.getDefinition(datasource)
+/**
+ * Fetches the schema of any kind of datasource.
+ * @param datasource the datasource to fetch the schema for
+ * @param options options for enriching the schema
+ */
+export const fetchDatasourceSchema = async (
+  datasource,
+  options = { enrichRelationships: false, formSchema: false }
+) => {
+  const instance = getDatasourceFetchInstance(datasource)
+  const definition = await instance?.getDefinition(datasource)
+  if (!definition) {
+    return null
+  }
 
   // Get the normal schema as long as we aren't wanting a form schema
   let schema
@@ -75,6 +85,15 @@ export const fetchDatasourceSchema = async (
   return instance.enrichSchema(schema)
 }
 
+/**
+ * Fetches the definition of any kind of datasource.
+ * @param datasource the datasource to fetch the schema for
+ */
+export const fetchDatasourceDefinition = async datasource => {
+  const instance = getDatasourceFetchInstance(datasource)
+  return await instance?.getDefinition(datasource)
+}
+
 /**
  * Fetches the schema of relationship fields for a SQL table schema
  * @param schema the schema to enrich
diff --git a/packages/frontend-core/src/api/viewsV2.js b/packages/frontend-core/src/api/viewsV2.js
index a2072c2e1da..d1cb07c6b09 100644
--- a/packages/frontend-core/src/api/viewsV2.js
+++ b/packages/frontend-core/src/api/viewsV2.js
@@ -6,6 +6,7 @@ export const buildViewV2Endpoints = API => ({
   fetchDefinition: async viewId => {
     return await API.get({
       url: `/api/v2/views/${encodeURIComponent(viewId)}`,
+      cache: true,
     })
   },
   /**

From 7cce68f126b4405a7f5276443d4f54e0f00558b3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 11:44:03 +0000
Subject: [PATCH 429/455] Provide fetchDatasourceDefinition via SDK

---
 packages/client/src/components/app/forms/Form.svelte | 4 ++--
 packages/client/src/sdk.js                           | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte
index 88333f75656..2071e85fccf 100644
--- a/packages/client/src/components/app/forms/Form.svelte
+++ b/packages/client/src/components/app/forms/Form.svelte
@@ -3,7 +3,6 @@
   import InnerForm from "./InnerForm.svelte"
   import { Helpers } from "@budibase/bbui"
   import { writable } from "svelte/store"
-  import { fetchDatasourceDefinition } from "utils/schema"
 
   export let dataSource
   export let theme
@@ -23,7 +22,8 @@
 
   const context = getContext("context")
   const component = getContext("component")
-  const { API, fetchDatasourceSchema } = getContext("sdk")
+  const { API, fetchDatasourceSchema, detchDatasourceDefinition } =
+    getContext("sdk")
 
   const getInitialFormStep = () => {
     const parsedFormStep = parseInt(initialFormStep)
diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index 50d3f857d5c..bd51924bf49 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -26,7 +26,10 @@ import Provider from "components/context/Provider.svelte"
 import Block from "components/Block.svelte"
 import BlockComponent from "components/BlockComponent.svelte"
 import { ActionTypes } from "./constants"
-import { fetchDatasourceSchema } from "./utils/schema.js"
+import {
+  fetchDatasourceSchema,
+  fetchDatasourceDefinition,
+} from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"
@@ -66,6 +69,7 @@ export default {
   linkable,
   getAction,
   fetchDatasourceSchema,
+  fetchDatasourceDefinition,
   fetchData,
   QueryUtils,
   ContextScopes: Constants.ContextScopes,

From fc3fcb5cbdc6c55bc64b744b42c28476925b528b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 11:48:16 +0000
Subject: [PATCH 430/455] Lint

---
 packages/client/src/components/app/forms/Form.svelte | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte
index 2071e85fccf..f470297d782 100644
--- a/packages/client/src/components/app/forms/Form.svelte
+++ b/packages/client/src/components/app/forms/Form.svelte
@@ -22,8 +22,7 @@
 
   const context = getContext("context")
   const component = getContext("component")
-  const { API, fetchDatasourceSchema, detchDatasourceDefinition } =
-    getContext("sdk")
+  const { fetchDatasourceSchema, fetchDatasourceDefinition } = getContext("sdk")
 
   const getInitialFormStep = () => {
     const parsedFormStep = parseInt(initialFormStep)

From b8f27b9bf77e5a8b8d7014ac6ac20348d5d884a8 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 12:09:01 +0000
Subject: [PATCH 431/455] Fix table imports

---
 .../DataTable/buttons/ImportButton.svelte     |  30 +--
 .../ExistingTableDataImport.svelte            | 181 ++++++++----------
 .../TableNavigator/TableDataImport.svelte     |  45 +++--
 .../modals/CreateTableModal.svelte            |  29 +--
 4 files changed, 128 insertions(+), 157 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
index 5e9850b290f..a2e7a2a1943 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
@@ -1,14 +1,7 @@
 <script>
-  import {
-    ActionButton,
-    Label,
-    Button,
-    Body,
-    Layout,
-    notifications,
-  } from "@budibase/bbui"
+  import { ActionButton, Button, Body, notifications } from "@budibase/bbui"
   import DetailPopover from "components/common/DetailPopover.svelte"
-  import TableDataImport from "components/backend/TableNavigator/TableDataImport.svelte"
+  import ExistingTableDataImport from "components/backend/TableNavigator/ExistingTableDataImport.svelte"
   import { createEventDispatcher } from "svelte"
   import { API } from "api"
 
@@ -72,17 +65,14 @@
     Import rows to an existing table from a CSV or JSON file. Only columns from
     the file which exist in the table will be imported.
   </Body>
-  <Layout gap="XS" noPadding>
-    <Label grey extraSmall>CSV or JSON file to import</Label>
-    <TableDataImport
-      {tableId}
-      {tableType}
-      bind:rows
-      bind:allValid
-      bind:displayColumn
-      bind:identifierFields
-    />
-  </Layout>
+  <ExistingTableDataImport
+    {tableId}
+    {tableType}
+    bind:rows
+    bind:allValid
+    bind:displayColumn
+    bind:identifierFields
+  />
   <div>
     <Button cta disabled={loading || !allValid} on:click={importData}>
       Import
diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
index f1d85a6a304..ce966d0daa9 100644
--- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
@@ -4,7 +4,7 @@
     BBReferenceFieldSubType,
     SourceName,
   } from "@budibase/types"
-  import { Select, Toggle, Multiselect } from "@budibase/bbui"
+  import { Select, Toggle, Multiselect, Label, Layout } from "@budibase/bbui"
   import { DB_TYPE_INTERNAL } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
@@ -140,84 +140,91 @@
   }
 </script>
 
-<div class="dropzone">
-  <input
-    disabled={!schema || loading}
-    id="file-upload"
-    accept="text/csv,application/json"
-    type="file"
-    on:change={handleFile}
-  />
-  <label for="file-upload" class:uploaded={rows.length > 0}>
-    {#if loading}
-      loading...
-    {:else if error}
-      error: {error}
-    {:else if fileName}
-      {fileName}
-    {:else}
-      Upload
+<Layout gap="S" noPadding>
+  <Layout noPadding gap="XS">
+    <Label grey extraSmall>CSV or JSON file to import</Label>
+    <div class="dropzone">
+      <input
+        disabled={!schema || loading}
+        id="file-upload"
+        accept="text/csv,application/json"
+        type="file"
+        on:change={handleFile}
+      />
+      <label for="file-upload" class:uploaded={rows.length > 0}>
+        {#if loading}
+          loading...
+        {:else if error}
+          error: {error}
+        {:else if fileName}
+          {fileName}
+        {:else}
+          Upload
+        {/if}
+      </label>
+    </div>
+  </Layout>
+
+  {#if fileName && Object.keys(validation).length === 0}
+    <div>No valid fields - please try another file.</div>
+  {:else if fileName && rows.length > 0 && !error}
+    <div>
+      {#each Object.keys(validation) as name}
+        <div class="field">
+          <span>{name}</span>
+          <Select
+            value={`${schema[name]?.type}${schema[name]?.subtype || ""}`}
+            options={typeOptions}
+            placeholder={null}
+            getOptionLabel={option => option.label}
+            getOptionValue={option => option.value}
+            disabled
+          />
+          <span
+            class={loading || validation[name]
+              ? "fieldStatusSuccess"
+              : "fieldStatusFailure"}
+          >
+            {validation[name] ? "Success" : "Failure"}
+          </span>
+        </div>
+      {/each}
+    </div>
+    <!-- SQL Server doesn't yet support overwriting rows by existing keys -->
+    {#if datasource?.source !== SourceName.SQL_SERVER}
+      <Toggle
+        bind:value={updateExistingRows}
+        on:change={() => (identifierFields = [])}
+        thin
+        text="Update existing rows"
+      />
     {/if}
-  </label>
-</div>
-{#if fileName && Object.keys(validation).length === 0}
-  <p>No valid fields, try another file</p>
-{:else if rows.length > 0 && !error}
-  <div class="schema-fields">
-    {#each Object.keys(validation) as name}
-      <div class="field">
-        <span>{name}</span>
-        <Select
-          value={`${schema[name]?.type}${schema[name]?.subtype || ""}`}
-          options={typeOptions}
-          placeholder={null}
-          getOptionLabel={option => option.label}
-          getOptionValue={option => option.value}
-          disabled
+    {#if updateExistingRows}
+      {#if tableType === DB_TYPE_INTERNAL}
+        <Multiselect
+          label="Identifier field(s)"
+          options={Object.keys(validation)}
+          bind:value={identifierFields}
         />
-        <span
-          class={loading || validation[name]
-            ? "fieldStatusSuccess"
-            : "fieldStatusFailure"}
-        >
-          {validation[name] ? "Success" : "Failure"}
-        </span>
-      </div>
-    {/each}
-  </div>
-  <br />
-  <!-- SQL Server doesn't yet support overwriting rows by existing keys -->
-  {#if datasource?.source !== SourceName.SQL_SERVER}
-    <Toggle
-      bind:value={updateExistingRows}
-      on:change={() => (identifierFields = [])}
-      thin
-      text="Update existing rows"
-    />
-  {/if}
-  {#if updateExistingRows}
-    {#if tableType === DB_TYPE_INTERNAL}
-      <Multiselect
-        label="Identifier field(s)"
-        options={Object.keys(validation)}
-        bind:value={identifierFields}
-      />
-    {:else}
-      <p>Rows will be updated based on the table's primary key.</p>
+      {:else}
+        <div>Rows will be updated based on the table's primary key.</div>
+      {/if}
+    {/if}
+    {#if invalidColumns.length > 0}
+      <Layout noPadding gap="XS">
+        <div>
+          The following columns are present in the data you wish to import, but
+          do not match the schema of this table and will be ignored:
+        </div>
+        <div>
+          {#each invalidColumns as column}
+            - {column}<br />
+          {/each}
+        </div>
+      </Layout>
     {/if}
   {/if}
-  {#if invalidColumns.length > 0}
-    <p class="spectrum-FieldLabel spectrum-FieldLabel--sizeM">
-      The following columns are present in the data you wish to import, but do
-      not match the schema of this table and will be ignored.
-    </p>
-    <ul class="ignoredList">
-      {#each invalidColumns as column}
-        <li>{column}</li>
-      {/each}
-    </ul>
-  {/if}
-{/if}
+</Layout>
 
 <style>
   .dropzone {
@@ -228,11 +235,9 @@
     border-radius: 10px;
     transition: all 0.3s;
   }
-
   input {
     display: none;
   }
-
   label {
     font-family: var(--font-sans);
     cursor: pointer;
@@ -240,7 +245,6 @@
     box-sizing: border-box;
     overflow: hidden;
     border-radius: var(--border-radius-s);
-    color: var(--ink);
     padding: var(--spacing-m) var(--spacing-l);
     transition: all 0.2s ease 0s;
     display: inline-flex;
@@ -254,20 +258,14 @@
     align-items: center;
     justify-content: center;
     width: 100%;
-    background-color: var(--grey-2);
-    font-size: var(--font-size-xs);
+    background-color: var(--spectrum-global-color-gray-300);
+    font-size: var(--font-size-s);
     line-height: normal;
     border: var(--border-transparent);
   }
-
   .uploaded {
-    color: var(--blue);
+    color: var(--spectrum-global-color-blue-600);
   }
-
-  .schema-fields {
-    margin-top: var(--spacing-xl);
-  }
-
   .field {
     display: grid;
     grid-template-columns: 2fr 2fr 1fr auto;
@@ -276,23 +274,14 @@
     grid-gap: var(--spacing-m);
     font-size: var(--spectrum-global-dimension-font-size-75);
   }
-
   .fieldStatusSuccess {
     color: var(--green);
     justify-self: center;
     font-weight: 600;
   }
-
   .fieldStatusFailure {
     color: var(--red);
     justify-self: center;
     font-weight: 600;
   }
-
-  .ignoredList {
-    margin: 0;
-    padding: 0;
-    list-style: none;
-    font-size: var(--spectrum-global-dimension-font-size-75);
-  }
 </style>
diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 77437628774..1694ec8b0e2 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { Select, Icon, Layout } from "@budibase/bbui"
+  import { Select, Icon, Layout, Label } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { utils } from "@budibase/shared-core"
   import { canBeDisplayColumn } from "@budibase/frontend-core"
@@ -185,25 +185,30 @@
 </script>
 
 <Layout noPadding gap="S">
-  <div class="dropzone">
-    <input
-      bind:this={fileInput}
-      disabled={loading}
-      id="file-upload"
-      accept="text/csv,application/json"
-      type="file"
-      on:change={handleFile}
-    />
-    <label for="file-upload" class:uploaded={rawRows.length > 0}>
-      {#if error}
-        Error: {error}
-      {:else if fileName}
-        {fileName}
-      {:else}
-        Upload
-      {/if}
-    </label>
-  </div>
+  <Layout gap="XS" noPadding>
+    <Label grey extraSmall>
+      Create a Table from a CSV or JSON file (Optional)
+    </Label>
+    <div class="dropzone">
+      <input
+        bind:this={fileInput}
+        disabled={loading}
+        id="file-upload"
+        accept="text/csv,application/json"
+        type="file"
+        on:change={handleFile}
+      />
+      <label for="file-upload" class:uploaded={rawRows.length > 0}>
+        {#if error}
+          Error: {error}
+        {:else if fileName}
+          {fileName}
+        {:else}
+          Upload
+        {/if}
+      </label>
+    </div>
+  </Layout>
 
   {#if rawRows.length > 0 && !error}
     <div>
diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
index 8a5484e2b21..5596ade5730 100644
--- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
@@ -1,13 +1,7 @@
 <script>
   import { goto, url } from "@roxi/routify"
   import { tables, datasources } from "stores/builder"
-  import {
-    notifications,
-    Input,
-    Label,
-    ModalContent,
-    Layout,
-  } from "@budibase/bbui"
+  import { notifications, Input, ModalContent } from "@budibase/bbui"
   import TableDataImport from "../TableDataImport.svelte"
   import {
     BUDIBASE_INTERNAL_DB_ID,
@@ -101,18 +95,11 @@
     bind:value={name}
     {error}
   />
-  <div>
-    <Layout gap="XS" noPadding>
-      <Label grey extraSmall
-        >Create a Table from a CSV or JSON file (Optional)</Label
-      >
-      <TableDataImport
-        {promptUpload}
-        bind:rows
-        bind:schema
-        bind:allValid
-        bind:displayColumn
-      />
-    </Layout>
-  </div>
+  <TableDataImport
+    {promptUpload}
+    bind:rows
+    bind:schema
+    bind:allValid
+    bind:displayColumn
+  />
 </ModalContent>

From f94d8ba870acb3a906d0a3fbb1356de4d0673d80 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 29 Oct 2024 12:18:36 +0000
Subject: [PATCH 432/455] Fixing merge artifact

---
 packages/backend-core/src/security/roles.ts | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index b7060ebdd7a..4076be93a04 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -238,8 +238,8 @@ export function builtinRoleToNumber(id: string) {
   const builtins = getBuiltinRoles()
   const MAX = Object.values(builtins).length + 1
   if (
-    compareRoleIds(id, BUILTIN_IDS.ADMIN) ||
-    compareRoleIds(id, BUILTIN_IDS.BUILDER)
+    roleIDsAreEqual(id, BUILTIN_IDS.ADMIN) ||
+    roleIDsAreEqual(id, BUILTIN_IDS.BUILDER)
   ) {
     return MAX
   }
@@ -278,7 +278,7 @@ export async function roleToNumber(id: string) {
       const highestBuiltin: number | undefined = role.inherits
         .map(roleId => {
           const foundRole = hierarchy.find(role =>
-            compareRoleIds(role._id!, roleId)
+            roleIDsAreEqual(role._id!, roleId)
           )
           if (foundRole) {
             return findNumber(foundRole) + 1
@@ -403,7 +403,7 @@ async function getAllUserRoles(
 ): Promise<RoleDoc[]> {
   const allRoles = await getAllRoles()
   // admins have access to all roles
-  if (compareRoleIds(userRoleId, BUILTIN_IDS.ADMIN)) {
+  if (roleIDsAreEqual(userRoleId, BUILTIN_IDS.ADMIN)) {
     return allRoles
   }
 
@@ -515,7 +515,7 @@ export async function getAllRoles(appId?: string): Promise<RoleDoc[]> {
     for (let builtinRoleId of externalBuiltinRoles) {
       const builtinRole = builtinRoles[builtinRoleId]
       const dbBuiltin = roles.filter(dbRole =>
-        compareRoleIds(dbRole._id!, builtinRoleId)
+        roleIDsAreEqual(dbRole._id!, builtinRoleId)
       )[0]
       if (dbBuiltin == null) {
         roles.push(builtinRole || builtinRoles.BASIC)
@@ -574,9 +574,9 @@ export class AccessController {
     if (
       tryingRoleId == null ||
       tryingRoleId === "" ||
-      compareRoleIds(tryingRoleId, BUILTIN_IDS.BUILDER) ||
-      compareRoleIds(userRoleId!, tryingRoleId) ||
-      compareRoleIds(userRoleId!, BUILTIN_IDS.BUILDER)
+      roleIDsAreEqual(tryingRoleId, BUILTIN_IDS.BUILDER) ||
+      roleIDsAreEqual(userRoleId!, tryingRoleId) ||
+      roleIDsAreEqual(userRoleId!, BUILTIN_IDS.BUILDER)
     ) {
       return true
     }

From 721b3ab6107aa628398b2aa4bf07e4e4330a3631 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 12:56:41 +0000
Subject: [PATCH 433/455] Account for logicalOperator inside UISearchFilter
 groups

---
 packages/shared-core/src/filters.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 003e6b5082e..7720fe2b8d0 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -488,7 +488,13 @@ export function buildQuery(
       if (onEmptyFilter) {
         query.onEmptyFilter = onEmptyFilter
       }
-      const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
+
+      // logicalOperator takes precendence over allOr
+      let operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
+      if (group.logicalOperator) {
+        operator = logicalOperatorFromUI(group.logicalOperator)
+      }
+
       return {
         [operator]: { conditions: filters.map(buildCondition).filter(f => f) },
       }

From 765b3d2f9796e83ad44262b01b03a6e59bff27c9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 15:10:02 +0000
Subject: [PATCH 434/455] Add tests for handling logical operators in
 UISearchFilter

---
 .../src/api/routes/tests/viewV2.spec.ts       | 91 +++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 44e5875b7e8..1f7000fe436 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -4298,6 +4298,97 @@ describe.each([
                 },
               ],
             },
+            {
+              name: "can handle logical operator any",
+              insert: [{ string: "bar" }, { string: "foo" }],
+              query: {
+                groups: [
+                  {
+                    logicalOperator: UILogicalOperator.ANY,
+                    filters: [
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "string",
+                        value: "foo",
+                      },
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "string",
+                        value: "bar",
+                      },
+                    ],
+                  },
+                ],
+              },
+              searchOpts: {
+                sort: "string",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ string: "bar" }, { string: "foo" }],
+            },
+            {
+              name: "can handle logical operator all",
+              insert: [
+                { string: "bar", num: 1 },
+                { string: "foo", num: 2 },
+              ],
+              query: {
+                groups: [
+                  {
+                    logicalOperator: UILogicalOperator.ALL,
+                    filters: [
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "string",
+                        value: "foo",
+                      },
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "num",
+                        value: 2,
+                      },
+                    ],
+                  },
+                ],
+              },
+              searchOpts: {
+                sort: "string",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ string: "foo", num: 2 }],
+            },
+            {
+              name: "overrides allOr with logical operators",
+              insert: [
+                { string: "bar", num: 1 },
+                { string: "foo", num: 1 },
+              ],
+              query: {
+                groups: [
+                  {
+                    logicalOperator: UILogicalOperator.ALL,
+                    filters: [
+                      { operator: "allOr" },
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "string",
+                        value: "foo",
+                      },
+                      {
+                        operator: BasicOperator.EQUAL,
+                        field: "num",
+                        value: 1,
+                      },
+                    ],
+                  },
+                ],
+              },
+              searchOpts: {
+                sort: "string",
+                sortOrder: SortOrder.ASCENDING,
+              },
+              expected: [{ string: "foo", num: 1 }],
+            },
           ]
 
           it.each(testCases)(

From 599370b6ed85f6d8c25257f7e6ab4d50e153f97d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 29 Oct 2024 15:19:32 +0000
Subject: [PATCH 435/455] Update tests

---
 .../server/src/api/routes/tests/viewV2.spec.ts   | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 1f7000fe436..15b9a29e11a 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -4329,8 +4329,8 @@ describe.each([
             {
               name: "can handle logical operator all",
               insert: [
-                { string: "bar", num: 1 },
-                { string: "foo", num: 2 },
+                { string: "bar", number: 1 },
+                { string: "foo", number: 2 },
               ],
               query: {
                 groups: [
@@ -4344,7 +4344,7 @@ describe.each([
                       },
                       {
                         operator: BasicOperator.EQUAL,
-                        field: "num",
+                        field: "number",
                         value: 2,
                       },
                     ],
@@ -4355,13 +4355,13 @@ describe.each([
                 sort: "string",
                 sortOrder: SortOrder.ASCENDING,
               },
-              expected: [{ string: "foo", num: 2 }],
+              expected: [{ string: "foo", number: 2 }],
             },
             {
               name: "overrides allOr with logical operators",
               insert: [
-                { string: "bar", num: 1 },
-                { string: "foo", num: 1 },
+                { string: "bar", number: 1 },
+                { string: "foo", number: 1 },
               ],
               query: {
                 groups: [
@@ -4376,7 +4376,7 @@ describe.each([
                       },
                       {
                         operator: BasicOperator.EQUAL,
-                        field: "num",
+                        field: "number",
                         value: 1,
                       },
                     ],
@@ -4387,7 +4387,7 @@ describe.each([
                 sort: "string",
                 sortOrder: SortOrder.ASCENDING,
               },
-              expected: [{ string: "foo", num: 1 }],
+              expected: [{ string: "foo", number: 1 }],
             },
           ]
 

From 313bd9b017b8e7132b7f1e054d093a4d03683a8e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 30 Oct 2024 11:21:37 +0000
Subject: [PATCH 436/455] add AI column from import if already defined

---
 packages/builder/src/components/backend/DataTable/formula.js  | 1 +
 .../design/settings/controls/Explanation/lines/Column.svelte  | 3 +++
 packages/server/src/utilities/rowProcessor/index.ts           | 4 ----
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/formula.js b/packages/builder/src/components/backend/DataTable/formula.js
index 7220a5ba4f5..a9491abfef0 100644
--- a/packages/builder/src/components/backend/DataTable/formula.js
+++ b/packages/builder/src/components/backend/DataTable/formula.js
@@ -8,6 +8,7 @@ const MAX_DEPTH = 1
 
 const TYPES_TO_SKIP = [
   FieldType.FORMULA,
+  FieldType.AI,
   FieldType.LONGFORM,
   FieldType.SIGNATURE_SINGLE,
   FieldType.ATTACHMENTS,
diff --git a/packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte b/packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte
index 4552a54ba39..ef2616f562a 100644
--- a/packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte
+++ b/packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte
@@ -80,6 +80,9 @@
     if (columnType === FieldType.FORMULA) {
       return "https://docs.budibase.com/docs/formula"
     }
+    if (columnType === FieldType.AI) {
+      return "https://docs.budibase.com/docs/ai"
+    }
     if (columnType === FieldType.OPTIONS) {
       return "https://docs.budibase.com/docs/options"
     }
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index ed74394a85a..910e9d220f4 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -216,10 +216,6 @@ export async function inputProcessing(
     if (field.type === FieldType.FORMULA) {
       delete clonedRow[key]
     }
-    // remove any AI values, they are to be generated
-    if (field.type === FieldType.AI) {
-      delete clonedRow[key]
-    }
     // otherwise coerce what is there to correct types
     else {
       clonedRow[key] = coerce(value, field.type)

From b06074128fe666f6d6a507f2a1c2c6fd32e9d1cc Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 30 Oct 2024 11:27:49 +0000
Subject: [PATCH 437/455] unit test to cover

---
 .../api/controllers/table/tests/utils.spec.ts | 62 ++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/table/tests/utils.spec.ts b/packages/server/src/api/controllers/table/tests/utils.spec.ts
index 68b00d3268a..0cac17dd069 100644
--- a/packages/server/src/api/controllers/table/tests/utils.spec.ts
+++ b/packages/server/src/api/controllers/table/tests/utils.spec.ts
@@ -1,4 +1,4 @@
-import { AutoFieldSubType, FieldType } from "@budibase/types"
+import { AIOperationEnum, AutoFieldSubType, FieldType } from "@budibase/types"
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
 import { importToRows } from "../utils"
 
@@ -92,5 +92,65 @@ describe("utils", () => {
         expect(result).toHaveLength(3)
       })
     })
+
+    it("Imports write as expected with AI columns", async () => {
+      await config.doInContext(config.appId, async () => {
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            autoId: {
+              name: "autoId",
+              type: FieldType.NUMBER,
+              subtype: AutoFieldSubType.AUTO_ID,
+              autocolumn: true,
+              constraints: {
+                type: FieldType.NUMBER,
+                presence: true,
+              },
+            },
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+              constraints: {
+                type: FieldType.STRING,
+                presence: true,
+              },
+            },
+            aicol: {
+              name: "aicol",
+              type: FieldType.AI,
+              operation: AIOperationEnum.PROMPT,
+              prompt: "Test prompt"
+            },
+          },
+        })
+
+        const data = [
+          { name: "Alice", aicol: "test" },
+          { name: "Bob", aicol: "test" },
+          { name: "Claire", aicol: "test" }
+        ]
+
+        const result = await importToRows(data, table, config.user?._id)
+        expect(result).toEqual([
+          expect.objectContaining({
+            autoId: 1,
+            name: "Alice",
+            aicol: "test",
+          }),
+          expect.objectContaining({
+            autoId: 2,
+            name: "Bob",
+            aicol: "test",
+          }),
+          expect.objectContaining({
+            autoId: 3,
+            name: "Claire",
+            aicol: "test",
+          }),
+        ])
+      })
+    })
   })
 })

From d9b685811ddf0c0c58f3a92ac4afb0664d3e119a Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 30 Oct 2024 11:32:54 +0000
Subject: [PATCH 438/455] lint

---
 packages/server/src/api/controllers/table/tests/utils.spec.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/table/tests/utils.spec.ts b/packages/server/src/api/controllers/table/tests/utils.spec.ts
index 0cac17dd069..f9d6d707e29 100644
--- a/packages/server/src/api/controllers/table/tests/utils.spec.ts
+++ b/packages/server/src/api/controllers/table/tests/utils.spec.ts
@@ -121,7 +121,7 @@ describe("utils", () => {
               name: "aicol",
               type: FieldType.AI,
               operation: AIOperationEnum.PROMPT,
-              prompt: "Test prompt"
+              prompt: "Test prompt",
             },
           },
         })
@@ -129,7 +129,7 @@ describe("utils", () => {
         const data = [
           { name: "Alice", aicol: "test" },
           { name: "Bob", aicol: "test" },
-          { name: "Claire", aicol: "test" }
+          { name: "Claire", aicol: "test" },
         ]
 
         const result = await importToRows(data, table, config.user?._id)

From 53fc45f07ee784141631bdd93fe5d2997d3fb95c Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 30 Oct 2024 14:14:49 +0000
Subject: [PATCH 439/455] make ai columns respect feature flags

---
 .../src/components/automation/SetupPanel/CronBuilder.svelte  | 5 ++---
 .../backend/DataTable/modals/CreateEditColumn.svelte         | 5 ++---
 .../[application]/data/table/[tableId]/[viewId]/index.svelte | 4 ++--
 .../app/[application]/data/table/[tableId]/index.svelte      | 5 ++---
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte b/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte
index 1bba349244e..51538944f4c 100644
--- a/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte
@@ -9,7 +9,7 @@
   } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { flags } from "stores/builder"
-  import { licensing } from "stores/portal"
+  import { featureFlags } from "stores/portal"
   import { API } from "api"
   import MagicWand from "../../../../assets/MagicWand.svelte"
 
@@ -26,8 +26,7 @@
   let aiCronPrompt = ""
   let loadingAICronExpression = false
 
-  $: aiEnabled =
-    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
+  $: aiEnabled = $featureFlags.AI_CUSTOM_CONFIGS || $featureFlags.BUDIBASE_AI
   $: {
     if (cronExpression) {
       try {
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 143e292c551..d16bca32030 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -26,7 +26,7 @@
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
-  import { licensing } from "stores/portal"
+  import { featureFlags } from "stores/portal"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -101,8 +101,7 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
-  $: aiEnabled =
-    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
+  $: aiEnabled = $featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
index bc21b04ad0b..39b1dc47682 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "stores/builder"
-  import { admin, themeStore, licensing } from "stores/portal"
+  import { admin, themeStore, featureFlags } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import { notifications } from "@budibase/bbui"
@@ -53,7 +53,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
-  aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
+  aiEnabled={$featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index e300768bf1d..a70bb6caa56 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -8,7 +8,7 @@
     rowActions,
     roles,
   } from "stores/builder"
-  import { themeStore, admin, licensing } from "stores/portal"
+  import { themeStore, admin, licensing, featureFlags } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
@@ -130,8 +130,7 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
-    aiEnabled={$licensing.budibaseAIEnabled ||
-      $licensing.customAIConfigsEnabled}
+    aiEnabled={$featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}

From 3f3160a44c06517558a52d51ad9f2f018c1d3a19 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 30 Oct 2024 14:17:50 +0000
Subject: [PATCH 440/455] default store values for AI features

---
 .../builder/app/[application]/data/table/[tableId]/index.svelte | 2 +-
 packages/builder/src/stores/portal/featureFlags.js              | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
index a70bb6caa56..5684e77e31b 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
@@ -8,7 +8,7 @@
     rowActions,
     roles,
   } from "stores/builder"
-  import { themeStore, admin, licensing, featureFlags } from "stores/portal"
+  import { themeStore, admin, featureFlags } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
diff --git a/packages/builder/src/stores/portal/featureFlags.js b/packages/builder/src/stores/portal/featureFlags.js
index 796fe76c979..902a8b488b7 100644
--- a/packages/builder/src/stores/portal/featureFlags.js
+++ b/packages/builder/src/stores/portal/featureFlags.js
@@ -4,6 +4,8 @@ import { auth } from "stores/portal"
 export const INITIAL_FEATUREFLAG_STATE = {
   SQS: false,
   DEFAULT_VALUES: false,
+  BUDIBASE_AI: false,
+  AI_CUSTOM_CONFIGS: false,
 }
 
 export const featureFlags = derived([auth], ([$auth]) => {

From 15ebfdf0e93b228613557ebf95a027c563d58b03 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 30 Oct 2024 14:56:11 +0000
Subject: [PATCH 441/455] Fix settings added and removed by conditions not
 triggering component refresh

---
 packages/client/src/components/Component.svelte | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte
index 9c8aada1bcc..7a1341c67c5 100644
--- a/packages/client/src/components/Component.svelte
+++ b/packages/client/src/components/Component.svelte
@@ -554,7 +554,12 @@
       cachedSettings = { ...allSettings }
       initialSettings = cachedSettings
     } else {
-      Object.keys(allSettings).forEach(key => {
+      // We need to compare all keys from both the current and previous settings, as
+      // keys may have disappeared in the current set which would otherwise be ignored
+      // if we only checked the current set keys
+      const keys = new Set(Object.keys(allSettings))
+      Object.keys(cachedSettings).forEach(key => keys.add(key))
+      keys.forEach(key => {
         const same = propsAreSame(allSettings[key], cachedSettings[key])
         if (!same) {
           // Updated cachedSettings (which is assigned by reference to

From 67a988ae2184f4025d62f0addea66462c54a8be4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 30 Oct 2024 15:05:06 +0000
Subject: [PATCH 442/455] Automatically refresh datasources after executing row
 actions via a button action

---
 packages/client/src/utils/buttonActions.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js
index 847a32116d5..e2b00710421 100644
--- a/packages/client/src/utils/buttonActions.js
+++ b/packages/client/src/utils/buttonActions.js
@@ -500,6 +500,10 @@ const rowActionHandler = async action => {
     sourceId: resourceId,
     rowId,
   })
+  // Refresh related datasources
+  await dataSourceStore.actions.invalidateDataSource(resourceId, {
+    invalidateRelationships: true,
+  })
 }
 
 const handlerMap = {

From 8478c3f8b04e476c1c043417a9559af9623c14ac Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 30 Oct 2024 16:29:20 +0000
Subject: [PATCH 443/455] adding assertion for AI

---
 packages/server/src/api/routes/tests/row.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 4461230ce1a..396173dfeef 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2462,7 +2462,6 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
-          [FieldType.AI]: undefined, // generated field
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2625,6 +2624,7 @@ describe.each([
           tableId: tableId,
           createdAt: new Date().toISOString(),
           updatedAt: new Date().toISOString(),
+          ai: null,
         }
         expect(allRows).toEqual([expectedRow, expectedRow, expectedRow])
       })

From b504f77d0f23eec97ab2ec54e4ea2e776a7af1d9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 30 Oct 2024 17:32:49 +0000
Subject: [PATCH 444/455] Fix view calculations when relationships are present.

---
 packages/backend-core/src/sql/sql.ts          |  10 +-
 .../api/controllers/row/ExternalRequest.ts    |  23 ++--
 .../src/api/routes/tests/viewV2.spec.ts       | 103 ++++++++++++++++++
 3 files changed, 119 insertions(+), 17 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 91dceaf5a66..f62bb0f7e3d 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -529,7 +529,7 @@ class InternalBuilder {
         if (!matchesTableName) {
           updatedKey = filterKey.replace(
             new RegExp(`^${relationship.column}.`),
-            `${aliases![relationship.tableName]}.`
+            `${aliases?.[relationship.tableName] || relationship.tableName}.`
           )
         } else {
           updatedKey = filterKey
@@ -1579,7 +1579,7 @@ class InternalBuilder {
     query = this.addFilters(query, filters, { relationship: true })
 
     // handle relationships with a CTE for all others
-    if (relationships?.length) {
+    if (relationships?.length && aggregations.length === 0) {
       const mainTable =
         this.query.tableAliases?.[this.query.endpoint.entityId] ||
         this.query.endpoint.entityId
@@ -1594,10 +1594,8 @@ class InternalBuilder {
       // add JSON aggregations attached to the CTE
       return this.addJsonRelationships(cte, tableName, relationships)
     }
-    // no relationships found - return query
-    else {
-      return query
-    }
+
+    return query
   }
 
   update(opts: QueryOptions): Knex.QueryBuilder {
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 62aa4d8d2a5..939ea399d9e 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -682,7 +682,19 @@ export class ExternalRequest<T extends Operation> {
     filters = this.prepareFilters(id, filters || {}, table)
     const relationships = buildExternalRelationships(table, this.tables)
 
+    let aggregations: Aggregation[] = []
+    if (sdk.views.isView(this.source)) {
+      const calculationFields = helpers.views.calculationFields(this.source)
+      for (const [key, field] of Object.entries(calculationFields)) {
+        aggregations.push({
+          ...field,
+          name: key,
+        })
+      }
+    }
+
     const incRelationships =
+      aggregations.length === 0 &&
       config.includeSqlRelationships === IncludeRelationship.INCLUDE
 
     // clean up row on ingress using schema
@@ -709,17 +721,6 @@ export class ExternalRequest<T extends Operation> {
       throw "Deletion must be filtered"
     }
 
-    let aggregations: Aggregation[] = []
-    if (sdk.views.isView(this.source)) {
-      const calculationFields = helpers.views.calculationFields(this.source)
-      for (const [key, field] of Object.entries(calculationFields)) {
-        aggregations.push({
-          ...field,
-          name: key,
-        })
-      }
-    }
-
     let json: QueryJson = {
       endpoint: {
         datasourceId: this.datasource._id!,
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 15b9a29e11a..7358cc13474 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -3659,6 +3659,109 @@ describe.each([
               )
             })
 
+            it("should be able to filter on relationships", async () => {
+              const companies = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    name: {
+                      name: "name",
+                      type: FieldType.STRING,
+                    },
+                  },
+                })
+              )
+
+              const employees = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    age: {
+                      type: FieldType.NUMBER,
+                      name: "age",
+                    },
+                    name: {
+                      type: FieldType.STRING,
+                      name: "name",
+                    },
+                    company: {
+                      type: FieldType.LINK,
+                      name: "company",
+                      tableId: companies._id!,
+                      relationshipType: RelationshipType.ONE_TO_MANY,
+                      fieldName: "company",
+                    },
+                  },
+                })
+              )
+
+              const view = await config.api.viewV2.create({
+                tableId: employees._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                queryUI: {
+                  groups: [
+                    {
+                      filters: [
+                        {
+                          operator: BasicOperator.EQUAL,
+                          field: "company.name",
+                          value: "Aperture Science Laboratories",
+                        },
+                      ],
+                    },
+                  ],
+                },
+                schema: {
+                  sum: {
+                    visible: true,
+                    calculationType: CalculationType.SUM,
+                    field: "age",
+                  },
+                },
+              })
+
+              const apertureScience = await config.api.row.save(
+                companies._id!,
+                {
+                  name: "Aperture Science Laboratories",
+                }
+              )
+
+              const blackMesa = await config.api.row.save(companies._id!, {
+                name: "Black Mesa",
+              })
+
+              await config.api.row.save(employees._id!, {
+                name: "Alice",
+                age: 25,
+                company: apertureScience._id,
+              })
+
+              await config.api.row.save(employees._id!, {
+                name: "Bob",
+                age: 30,
+                company: apertureScience._id,
+              })
+
+              await config.api.row.save(employees._id!, {
+                name: "Charly",
+                age: 27,
+                company: blackMesa._id,
+              })
+
+              await config.api.row.save(employees._id!, {
+                name: "Danny",
+                age: 15,
+                company: blackMesa._id,
+              })
+
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {},
+              })
+
+              expect(rows).toHaveLength(1)
+              expect(rows[0].sum).toEqual(55)
+            })
+
             it("should be able to filter rows on the view itself", async () => {
               const table = await config.api.table.save(
                 saveTableRequest({

From e06c0629a0e8be425e51e09ac55451325633f34a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 30 Oct 2024 17:40:15 +0000
Subject: [PATCH 445/455] Respond to PR comments.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 45 +++++++++----------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 7358cc13474..1f2470f097e 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -3730,29 +3730,28 @@ describe.each([
                 name: "Black Mesa",
               })
 
-              await config.api.row.save(employees._id!, {
-                name: "Alice",
-                age: 25,
-                company: apertureScience._id,
-              })
-
-              await config.api.row.save(employees._id!, {
-                name: "Bob",
-                age: 30,
-                company: apertureScience._id,
-              })
-
-              await config.api.row.save(employees._id!, {
-                name: "Charly",
-                age: 27,
-                company: blackMesa._id,
-              })
-
-              await config.api.row.save(employees._id!, {
-                name: "Danny",
-                age: 15,
-                company: blackMesa._id,
-              })
+              await Promise.all([
+                config.api.row.save(employees._id!, {
+                  name: "Alice",
+                  age: 25,
+                  company: apertureScience._id,
+                }),
+                config.api.row.save(employees._id!, {
+                  name: "Bob",
+                  age: 30,
+                  company: apertureScience._id,
+                }),
+                config.api.row.save(employees._id!, {
+                  name: "Charly",
+                  age: 27,
+                  company: blackMesa._id,
+                }),
+                config.api.row.save(employees._id!, {
+                  name: "Danny",
+                  age: 15,
+                  company: blackMesa._id,
+                }),
+              ])
 
               const { rows } = await config.api.viewV2.search(view.id, {
                 query: {},

From 920953a4eeb38eaddf75c07e8d2529f66a9d7d40 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 30 Oct 2024 17:50:04 +0000
Subject: [PATCH 446/455] Throw an error when attempting to bulkImport a
 relationship field into an internal table. It doesn't work yet.

---
 .../server/src/api/controllers/table/utils.ts | 15 ++++++---
 .../server/src/api/routes/tests/row.spec.ts   | 33 +++++++++++++++++++
 2 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index 96c01a15b88..743cce410b0 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -15,7 +15,7 @@ import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { context, events, features } from "@budibase/backend-core"
+import { context, events, features, HTTPError } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
   Database,
@@ -145,14 +145,21 @@ export async function importToRows(
     // the real schema of the table passed in, not the clone used for
     // incrementing auto IDs
     for (const [fieldName, schema] of Object.entries(originalTable.schema)) {
-      const rowVal = Array.isArray(row[fieldName])
-        ? row[fieldName]
-        : [row[fieldName]]
+      if (schema.type === FieldType.LINK) {
+        throw new HTTPError(
+          `Can't bulk import relationship fields for internal databases, found value in field "${fieldName}"`,
+          400
+        )
+      }
+
       if (
         (schema.type === FieldType.OPTIONS ||
           schema.type === FieldType.ARRAY) &&
         row[fieldName]
       ) {
+        const rowVal = Array.isArray(row[fieldName])
+          ? row[fieldName]
+          : [row[fieldName]]
         let merged = [...schema.constraints!.inclusion!, ...rowVal]
         let superSet = new Set(merged)
         schema.constraints!.inclusion = Array.from(superSet)
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 4461230ce1a..4a19995030f 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1823,6 +1823,39 @@ describe.each([
         expect(row.autoId).toEqual(3)
       })
 
+    isInternal &&
+      it("should reject bulkImporting relationship fields", async () => {
+        const table1 = await config.api.table.save(saveTableRequest())
+        const table2 = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              relationship: {
+                name: "relationship",
+                type: FieldType.LINK,
+                tableId: table1._id!,
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                fieldName: "relationship",
+              },
+            },
+          })
+        )
+
+        const table1Row1 = await config.api.row.save(table1._id!, {})
+        await config.api.row.bulkImport(
+          table2._id!,
+          {
+            rows: [{ relationship: [table1Row1._id!] }],
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Can\'t bulk import relationship fields for internal databases, found value in field "relationship"',
+            },
+          }
+        )
+      })
+
     it("should be able to bulkImport rows", async () => {
       const table = await config.api.table.save(
         saveTableRequest({

From 945cf52f02f587501534acbf4b505d7a0ffa325b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 31 Oct 2024 11:54:31 +0000
Subject: [PATCH 447/455] Allow fields on count calculations.

---
 packages/backend-core/src/sql/sql.ts          |  9 +-
 .../src/api/controllers/view/viewsV2.ts       | 30 ++++---
 .../src/api/routes/tests/viewV2.spec.ts       | 84 +++++++++++++++++--
 .../src/sdk/app/rows/search/internal/sqs.ts   |  1 +
 packages/server/src/sdk/app/views/index.ts    | 78 ++++++++++++-----
 packages/types/src/documents/app/view.ts      |  2 +-
 packages/types/src/sdk/row.ts                 |  2 +-
 7 files changed, 163 insertions(+), 43 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index f62bb0f7e3d..7eeeb5cca06 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1091,7 +1091,14 @@ class InternalBuilder {
             )
           }
         } else {
-          query = query.count(`* as ${aggregation.name}`)
+          if (this.client === SqlClient.ORACLE) {
+            const field = this.convertClobs(`${tableName}.${aggregation.field}`)
+            query = query.select(
+              this.knex.raw(`COUNT(??) as ??`, [field, aggregation.name])
+            )
+          } else {
+            query = query.count(`${aggregation.field} as ${aggregation.name}`)
+          }
         }
       } else {
         const fieldSchema = this.getFieldSchema(aggregation.field)
diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index f864df9e9ec..753125e0f6d 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -12,6 +12,8 @@ import {
   RelationSchemaField,
   ViewFieldMetadata,
   CalculationType,
+  CountDistinctCalculationFieldMetadata,
+  CountCalculationFieldMetadata,
 } from "@budibase/types"
 import { builderSocket, gridSocket } from "../../../websockets"
 import { helpers } from "@budibase/shared-core"
@@ -22,27 +24,31 @@ function stripUnknownFields(
   if (helpers.views.isCalculationField(field)) {
     if (field.calculationType === CalculationType.COUNT) {
       if ("distinct" in field && field.distinct) {
-        return {
-          order: field.order,
-          width: field.width,
-          visible: field.visible,
-          readonly: field.readonly,
-          icon: field.icon,
-          distinct: field.distinct,
-          calculationType: field.calculationType,
-          field: field.field,
-          columns: field.columns,
-        }
+        const strippedField: RequiredKeys<CountDistinctCalculationFieldMetadata> =
+          {
+            order: field.order,
+            width: field.width,
+            visible: field.visible,
+            readonly: field.readonly,
+            icon: field.icon,
+            distinct: field.distinct,
+            calculationType: field.calculationType,
+            field: field.field,
+            columns: field.columns,
+          }
+        return strippedField
       } else {
-        return {
+        const strippedField: RequiredKeys<CountCalculationFieldMetadata> = {
           order: field.order,
           width: field.width,
           visible: field.visible,
           readonly: field.readonly,
           icon: field.icon,
           calculationType: field.calculationType,
+          field: field.field,
           columns: field.columns,
         }
+        return strippedField
       }
     }
     const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 1f2470f097e..415b22d4072 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -693,6 +693,12 @@ describe.each([
                 calculationType: CalculationType.COUNT,
                 field: "Price",
               },
+              countDistinct: {
+                visible: true,
+                calculationType: CalculationType.COUNT,
+                distinct: true,
+                field: "Price",
+              },
               min: {
                 visible: true,
                 calculationType: CalculationType.MIN,
@@ -708,11 +714,6 @@ describe.each([
                 calculationType: CalculationType.AVG,
                 field: "Price",
               },
-              sum2: {
-                visible: true,
-                calculationType: CalculationType.SUM,
-                field: "Price",
-              },
             },
           },
           {
@@ -763,10 +764,12 @@ describe.each([
               count: {
                 visible: true,
                 calculationType: CalculationType.COUNT,
+                field: "Price",
               },
               count2: {
                 visible: true,
                 calculationType: CalculationType.COUNT,
+                field: "Price",
               },
             },
           },
@@ -774,7 +777,7 @@ describe.each([
             status: 400,
             body: {
               message:
-                'Duplicate calculation on field "*", calculation type "count"',
+                'Duplicate calculation on field "Price", calculation type "count"',
             },
           }
         )
@@ -805,7 +808,7 @@ describe.each([
             status: 400,
             body: {
               message:
-                'Duplicate calculation on field "Price", calculation type "count"',
+                'Duplicate calculation on field "Price", calculation type "count distinct"',
             },
           }
         )
@@ -820,6 +823,7 @@ describe.each([
             count: {
               visible: true,
               calculationType: CalculationType.COUNT,
+              field: "Price",
             },
             count2: {
               visible: true,
@@ -831,6 +835,26 @@ describe.each([
         })
       })
 
+      it("does not confuse counts on different fields in the duplicate check", async () => {
+        await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            count: {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+              field: "Price",
+            },
+            count2: {
+              visible: true,
+              calculationType: CalculationType.COUNT,
+              field: "Category",
+            },
+          },
+        })
+      })
+
       !isLucene &&
         it("does not get confused when a calculation field shadows a basic one", async () => {
           const table = await config.api.table.save(
@@ -1607,6 +1631,7 @@ describe.each([
             view.schema!.count = {
               visible: true,
               calculationType: CalculationType.COUNT,
+              field: "age",
             }
             await config.api.viewV2.update(view)
 
@@ -3761,6 +3786,51 @@ describe.each([
               expect(rows[0].sum).toEqual(55)
             })
 
+            it("should be able to count non-numeric fields", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    firstName: {
+                      type: FieldType.STRING,
+                      name: "firstName",
+                    },
+                    lastName: {
+                      type: FieldType.STRING,
+                      name: "lastName",
+                    },
+                  },
+                })
+              )
+
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  count: {
+                    visible: true,
+                    calculationType: CalculationType.COUNT,
+                    field: "firstName",
+                  },
+                },
+              })
+
+              await config.api.row.bulkImport(table._id!, {
+                rows: [
+                  { firstName: "Jane", lastName: "Smith" },
+                  { firstName: "Jane", lastName: "Doe" },
+                  { firstName: "Alice", lastName: "Smith" },
+                ],
+              })
+
+              const { rows } = await config.api.viewV2.search(view.id, {
+                query: {},
+              })
+
+              expect(rows).toHaveLength(1)
+              expect(rows[0].count).toEqual(3)
+            })
+
             it("should be able to filter rows on the view itself", async () => {
               const table = await config.api.table.save(
                 saveTableRequest({
diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index e3bbca271e8..a160bfbb3e1 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -351,6 +351,7 @@ export async function search(
           aggregations.push({
             name: key,
             calculationType: field.calculationType,
+            field: mapToUserColumn(field.field),
           })
         }
       } else {
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index cbf3513dd09..a259dd5505f 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -61,42 +61,77 @@ export function isView(view: any): view is ViewV2 {
   return view.id && docIds.isViewId(view.id) && view.version === 2
 }
 
-async function guardCalculationViewSchema(
-  table: Table,
-  view: Omit<ViewV2, "id" | "version">
-) {
+function guardDuplicateCalculationFields(view: Omit<ViewV2, "id" | "version">) {
+  const seen: Record<string, Record<CalculationType, boolean>> = {}
   const calculationFields = helpers.views.calculationFields(view)
+  for (const name of Object.keys(calculationFields)) {
+    const schema = calculationFields[name]
+    const isCount = schema.calculationType === CalculationType.COUNT
+    const isDistinct = "distinct" in schema
 
-  if (Object.keys(calculationFields).length > 5) {
-    throw new HTTPError(
-      "Calculation views can only have a maximum of 5 fields",
-      400
-    )
+    if (isCount && isDistinct) {
+      // We do these separately.
+      continue
+    }
+
+    if (seen[schema.field]?.[schema.calculationType]) {
+      throw new HTTPError(
+        `Duplicate calculation on field "${schema.field}", calculation type "${schema.calculationType}"`,
+        400
+      )
+    }
+    seen[schema.field] ??= {} as Record<CalculationType, boolean>
+    seen[schema.field][schema.calculationType] = true
   }
+}
 
+function guardDuplicateCountDistinctFields(
+  view: Omit<ViewV2, "id" | "version">
+) {
   const seen: Record<string, Record<CalculationType, boolean>> = {}
-
+  const calculationFields = helpers.views.calculationFields(view)
   for (const name of Object.keys(calculationFields)) {
     const schema = calculationFields[name]
     const isCount = schema.calculationType === CalculationType.COUNT
-    const isDistinct = isCount && "distinct" in schema && schema.distinct
+    if (!isCount) {
+      continue
+    }
 
-    const field = isCount && !isDistinct ? "*" : schema.field
-    if (seen[field]?.[schema.calculationType]) {
+    const isDistinct = "distinct" in schema
+    if (!isDistinct) {
+      // We do these separately.
+      continue
+    }
+
+    if (seen[schema.field]?.[schema.calculationType]) {
       throw new HTTPError(
-        `Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`,
+        `Duplicate calculation on field "${schema.field}", calculation type "${schema.calculationType} distinct"`,
         400
       )
     }
-    seen[field] ??= {} as Record<CalculationType, boolean>
-    seen[field][schema.calculationType] = true
+    seen[schema.field] ??= {} as Record<CalculationType, boolean>
+    seen[schema.field][schema.calculationType] = true
+  }
+}
 
-    // Count fields that aren't distinct don't need to reference another field,
-    // so we don't validate it.
-    if (isCount && !isDistinct) {
-      continue
-    }
+async function guardCalculationViewSchema(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const calculationFields = helpers.views.calculationFields(view)
+
+  guardDuplicateCalculationFields(view)
+  guardDuplicateCountDistinctFields(view)
+
+  if (Object.keys(calculationFields).length > 5) {
+    throw new HTTPError(
+      "Calculation views can only have a maximum of 5 fields",
+      400
+    )
+  }
 
+  for (const name of Object.keys(calculationFields)) {
+    const schema = calculationFields[name]
     if (!schema.field) {
       throw new HTTPError(
         `Calculation field "${name}" is missing a "field" property`,
@@ -112,6 +147,7 @@ async function guardCalculationViewSchema(
       )
     }
 
+    const isCount = schema.calculationType === CalculationType.COUNT
     if (!isCount && !isNumeric(targetSchema.type)) {
       throw new HTTPError(
         `Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts
index 3ae120fccac..1212031f24e 100644
--- a/packages/types/src/documents/app/view.ts
+++ b/packages/types/src/documents/app/view.ts
@@ -54,12 +54,12 @@ export interface NumericCalculationFieldMetadata
 
 export interface CountCalculationFieldMetadata extends BasicViewFieldMetadata {
   calculationType: CalculationType.COUNT
+  field: string
 }
 
 export interface CountDistinctCalculationFieldMetadata
   extends CountCalculationFieldMetadata {
   distinct: true
-  field: string
 }
 
 export type ViewCalculationFieldMetadata =
diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts
index ea46e825082..6328a80ba16 100644
--- a/packages/types/src/sdk/row.ts
+++ b/packages/types/src/sdk/row.ts
@@ -18,11 +18,11 @@ export interface NumericAggregation extends BaseAggregation {
 
 export interface CountAggregation extends BaseAggregation {
   calculationType: CalculationType.COUNT
+  field: string
 }
 
 export interface CountDistinctAggregation extends CountAggregation {
   distinct: true
-  field: string
 }
 
 export type Aggregation =

From 0b8199be746240670bafd3cfff247304a8498054 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 31 Oct 2024 12:19:57 +0000
Subject: [PATCH 448/455] Use existing _id values for non DS+ in tables,
 trusting that they are unique

---
 .../src/components/grid/lib/constants.js      |  2 ++
 .../src/components/grid/lib/utils.js          | 19 +++++++++++++------
 .../src/components/grid/stores/rows.js        | 17 +++++++++++------
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/lib/constants.js b/packages/frontend-core/src/components/grid/lib/constants.js
index 6ea7a981786..81b4a2f823d 100644
--- a/packages/frontend-core/src/components/grid/lib/constants.js
+++ b/packages/frontend-core/src/components/grid/lib/constants.js
@@ -10,6 +10,8 @@ export const DefaultColumnWidth = 200
 export const MinColumnWidth = 80
 export const NewRowID = "new"
 export const BlankRowID = "blank"
+export const GeneratedIDPrefix = "‽‽"
+export const CellIDSeparator = "‽‽"
 export const RowPageSize = 100
 export const FocusedCellMinOffset = ScrollBarSize * 3
 export const ControlsHeight = 50
diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js
index b5485abc36f..f1f33d9950e 100644
--- a/packages/frontend-core/src/components/grid/lib/utils.js
+++ b/packages/frontend-core/src/components/grid/lib/utils.js
@@ -1,18 +1,17 @@
-// We can't use "-" as a separator as this can be present in the ID
-// or column name, so we use something very unusual to avoid this problem
-const JOINING_CHARACTER = "‽‽"
+import { GeneratedIDPrefix, CellIDSeparator } from "./constants"
+import { Helpers } from "@budibase/bbui"
 
 export const parseCellID = cellId => {
   if (!cellId) {
     return { rowId: undefined, field: undefined }
   }
-  const parts = cellId.split(JOINING_CHARACTER)
+  const parts = cellId.split(CellIDSeparator)
   const field = parts.pop()
-  return { rowId: parts.join(JOINING_CHARACTER), field }
+  return { rowId: parts.join(CellIDSeparator), field }
 }
 
 export const getCellID = (rowId, fieldName) => {
-  return `${rowId}${JOINING_CHARACTER}${fieldName}`
+  return `${rowId}${CellIDSeparator}${fieldName}`
 }
 
 export const parseEventLocation = e => {
@@ -21,3 +20,11 @@ export const parseEventLocation = e => {
     y: e.clientY ?? e.touches?.[0]?.clientY,
   }
 }
+
+export const generateRowID = () => {
+  return `${GeneratedIDPrefix}${Helpers.uuid()}`
+}
+
+export const isGeneratedRowID = id => {
+  return id?.startsWith(GeneratedIDPrefix)
+}
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 4cfaa1cfd59..122bbc295a9 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -1,7 +1,12 @@
 import { writable, derived, get } from "svelte/store"
 import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
-import { getCellID, parseCellID } from "../lib/utils"
+import {
+  generateRowID,
+  getCellID,
+  isGeneratedRowID,
+  parseCellID,
+} from "../lib/utils"
 import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
@@ -634,10 +639,10 @@ export const createActions = context => {
       newRow = newRows[i]
 
       // Ensure we have a unique _id.
-      // This means generating one for non DS+, overwriting any that may already
-      // exist as we cannot allow duplicates.
-      if (!$hasBudibaseIdentifiers) {
-        newRow._id = Helpers.uuid()
+      // We generate one for non DS+ where required, but trust that any existing
+      // _id values are unique (e.g. Mongo)
+      if (!$hasBudibaseIdentifiers && !newRow._id?.length) {
+        newRow._id = generateRowID()
       }
 
       if (!rowCacheMap[newRow._id]) {
@@ -674,7 +679,7 @@ export const createActions = context => {
     let clone = { ...row }
     delete clone.__idx
     delete clone.__metadata
-    if (!get(hasBudibaseIdentifiers)) {
+    if (!get(hasBudibaseIdentifiers) && isGeneratedRowID(clone._id)) {
       delete clone._id
     }
     return clone

From afbec934f31e132f637aa944f04599da2c06395e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 31 Oct 2024 12:28:52 +0000
Subject: [PATCH 449/455] Clean table data before using it as additional data
 context to better reflect usable structure

---
 packages/client/src/components/app/GridBlock.svelte | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index c6b08e3869a..3d9dcc028ed 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -60,8 +60,10 @@
 
   // Provide additional data context for live binding eval
   export const getAdditionalDataContext = () => {
-    const rows = get(grid?.getContext()?.rows)
-    const goldenRow = generateGoldenSample(rows)
+    const gridContext = grid?.getContext()
+    const rows = get(gridContext?.rows) || []
+    const cleaned = rows.map(gridContext?.rows.actions.cleanRow || (x => x))
+    const goldenRow = generateGoldenSample(cleaned)
     const id = get(component).id
     return {
       // Not sure what this one is for...

From b9ba76f639aefaed40c829910bbf3aa8523737db Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 31 Oct 2024 12:33:07 +0000
Subject: [PATCH 450/455] Avoid creating new function every iteration of
 array.map

---
 packages/client/src/components/app/GridBlock.svelte | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 3d9dcc028ed..4a8dfb4fcbe 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -62,7 +62,8 @@
   export const getAdditionalDataContext = () => {
     const gridContext = grid?.getContext()
     const rows = get(gridContext?.rows) || []
-    const cleaned = rows.map(gridContext?.rows.actions.cleanRow || (x => x))
+    const clean = gridContext?.rows.actions.cleanRow || (x => x)
+    const cleaned = rows.map(clean)
     const goldenRow = generateGoldenSample(cleaned)
     const id = get(component).id
     return {

From 0bc664775116e1c74803a78e96fcf1f6fd12208a Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 31 Oct 2024 14:25:37 +0000
Subject: [PATCH 451/455] fix AI tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 396173dfeef..cf3c87c4306 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2462,6 +2462,7 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
+          [FieldType.AI]: "LLM Output",
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2493,6 +2494,7 @@ describe.each([
           }),
           [FieldType.FORMULA]: fullSchema[FieldType.FORMULA].formula,
           [FieldType.AUTO]: expect.any(Number),
+          [FieldType.AI]: expect.any(String),
           [FieldType.JSON]: rowValues[FieldType.JSON],
           [FieldType.INTERNAL]: rowValues[FieldType.INTERNAL],
           [FieldType.BARCODEQR]: rowValues[FieldType.BARCODEQR],
@@ -2565,7 +2567,7 @@ describe.each([
               expectedRowData["bb_reference_single"].sample,
               false
             ),
-            ai: null,
+            ai: "LLM Output",
           },
         ])
       })
@@ -2624,7 +2626,6 @@ describe.each([
           tableId: tableId,
           createdAt: new Date().toISOString(),
           updatedAt: new Date().toISOString(),
-          ai: null,
         }
         expect(allRows).toEqual([expectedRow, expectedRow, expectedRow])
       })

From c85dd1f6f89070c338f3e8487ee61bbd5f42ba0d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 31 Oct 2024 14:30:54 +0000
Subject: [PATCH 452/455] Allow counting on non-numeric fields

---
 .../buttons/grid/GridViewCalculationButton.svelte   | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index f6ef6e60990..851dfb6777e 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -90,8 +90,15 @@
   const getFieldOptions = (self, calculations, schema) => {
     return Object.entries(schema)
       .filter(([field, fieldSchema]) => {
-        // Only allow numeric fields that are not calculations themselves
-        if (fieldSchema.calculationType || !isNumeric(fieldSchema.type)) {
+        // Don't allow other calculation columns
+        if (fieldSchema.calculationType) {
+          return false
+        }
+        // Only allow numeric columns for most calculation types
+        if (
+          self.type !== CalculationType.COUNT &&
+          !isNumeric(fieldSchema.type)
+        ) {
           return false
         }
         // Don't allow duplicates
@@ -257,6 +264,4 @@
   .group-by {
     grid-column: 2 / 5;
   }
-  span {
-  }
 </style>

From 986cef6b25ebdcbb7260bd69004983908a8ad24d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 31 Oct 2024 14:35:00 +0000
Subject: [PATCH 453/455] Update view calculation explanation text

---
 .../DataTable/buttons/grid/GridViewCalculationButton.svelte     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
index 851dfb6777e..7b279d3948b 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte
@@ -241,7 +241,7 @@
   <InfoDisplay
     icon="Help"
     quiet
-    body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
+    body="Most calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
   />
 
   <div>

From 78554bdc7c1926306428f7d913787366f4929f1c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 31 Oct 2024 17:16:06 +0000
Subject: [PATCH 454/455] Fix flaky export test.

---
 packages/server/package.json                  |  1 +
 .../src/api/controllers/view/exporters.ts     | 30 +++++++++-------
 .../server/src/api/routes/tests/row.spec.ts   | 34 +++++++++++++++++++
 packages/server/src/utilities/csv.ts          |  6 ++--
 4 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 76dd03b5a85..da770a3ed99 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -75,6 +75,7 @@
     "content-disposition": "^0.5.4",
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
+    "csv": "6.3.10",
     "curlconverter": "3.21.0",
     "dd-trace": "5.2.0",
     "dotenv": "8.2.0",
diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts
index 3269133d4bb..87e4bfc056c 100644
--- a/packages/server/src/api/controllers/view/exporters.ts
+++ b/packages/server/src/api/controllers/view/exporters.ts
@@ -9,29 +9,33 @@ function getHeaders(
   return headers.map(header => `"${customHeaders[header] || header}"`)
 }
 
+function escapeCsvString(str: string) {
+  return str.replace(/"/g, '""')
+}
+
 export function csv(
   headers: string[],
   rows: Row[],
   delimiter: string = ",",
   customHeaders: { [key: string]: string } = {}
 ) {
-  let csv = getHeaders(headers, customHeaders).join(delimiter)
+  let csvRows = [getHeaders(headers, customHeaders)]
 
   for (let row of rows) {
-    csv = `${csv}\n${headers
-      .map(header => {
-        let val = row[header]
-        val =
-          typeof val === "object" && !(val instanceof Date)
-            ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
-            : val !== undefined
-            ? `"${val}"`
-            : ""
-        return val.trim()
+    csvRows.push(
+      headers.map(header => {
+        const val = row[header]
+        if (typeof val === "object" && !(val instanceof Date)) {
+          return `"${JSON.stringify(val).replace(/"/g, "'")}"`
+        }
+        if (val !== undefined) {
+          return `"${escapeCsvString(val.toString())}"`
+        }
+        return ""
       })
-      .join(delimiter)}`
+    )
   }
-  return csv
+  return csvRows.map(row => row.join(delimiter)).join("\n")
 }
 
 export function json(rows: Row[]) {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index cf3c87c4306..252731bc16f 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2597,6 +2597,40 @@ describe.each([
         })
       })
 
+      it("can handle csv-special characters in strings", async () => {
+        const badString = 'test":, wow", "test": "wow"'
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              string: {
+                type: FieldType.STRING,
+                name: "string",
+              },
+            },
+          })
+        )
+
+        await config.api.row.save(table._id!, { string: badString })
+
+        const exportedValue = await config.api.row.exportRows(
+          table._id!,
+          { query: {} },
+          RowExportFormat.CSV
+        )
+
+        const json = await config.api.table.csvToJson(
+          {
+            csvString: exportedValue,
+          },
+          {
+            status: 200,
+          }
+        )
+
+        expect(json).toHaveLength(1)
+        expect(json[0].string).toEqual(badString)
+      })
+
       it("exported data can be re-imported", async () => {
         // export all
         const exportedValue = await config.api.row.exportRows(
diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts
index 2fab1d11a4e..43d712165ac 100644
--- a/packages/server/src/utilities/csv.ts
+++ b/packages/server/src/utilities/csv.ts
@@ -5,8 +5,10 @@ export async function jsonFromCsvString(csvString: string) {
     csvString
   )
 
-  // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
-  // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
+  // By default the csvtojson library casts empty values as empty strings. This
+  // is causing issues on conversion.  ignoreEmpty will remove the key completly
+  // if empty, so creating this empty object will ensure we return the values
+  // with the keys but empty values
   const result = await csv({ ignoreEmpty: false }).fromString(csvString)
   result.forEach((r, i) => {
     for (const [key] of Object.entries(r).filter(([, value]) => value === "")) {

From b4e758d62971b29c8d4de58273c73561e4e53ae3 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 31 Oct 2024 17:23:51 +0000
Subject: [PATCH 455/455] No longer using the csv package.

---
 packages/server/package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index da770a3ed99..76dd03b5a85 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -75,7 +75,6 @@
     "content-disposition": "^0.5.4",
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
-    "csv": "6.3.10",
     "curlconverter": "3.21.0",
     "dd-trace": "5.2.0",
     "dotenv": "8.2.0",