Sam/oriole17

.github/workflows/ami-release-nix.yml

@@ -3,8 +3,9 @@ name: Release AMI Nix
 on:
   push:
     branches:
-      - develop
-      - release/*
+      #- develop
+      #- release/*
+      - sam/oriole17
     paths:
       - '.github/workflows/ami-release-nix.yml'
       - 'common-nix.vars.pkr.hcl'

.github/workflows/nix-build.yml

@@ -40,6 +40,7 @@ jobs:
           role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
           aws-region: "us-east-1"
           output-credentials: true
+          role-duration-seconds: 7200
       - name: write secret key
         # use python so we don't interpolate the secret into the workflow logs, in case of bugs
         run: |

.github/workflows/test.yml

@@ -4,7 +4,8 @@ on:
   push:
     branches:
       - develop
-  pull_request:
+  #TODO PR uncomment this line
+  #pull_request:
   workflow_dispatch:
 
 jobs:

ansible/tasks/setup-postgres.yml

@@ -204,23 +204,49 @@
     ansible_command_timeout: 60
   when: debpkg_mode
 
-- name: Initialize the database stage2_nix
-  become: yes
-  become_user: postgres
-  shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin"
-  args:
-    executable: /bin/bash
-  environment:
-    LANG: en_US.UTF-8
-    LANGUAGE: en_US.UTF-8
-    LC_ALL: en_US.UTF-8
-    LC_CTYPE: en_US.UTF-8
-    LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
-  vars:
-    ansible_command_timeout: 60
-    # Circumvents the following error:
-    # "Timeout (12s) waiting for privilege escalation prompt"
-  when: stage2_nix 
+- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary
+  block:
+    - name: Check if psql_version is psql_orioledb
+      set_fact:
+        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
+
+    - name: Initialize the database stage2_nix (non-orioledb)
+      become: yes
+      become_user: postgres
+      shell: source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data -o "--allow-group-access" -o "--username=supabase_admin"
+      args:
+        executable: /bin/bash
+      environment:
+        LANG: en_US.UTF-8
+        LANGUAGE: en_US.UTF-8
+        LC_ALL: en_US.UTF-8
+        LC_CTYPE: en_US.UTF-8
+        LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
+      vars:
+        ansible_command_timeout: 60
+      when: stage2_nix and not is_psql_oriole
+
+    - name: Initialize the database stage2_nix (orioledb)
+      become: yes
+      become_user: postgres
+      shell: >
+        source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data 
+        --allow-group-access 
+        --username=supabase_admin 
+        --locale-provider=icu 
+        --encoding=UTF-8 
+        --icu-locale=en_US.UTF-8 
+      args:
+        executable: /bin/bash
+      environment:
+        LANG: en_US.UTF-8
+        LANGUAGE: en_US.UTF-8
+        LC_ALL: en_US.UTF-8
+        LC_CTYPE: en_US.UTF-8
+        LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
+      vars:
+        ansible_command_timeout: 60
+      when: stage2_nix and is_psql_oriole
 
 - name: copy PG systemd unit
   template:

ansible/tasks/stage2-setup-postgres.yml

@@ -4,6 +4,65 @@
 #     sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11"
 # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task
 
+- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary
+  block:
+    - name: Check if psql_version is psql_orioledb-16
+      set_fact:
+        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
+
+    - name: Remove specified extensions from postgresql.conf if oriole-16 build
+      ansible.builtin.command:
+        cmd: >
+          sed -i 's/ timescaledb,//g' 
+          /etc/postgresql/postgresql.conf
+      when: is_psql_oriole and stage2_nix
+      become: yes
+
+    - name: Remove specified extensions from supautils.conf if oriole-16 build
+      ansible.builtin.command:
+        cmd: >
+          sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' 
+          /etc/postgresql-custom/supautils.conf
+      when: is_psql_oriole and stage2_nix
+      become: yes
+
+    - name: Remove db_user_namespace from postgresql.conf if oriole-xx build
+      ansible.builtin.command:
+        cmd: >
+          sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' 
+          /etc/postgresql/postgresql.conf
+      when: is_psql_oriole and stage2_nix
+      become: yes
+
+    - name: Append orioledb to shared_preload_libraries append within closing quote
+      ansible.builtin.command:
+        cmd: >
+          sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/'
+          /etc/postgresql/postgresql.conf
+      when: is_psql_oriole and stage2_nix
+      become: yes
+
+    - name: Add default_table_access_method setting
+      ansible.builtin.lineinfile:
+        path: /etc/postgresql/postgresql.conf
+        line: "default_table_access_method = 'orioledb'"
+        state: present
+      when: is_psql_oriole and stage2_nix
+      become: yes
+
+    # TODO PR may need to remove this.
+    # - name: Add ICU locale settings
+    #   ansible.builtin.lineinfile:
+    #     path: /etc/postgresql/postgresql.conf
+    #     line: "{{ item }}"
+    #     state: present
+    #   with_items:
+    #     - "lc_collate = 'en-US-x-icu'"
+    #     - "lc_ctype = 'en-US-x-icu'"
+    #     - "icu_locale = 'en-US'"
+    #   when: is_psql_oriole and stage2_nix
+    #   become: yes
+
 - name: Install Git for Nix package management
   become: yes
   apt:
@@ -16,7 +75,6 @@
   become: yes
   shell: |
     sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin"
-#TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged
   when: stage2_nix
 
 - name: Install pg_prove from nix binary cache
@@ -43,14 +101,15 @@
     sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src"
   when: stage2_nix
 
-- name: Install Git for Nix package management
+- name: Remove Git after Nix package installations
   become: yes
   apt:
     name: git
-    state: present
-    update_cache: yes
+    state: absent
+    autoremove: yes  
+    purge: yes       
   when: stage2_nix
-  
+
 - name: Set ownership and permissions for /etc/ssl/private
   become: yes
   file:
@@ -215,13 +274,19 @@
     recurse: yes
   when: stage2_nix
 
-- name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir
-  shell: >
-    sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \
-    sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \
-    && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/"
-  become: yes
-  when: stage2_nix
+- name: Check psql_version and run postgis linking if not oriole-16
+  block:
+    - name: Check if psql_version is psql_orioledb-17
+      set_fact:
+        is_psql_oriole: "{{ psql_version == 'psql_orioledb-17' }}"
+
+    - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir
+      shell: >
+        sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \
+        sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \
+        && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/"
+      become: yes
+      when: stage2_nix and not is_psql_oriole
 
 - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets
   file:
@@ -256,8 +321,17 @@
     line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh'
   when: stage2_nix
 
+#TODO PR may need to remove this.
+- name: Create symbolic link for pgsodium_getkey script
+  file:
+    src: "/usr/lib/postgresql/bin/pgsodium_getkey.sh"
+    dest: "/usr/lib/postgresql/share/postgresql/extension/pgsodium_getkey"
+    state: link
+  become: yes
+  when: stage2_nix
+
 - name: Append GRN_PLUGINS_DIR to /etc/environment.d/postgresql.env
   ansible.builtin.lineinfile:
     path: /etc/environment.d/postgresql.env
     line: 'GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins'
-  become: yes
+  become: yes

ansible/tasks/test-image.yml

@@ -4,11 +4,22 @@
       - libtap-parser-sourcehandler-pgtap-perl
   when: debpkg_mode
 
+# - name: Temporarily disable PG Sodium references in config
+#   become: yes
+#   become_user: postgres
+#   shell:
+#     cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf
+#   when: debpkg_mode or stage2_nix
+
 - name: Temporarily disable PG Sodium references in config
   become: yes
   become_user: postgres
   shell:
-    cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf
+    cmd: >
+      sed -i.bak
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
+      /etc/postgresql/postgresql.conf
   when: debpkg_mode or stage2_nix
 
 - name: Start Postgres Database to load all extensions.
@@ -46,6 +57,64 @@
     LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
   when: stage2_nix
 
+
+- name: Check psql_version and modify migrations if oriole-xx
+  block:
+    - name: Check if psql_version is psql_orioledb-xx
+      set_fact:
+        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
+
+    - name: Remove specified extensions from SQL file
+      ansible.builtin.command:
+        cmd: >
+          sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql
+      when: is_psql_oriole
+      become: yes
+
+    - name: Remove specified extension files from extensions directory
+      ansible.builtin.find:
+        paths: /tmp/migrations/tests/extensions
+        patterns: 
+          - '*timescaledb*.sql'
+          - '*postgis*.sql'
+          - '*pgrouting*.sql'
+          - '*plv8*.sql'
+          - '*pg_jsonschema*.sql'
+          - '*pg_graphql*.sql'
+      register: files_to_remove
+      when: is_psql_oriole
+
+    - name: Delete matched extension files
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: absent
+      loop: "{{ files_to_remove.files }}"
+      when: is_psql_oriole
+      become: yes
+
+    - name: Remove specified extensions from SQL file
+      ansible.builtin.command:
+        cmd: >
+          sed -i "/'pg_graphql',/d" /tmp/unit-tests/unit-test-01.sql
+      when: is_psql_oriole
+      become: yes
+
+    - name: Remove graphql schema test
+      lineinfile:
+        path: /tmp/migrations/tests/database/exists.sql
+        regexp: "^SELECT has_schema\\('graphql'\\);$"
+        state: absent
+      become: yes
+      when: is_psql_oriole
+
+    - name: Remove graphql schema test
+      lineinfile:
+        path: /tmp/migrations/tests/database/exists.sql
+        regexp: "^SELECT has_schema\\('graphql_public'\\);$"
+        state: absent
+      become: yes
+      when: is_psql_oriole
+
 - name: Run Unit tests (with filename unit-test-*) on Postgres Database
   shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql
   register: retval

ansible/vars.yml

@@ -2,17 +2,18 @@ supabase_internal: true
 ebssurrogate_mode: true
 async_mode: true
 
-# postgresql_major: "15"
-# postgresql_release: "15.1"
-# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c
 postgres_major:
-  - "15"
-  - "16"
+  #- "15"
+  #- "16"
+  - "orioledb-17"
 
 # Full version strings for each major version
+# TOD PR uncomment these lines
 postgres_release:
-  postgres15: "15.8.1.004"
-  postgres16: "16.3.1.010"
+  #postgres15: "15.8.1.003-staging-5"
+  #postgres16: "16.3.1.000-staging-5"
+  #postgresorioledb-16: "orioledb-16.3.1.000-staging-12"
+  postgresorioledb-17: "orioledb-17.0.1.000-staging-3"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

docker/nix/build_nix.sh

@@ -7,7 +7,11 @@ if [ -d "/workspace" ]; then
 fi
 nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link
 nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link
+#no nix flake check on oriole yet
 nix build .#psql_15/bin -o psql_15
 nix build .#psql_16/bin -o psql_16
+#nix build .#psql_orioledb-16/bin -o psql_orioledb_16
+nix build .#psql_orioledb-17/bin -o psql_orioledb_17
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17

ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -214,7 +214,10 @@ EOF
 	# Run Ansible playbook
 	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp 
 	export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
-	ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' $ARGS
+	ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \
+	--extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false} ' \
+	--extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \
+	$ARGS
 }
 
 function update_systemd_services {