From 3a9302425729835aa0148403c0de1d4c723e20cc Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Wed, 28 Jun 2023 23:08:40 +0200
Subject: [PATCH 01/10] test: Add Python script to test UIs

---
 scripts/release.sh  |  27 +----
 scripts/test-uis.py | 256 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 261 insertions(+), 22 deletions(-)
 create mode 100644 scripts/test-uis.py

diff --git a/scripts/release.sh b/scripts/release.sh
index 29c51d504d..7e05fc14f4 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -115,6 +115,8 @@ validate_default_project() {
         export hello_world_frontend_url="http://localhost:$webserver_port/?canisterId=$hello_world_frontend_canister_id"
         export candid_ui_url="http://localhost:$webserver_port/?canisterId=$candid_ui_id&id=$application_canister_id"
 
+        pip install playwright==1.35.0
+
         echo
         echo "=================================================="
         echo "dfx project directory: $(pwd)"
@@ -122,30 +124,11 @@ validate_default_project() {
         echo "candid URL: $candid_ui_url"
         echo "=================================================="
         echo
-        echo "[1/4] Verify 'hello' functionality in a browser."
-        echo "  - Open this URL in your web browser with empty cache or 'Private Browsing' mode"
-        echo "  - Type a name and verify the response."
-        echo
-        echo "  $hello_world_frontend_url"
-        echo
-        wait_for_response 'frontend UI passes'
-        echo
-        echo "[2/4] Verify there are no errors in the console by opening the Developer Tools."
-        echo
-        wait_for_response 'no errors on console'
-        echo
-        echo "[3/4] Verify the Candid UI."
-        echo
-        echo "  - Open this URL in your web browser with empty cache or 'Private Browsing' mode"
-        echo "  - Verify UI loads, then test the greet function by entering text and clicking *Call* or clicking *Lucky*"
-        echo
-        echo "  $candid_ui_url"
-        echo
-        wait_for_response 'candid UI passes'
+        echo "Verify the Python script output."
         echo
-        echo "[4/4] Verify there are no errors in the console by opening the Developer Tools."
+        python3 scripts/test-uis.py --frontend_url "$hello_world_frontend_url" --candid_url "$candid_ui_url" --browsers chromium firefox webkit
         echo
-        wait_for_response 'no errors on console'
+        wait_for_response 'Python script logs are ok'
         echo
 
         dfx stop
diff --git a/scripts/test-uis.py b/scripts/test-uis.py
new file mode 100644
index 0000000000..220601f12b
--- /dev/null
+++ b/scripts/test-uis.py
@@ -0,0 +1,256 @@
+'''
+Automate frontend tests by using Playwright.
+The script tests the following UIs:
+
+1. Frontend UI.
+2. Candid UI.
+
+Examples:
+
+$ python3 test-uis.py --frontend_url '...' --browser chromium firefox webkit  # Only test the frontend UI
+$ python3 test-uis.py --candid_url '...' --browser chromium firefox webkit  # Only test the Candid UI
+$ python3 test-uis.py --frontend_url '...' --candid_url '...' --browser chromium firefox webkit  # Test both UIs
+'''
+import argparse
+import logging
+import re
+import sys
+
+from playwright.sync_api import sync_playwright
+
+_CHROMIUM_BROWSER = 'chromium'
+_FIREFOX_BROWSER = 'firefox'
+_WEBKIT_BROWSER = 'webkit'
+_SUPPORTED_BROWSERS = {
+    _CHROMIUM_BROWSER,
+    _FIREFOX_BROWSER,
+    _WEBKIT_BROWSER,
+}
+_CANDID_UI_WARNINGS_TO_IGNORE = [
+    ('Invalid asm.js: Unexpected token', '/index.js'),
+    ('Expected to find result for path [object Object], but instead found nothing.', '/index.js'),
+    ('''
+Error: Server returned an error:
+  Code: 404 (Not Found)
+  Body: Custom section name not found.
+
+    at j.readState (http://localhost:4943/index.js:2:11709)
+    at async http://localhost:4943/index.js:2:97683
+    at async Promise.all (index 0)
+    at async Module.UA (http://localhost:4943/index.js:2:98732)
+    at async Object.getNames (http://localhost:4943/index.js:2:266156)
+    at async http://localhost:4943/index.js:2:275479'''.strip(), '/index.js')
+]
+_CANDID_UI_ERRORS_TO_IGNORE = [
+    ('Failed to load resource: the server responded with a status of 404 (Not Found)', '/read_state'),
+]
+
+
+def _validate_browsers(browser):
+    if browser not in _SUPPORTED_BROWSERS:
+        logging.error(f'Browser {browser} not supported')
+        sys.exit(1)
+
+    return browser
+
+
+def _get_argument_parser():
+    parser = argparse.ArgumentParser(description='Test the Frontend and Candid UIs')
+
+    parser.add_argument('--frontend_url', help='Frontend UI url')
+    parser.add_argument('--candid_url', help='Candid UI url')
+
+    parser.add_argument('--browsers', nargs='+', type=_validate_browsers,
+                        help=f'Test against the specified browsers ({_SUPPORTED_BROWSERS})')
+
+    return parser
+
+
+def _validate_args(args):
+    has_err = False
+
+    if not args.frontend_url and not args.candid_url:
+        logging.error('Either "--frontend_url" or "--candid_url" must be specified to start the tests')
+        has_err = True
+
+    if not args.browsers:
+        logging.error('At least one browser must be specified')
+        logging.error(f'Possible browsers: {_SUPPORTED_BROWSERS}')
+        has_err = True
+
+    if has_err:
+        sys.exit(1)
+
+
+def _get_browser_obj(playwright, browser_name):
+    if browser_name == _CHROMIUM_BROWSER:
+        return playwright.chromium
+    if browser_name == _FIREFOX_BROWSER:
+        return playwright.firefox
+    if browser_name == _WEBKIT_BROWSER:
+        return playwright.webkit
+
+    return None
+
+
+def _check_console_logs(console_logs):
+    logging.info('Checking console logs')
+
+    has_err = False
+    for log in console_logs:
+        if log.type not in {'warning', 'error'}:
+            continue
+
+        # Skip all `Error with Permissions-Policy header: Unrecognized feature` warnings
+        perm_policy_warn = 'Error with Permissions-Policy header:'
+        if perm_policy_warn in log.text:
+            logging.warning(f'Skipping Permissions-Policy warning. log.text="{log.text}"')
+            continue
+
+        url = log.location.get('url')
+        if not url:
+            raise RuntimeError(f'Cannot find "url" during log parsing (log.type={log.type}, log.text="{log.text}", log.location="{log.location}")')
+
+        for actual_text, endpoint in (_CANDID_UI_ERRORS_TO_IGNORE if log.type == 'error' else _CANDID_UI_WARNINGS_TO_IGNORE):
+            if actual_text == log.text and endpoint in url:
+                logging.warning(f'Found {log.type}, but it was expected (log.type="{actual_text}", endpoint="{endpoint}")')
+                break
+        else:
+            logging.error(f'Found unexpected console log {log.type}. Text: "{log.text}"')
+            has_err = True
+
+    if has_err:
+        raise RuntimeError('Console has unexpected warnings and/or errors. Check previous logs')
+
+    logging.info('Console logs are ok')
+
+
+def _click_button(page, button):
+    logging.info(f'Clicking button "{button}"')
+    page.get_by_role('button', name=button).click()
+
+
+def _set_text(page, text, value):
+    logging.info(f'Setting text to "{value}"')
+    page.get_by_placeholder(text).fill(value)
+
+
+def _test_frontend_ui_handler(browser, context, page):
+    # Set the name & Click the button
+    name = 'my name'
+    logging.info(f'Setting name "{name}"')
+    page.get_by_label('Enter your name:').fill(name)
+    _click_button(page, 'Click Me!')
+
+    # Check if `#greeting` is populated correctly
+    greeting_id = '#greeting'
+    greeting_obj = page.query_selector(greeting_id)
+    if greeting_obj:
+        actual_value = greeting_obj.inner_text()
+        expected_value = f'Hello, {name}!'
+        if actual_value == expected_value:
+            logging.info(f'"{actual_value}" found in "{greeting_id}"')
+        else:
+            raise RuntimeError(f'Expected greeting message is "{expected_value}", but found "{actual_value}"')
+    else:
+        raise RuntimeError(f'Cannot find {greeting_id} selector')
+
+
+def _test_candid_ui_handler(browser, context, page):
+    # Set the text & Click the "Query" button
+    text = 'hello, world'
+    _set_text(page, 'text', text)
+    _click_button(page, 'Query')
+
+    # Reset the text & Click the "Random" button
+    _set_text(page, 'text', '')
+    _click_button(page, 'Random')
+    # ~
+
+    # Check if `#output-list` is populated correctly
+    output_list_id = '#output-list'
+    output_list_obj = page.query_selector(output_list_id)
+    if output_list_obj:
+        output_list_lines = output_list_obj.inner_text().split('\n')
+        actual_num_lines, expected_num_lines = len(output_list_lines), 4
+        if actual_num_lines != expected_num_lines:
+            raise RuntimeError(f'Expected {expected_num_lines} lines of text but found {actual_num_lines}')
+
+        # Extract random text from third line
+        random_text = re.search(r'"([^"]*)"', output_list_lines[2])
+        if not random_text:
+            raise RuntimeError(f'Cannot extract the random text from the third line: {output_list_lines[2]}')
+        random_text = random_text.group(1)
+
+        for i, text_str in enumerate([text, random_text]):
+            l1, l2 = (i * 2), (i * 2 + 1)
+
+            # First output line
+            actual_line, expected_line = output_list_lines[l1], f'› greet("{text_str}")'
+            if actual_line != expected_line:
+                raise RuntimeError(f'Expected {expected_line} line, but found {actual_line} (line {l1})')
+            logging.info(f'"{actual_line}" found in {output_list_id} at position {l1}')
+
+            # Second output line
+            actual_line, expected_line = output_list_lines[l2], f'("Hello, {text_str}!")'
+            if actual_line != expected_line:
+                raise RuntimeError(f'Expected {expected_line} line, but found {actual_line} (line {l2})')
+            logging.info(f'"{actual_line}" found in {output_list_id} at position {l2}')
+
+        logging.info(f'{output_list_id} lines are defined correctly')
+    else:
+        raise RuntimeError(f'Cannot find {output_list_id} selector')
+
+
+def _test_ui(url, ui_name, handler, browsers):
+    logging.info(f'Testing "{ui_name}" at "{url}"')
+
+    has_err = False
+    with sync_playwright() as playwright:
+        for browser_name in browsers:
+            logging.info(f'Checking "{browser_name}" browser')
+            browser = _get_browser_obj(playwright, browser_name)
+            if not browser:
+                raise RuntimeError(f'Cannot determine browser object for browser {browser_name}')
+
+            try:
+                browser = playwright.chromium.launch(headless=True)
+                context = browser.new_context()
+                page = context.new_page()
+
+                # Attach a listener to the page's console events
+                console_logs = []
+                page.on('console', lambda msg: console_logs.append(msg))
+
+                page.goto(url)
+
+                handler(browser, context, page)
+                _check_console_logs(console_logs)
+            except Exception as e:
+                logging.error(f'Error: {str(e)}')
+                has_err = True
+            finally:
+                if context:
+                    context.close()
+                if browser:
+                    browser.close()
+
+    if has_err:
+        sys.exit(1)
+
+
+def _main():
+    args = _get_argument_parser().parse_args()
+    _validate_args(args)
+
+    if args.frontend_url:
+        _test_ui(args.frontend_url, 'Frontend UI', _test_frontend_ui_handler, args.browsers)
+    if args.candid_url:
+        _test_ui(args.candid_url, 'Candid UI', _test_candid_ui_handler, args.browsers)
+
+    logging.info('DONE!')
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    _main()

From 7eea0230b2d981681ca0ec138f9b8d8c442c2625 Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Tue, 18 Jul 2023 21:21:59 +0200
Subject: [PATCH 02/10] Run ui script from e2e tests

---
 .github/workflows/e2e.yml | 42 ++++++++++++++++++++++++++++++++++++++-
 scripts/release.sh        | 27 ++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 0d7eacfb55..c231310a1c 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -152,10 +152,50 @@ jobs:
       - name: Run e2e test
         run: timeout 2400 bats "e2e/$E2E_TEST"
 
+  ui_test:
+    runs-on: ${{ matrix.os }}
+    needs: [build_dfx]
+    strategy:
+      matrix:
+        os: [macos-11, ubuntu-20.04]
+        rust: ['1.60.0']
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v2
+      - name: Setting up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - name: Installing playwright
+        run: |
+          pip install playwright==1.35.0
+          playwright install
+      - name: Download dfx binary
+        uses: actions/download-artifact@v2
+        with:
+          name: dfx-${{ matrix.os }}-rs-${{ matrix.rust }}
+          path: /usr/local/bin
+      - name: Setup dfx binary
+        run: chmod +x /usr/local/bin/dfx
+      - name: Deploy default dfx project
+        run: |
+          dfx new e2e_project
+          cd e2e_project
+          dfx start --background --clean
+          dfx deploy 2>&1 | tee deploy.log
+          echo FRONTEND_URL=$(grep "_frontend:" deploy.log | grep -Eo "(http|https)://[a-zA-Z0-9./?=_&%:-]*") >> $GITHUB_ENV
+          echo CANDID_URL=$(grep "_backend:" deploy.log | grep -Eo "(http|https)://[a-zA-Z0-9./?=_&%:-]*") >> $GITHUB_ENV
+      - name: Running the Python script
+        run: |
+          python scripts/test-uis.py \
+            --frontend_url "$FRONTEND_URL" \
+            --candid_url "$CANDID_URL" \
+            --browser chromium firefox webkit
+
   aggregate:
     name: e2e:required
     if: ${{ always() }}
-    needs: [test, smoke]
+    needs: [test, smoke, ui_test]
     runs-on: ubuntu-latest
     steps:
       - name: check smoke test result
diff --git a/scripts/release.sh b/scripts/release.sh
index 7e05fc14f4..29c51d504d 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -115,8 +115,6 @@ validate_default_project() {
         export hello_world_frontend_url="http://localhost:$webserver_port/?canisterId=$hello_world_frontend_canister_id"
         export candid_ui_url="http://localhost:$webserver_port/?canisterId=$candid_ui_id&id=$application_canister_id"
 
-        pip install playwright==1.35.0
-
         echo
         echo "=================================================="
         echo "dfx project directory: $(pwd)"
@@ -124,11 +122,30 @@ validate_default_project() {
         echo "candid URL: $candid_ui_url"
         echo "=================================================="
         echo
-        echo "Verify the Python script output."
+        echo "[1/4] Verify 'hello' functionality in a browser."
+        echo "  - Open this URL in your web browser with empty cache or 'Private Browsing' mode"
+        echo "  - Type a name and verify the response."
+        echo
+        echo "  $hello_world_frontend_url"
+        echo
+        wait_for_response 'frontend UI passes'
+        echo
+        echo "[2/4] Verify there are no errors in the console by opening the Developer Tools."
+        echo
+        wait_for_response 'no errors on console'
+        echo
+        echo "[3/4] Verify the Candid UI."
+        echo
+        echo "  - Open this URL in your web browser with empty cache or 'Private Browsing' mode"
+        echo "  - Verify UI loads, then test the greet function by entering text and clicking *Call* or clicking *Lucky*"
+        echo
+        echo "  $candid_ui_url"
+        echo
+        wait_for_response 'candid UI passes'
         echo
-        python3 scripts/test-uis.py --frontend_url "$hello_world_frontend_url" --candid_url "$candid_ui_url" --browsers chromium firefox webkit
+        echo "[4/4] Verify there are no errors in the console by opening the Developer Tools."
         echo
-        wait_for_response 'Python script logs are ok'
+        wait_for_response 'no errors on console'
         echo
 
         dfx stop

From 6183be5f6d0453ed5b82bf2de191aea35726ac23 Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Wed, 19 Jul 2023 20:48:21 +0200
Subject: [PATCH 03/10] Fix OS, Rust and actions versions

---
 .github/workflows/e2e.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index c231310a1c..79cf48a036 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -157,13 +157,13 @@ jobs:
     needs: [build_dfx]
     strategy:
       matrix:
-        os: [macos-11, ubuntu-20.04]
-        rust: ['1.60.0']
+        os: [macos-12, ubuntu-20.04, ubuntu-22.04]
+        rust: ['1.65.0']
     steps:
       - name: Checking out repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setting up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Installing playwright
@@ -171,7 +171,7 @@ jobs:
           pip install playwright==1.35.0
           playwright install
       - name: Download dfx binary
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: dfx-${{ matrix.os }}-rs-${{ matrix.rust }}
           path: /usr/local/bin

From 251c07dab0dd1fef9724743afda3deaa5e827fea Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Thu, 27 Jul 2023 22:24:08 +0200
Subject: [PATCH 04/10] Fix SSL webkit issue

---
 scripts/test-uis.py | 62 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 50 insertions(+), 12 deletions(-)

diff --git a/scripts/test-uis.py b/scripts/test-uis.py
index 220601f12b..6a6b2e9bc3 100644
--- a/scripts/test-uis.py
+++ b/scripts/test-uis.py
@@ -15,6 +15,7 @@
 import logging
 import re
 import sys
+from enum import Enum
 
 from playwright.sync_api import sync_playwright
 
@@ -27,6 +28,7 @@
     _WEBKIT_BROWSER,
 }
 _CANDID_UI_WARNINGS_TO_IGNORE = [
+    ('Error', '/index.js'),
     ('Invalid asm.js: Unexpected token', '/index.js'),
     ('Expected to find result for path [object Object], but instead found nothing.', '/index.js'),
     ('''
@@ -39,11 +41,22 @@
     at async Promise.all (index 0)
     at async Module.UA (http://localhost:4943/index.js:2:98732)
     at async Object.getNames (http://localhost:4943/index.js:2:266156)
-    at async http://localhost:4943/index.js:2:275479'''.strip(), '/index.js')
+    at async http://localhost:4943/index.js:2:275479'''.strip(), '/index.js'),
+    ('''
+Error: Server returned an error:
+  Code: 404 (Not Found)
+  Body: Custom section name not found.'''.strip(), '/index.js'),
 ]
 _CANDID_UI_ERRORS_TO_IGNORE = [
     ('Failed to load resource: the server responded with a status of 404 (Not Found)', '/read_state'),
 ]
+# `page.route` does not support additional function parameters
+_FRONTEND_URL = None
+
+
+class _UI(Enum):
+    CANDID = 1
+    FRONTEND = 2
 
 
 def _validate_browsers(browser):
@@ -112,11 +125,12 @@ def _check_console_logs(console_logs):
             raise RuntimeError(f'Cannot find "url" during log parsing (log.type={log.type}, log.text="{log.text}", log.location="{log.location}")')
 
         for actual_text, endpoint in (_CANDID_UI_ERRORS_TO_IGNORE if log.type == 'error' else _CANDID_UI_WARNINGS_TO_IGNORE):
-            if actual_text == log.text and endpoint in url:
+            if actual_text == log.text.strip() and endpoint in url:
                 logging.warning(f'Found {log.type}, but it was expected (log.type="{actual_text}", endpoint="{endpoint}")')
                 break
         else:
-            logging.error(f'Found unexpected console log {log.type}. Text: "{log.text}"')
+            logging.error(f'Found unexpected console log {log.type}. Text: "{log.text}", url: {url}')
+
             has_err = True
 
     if has_err:
@@ -132,10 +146,10 @@ def _click_button(page, button):
 
 def _set_text(page, text, value):
     logging.info(f'Setting text to "{value}"')
-    page.get_by_placeholder(text).fill(value)
+    page.get_by_role('textbox', name=text).fill(value)
 
 
-def _test_frontend_ui_handler(browser, context, page):
+def _test_frontend_ui_handler(page):
     # Set the name & Click the button
     name = 'my name'
     logging.info(f'Setting name "{name}"')
@@ -156,7 +170,7 @@ def _test_frontend_ui_handler(browser, context, page):
         raise RuntimeError(f'Cannot find {greeting_id} selector')
 
 
-def _test_candid_ui_handler(browser, context, page):
+def _test_candid_ui_handler(page):
     # Set the text & Click the "Query" button
     text = 'hello, world'
     _set_text(page, 'text', text)
@@ -202,8 +216,24 @@ def _test_candid_ui_handler(browser, context, page):
         raise RuntimeError(f'Cannot find {output_list_id} selector')
 
 
-def _test_ui(url, ui_name, handler, browsers):
-    logging.info(f'Testing "{ui_name}" at "{url}"')
+def _handle_route_for_webkit(route):
+    url = route.request.url.replace('https://', 'http://')
+
+    headers = None
+    if any(map(url.endswith, ['.css', '.js', '.svg'])):
+        global _FRONTEND_URL
+        assert _FRONTEND_URL
+        headers = {
+            'referer': _FRONTEND_URL,
+        }
+
+    response = route.fetch(url=url, headers=headers)
+    assert response.status == 200, f'Expected 200 status code, but got {response.status}. Url: {url}'
+    route.fulfill(response=response)
+
+
+def _test_ui(ui, url, handler, browsers):
+    logging.info(f'Testing "{str(ui)}" at "{url}"')
 
     has_err = False
     with sync_playwright() as playwright:
@@ -214,7 +244,7 @@ def _test_ui(url, ui_name, handler, browsers):
                 raise RuntimeError(f'Cannot determine browser object for browser {browser_name}')
 
             try:
-                browser = playwright.chromium.launch(headless=True)
+                browser = browser.launch(headless=True)
                 context = browser.new_context()
                 page = context.new_page()
 
@@ -222,9 +252,17 @@ def _test_ui(url, ui_name, handler, browsers):
                 console_logs = []
                 page.on('console', lambda msg: console_logs.append(msg))
 
+                # Webkit forces HTTPS:
+                #   - https://github.com/microsoft/playwright/issues/12975
+                #   - https://stackoverflow.com/questions/46394682/safari-keeps-forcing-https-on-localhost
+                if ui == _UI.FRONTEND and browser_name == _WEBKIT_BROWSER:
+                    global _FRONTEND_URL
+                    _FRONTEND_URL = url
+                    page.route('**/*', _handle_route_for_webkit)
+
                 page.goto(url)
 
-                handler(browser, context, page)
+                handler(page)
                 _check_console_logs(console_logs)
             except Exception as e:
                 logging.error(f'Error: {str(e)}')
@@ -244,9 +282,9 @@ def _main():
     _validate_args(args)
 
     if args.frontend_url:
-        _test_ui(args.frontend_url, 'Frontend UI', _test_frontend_ui_handler, args.browsers)
+        _test_ui(_UI.FRONTEND, args.frontend_url, _test_frontend_ui_handler, args.browsers)
     if args.candid_url:
-        _test_ui(args.candid_url, 'Candid UI', _test_candid_ui_handler, args.browsers)
+        _test_ui(_UI.CANDID, args.candid_url, _test_candid_ui_handler, args.browsers)
 
     logging.info('DONE!')
 

From 9291be8f40b44d49a453b2b101e63c235e19ff74 Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Sun, 6 Aug 2023 20:18:17 +0200
Subject: [PATCH 05/10] Install playwright browsers dependencies

---
 .github/workflows/e2e.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 79cf48a036..54ba733311 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -170,6 +170,7 @@ jobs:
         run: |
           pip install playwright==1.35.0
           playwright install
+          playwright install-deps
       - name: Download dfx binary
         uses: actions/download-artifact@v3
         with:

From 7c87190b62e1ad5e286e51ee423a77212238e7bd Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Sun, 20 Aug 2023 18:26:49 +0200
Subject: [PATCH 06/10] Get Rust version from rust-toolchain.toml

---
 .github/workflows/e2e.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 54ba733311..ef6c4bc316 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -158,7 +158,6 @@ jobs:
     strategy:
       matrix:
         os: [macos-12, ubuntu-20.04, ubuntu-22.04]
-        rust: ['1.65.0']
     steps:
       - name: Checking out repo
         uses: actions/checkout@v3
@@ -174,7 +173,7 @@ jobs:
       - name: Download dfx binary
         uses: actions/download-artifact@v3
         with:
-          name: dfx-${{ matrix.os }}-rs-${{ matrix.rust }}
+          name: dfx-${{ matrix.os }}-rs-${{ hashFiles('rust-toolchain.toml') }}
           path: /usr/local/bin
       - name: Setup dfx binary
         run: chmod +x /usr/local/bin/dfx

From 52b7874404de0aa680f1c1da15e90760c891ea36 Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Tue, 29 Aug 2023 23:03:19 +0200
Subject: [PATCH 07/10] Wait for selector to be introduced in the scope

---
 scripts/test-uis.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/test-uis.py b/scripts/test-uis.py
index 6a6b2e9bc3..08c4e25e07 100644
--- a/scripts/test-uis.py
+++ b/scripts/test-uis.py
@@ -158,7 +158,8 @@ def _test_frontend_ui_handler(page):
 
     # Check if `#greeting` is populated correctly
     greeting_id = '#greeting'
-    greeting_obj = page.query_selector(greeting_id)
+    timeout_ms = 60000
+    greeting_obj = page.wait_for_selector(greeting_id, timeout=timeout_ms)
     if greeting_obj:
         actual_value = greeting_obj.inner_text()
         expected_value = f'Hello, {name}!'
@@ -183,7 +184,8 @@ def _test_candid_ui_handler(page):
 
     # Check if `#output-list` is populated correctly
     output_list_id = '#output-list'
-    output_list_obj = page.query_selector(output_list_id)
+    timeout_ms = 60000
+    output_list_obj = page.wait_for_selector(output_list_id, timeout=timeout_ms)
     if output_list_obj:
         output_list_lines = output_list_obj.inner_text().split('\n')
         actual_num_lines, expected_num_lines = len(output_list_lines), 4

From c39a22c0ccdd95c458b52cd7f0590ce199a947b7 Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Tue, 12 Sep 2023 22:08:45 +0200
Subject: [PATCH 08/10] Upgrade playwright to 1.37.0

---
 .github/workflows/e2e.yml |  2 +-
 scripts/test-uis.py       | 14 ++++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index ef6c4bc316..f64456f328 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -167,7 +167,7 @@ jobs:
           python-version: '3.9'
       - name: Installing playwright
         run: |
-          pip install playwright==1.35.0
+          pip install playwright==1.37.0
           playwright install
           playwright install-deps
       - name: Download dfx binary
diff --git a/scripts/test-uis.py b/scripts/test-uis.py
index 08c4e25e07..55597695a1 100644
--- a/scripts/test-uis.py
+++ b/scripts/test-uis.py
@@ -177,20 +177,26 @@ def _test_candid_ui_handler(page):
     _set_text(page, 'text', text)
     _click_button(page, 'Query')
 
+    # Check if `#output-list` is populated correctly (after the first click)
+    output_list_id = '#output-list'
+    timeout_ms = 60000
+    _ = page.wait_for_selector(output_list_id, timeout=timeout_ms)
+
     # Reset the text & Click the "Random" button
     _set_text(page, 'text', '')
     _click_button(page, 'Random')
     # ~
 
-    # Check if `#output-list` is populated correctly
-    output_list_id = '#output-list'
-    timeout_ms = 60000
+    # Check if `#output-list` is populated correctly  (after the second click)
     output_list_obj = page.wait_for_selector(output_list_id, timeout=timeout_ms)
     if output_list_obj:
         output_list_lines = output_list_obj.inner_text().split('\n')
         actual_num_lines, expected_num_lines = len(output_list_lines), 4
         if actual_num_lines != expected_num_lines:
-            raise RuntimeError(f'Expected {expected_num_lines} lines of text but found {actual_num_lines}')
+            err = [f'Expected {expected_num_lines} lines of text but found {actual_num_lines}']
+            err.append('Lines:')
+            err.extend(output_list_lines)
+            raise RuntimeError('\n'.join(err))
 
         # Extract random text from third line
         random_text = re.search(r'"([^"]*)"', output_list_lines[2])

From 01e8d661cd742e5b4297240891843431d73d053a Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Tue, 28 Nov 2023 21:34:03 +0100
Subject: [PATCH 09/10] Add fallback logic for the Candid UI

---
 .github/workflows/e2e.yml |   2 +-
 scripts/test-uis.py       | 217 ++++++++++++++++++++++----------------
 2 files changed, 129 insertions(+), 90 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index f64456f328..32b4623716 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -167,7 +167,7 @@ jobs:
           python-version: '3.9'
       - name: Installing playwright
         run: |
-          pip install playwright==1.37.0
+          pip install playwright==1.40.0
           playwright install
           playwright install-deps
       - name: Download dfx binary
diff --git a/scripts/test-uis.py b/scripts/test-uis.py
index 55597695a1..5752936c29 100644
--- a/scripts/test-uis.py
+++ b/scripts/test-uis.py
@@ -1,4 +1,4 @@
-'''
+"""
 Automate frontend tests by using Playwright.
 The script tests the following UIs:
 
@@ -10,28 +10,30 @@
 $ python3 test-uis.py --frontend_url '...' --browser chromium firefox webkit  # Only test the frontend UI
 $ python3 test-uis.py --candid_url '...' --browser chromium firefox webkit  # Only test the Candid UI
 $ python3 test-uis.py --frontend_url '...' --candid_url '...' --browser chromium firefox webkit  # Test both UIs
-'''
+"""
 import argparse
 import logging
 import re
 import sys
+import time
 from enum import Enum
 
 from playwright.sync_api import sync_playwright
 
-_CHROMIUM_BROWSER = 'chromium'
-_FIREFOX_BROWSER = 'firefox'
-_WEBKIT_BROWSER = 'webkit'
+_CHROMIUM_BROWSER = "chromium"
+_FIREFOX_BROWSER = "firefox"
+_WEBKIT_BROWSER = "webkit"
 _SUPPORTED_BROWSERS = {
     _CHROMIUM_BROWSER,
     _FIREFOX_BROWSER,
     _WEBKIT_BROWSER,
 }
 _CANDID_UI_WARNINGS_TO_IGNORE = [
-    ('Error', '/index.js'),
-    ('Invalid asm.js: Unexpected token', '/index.js'),
-    ('Expected to find result for path [object Object], but instead found nothing.', '/index.js'),
-    ('''
+    ("Error", "/index.js"),
+    ("Invalid asm.js: Unexpected token", "/index.js"),
+    ("Expected to find result for path [object Object], but instead found nothing.", "/index.js"),
+    (
+        """
 Error: Server returned an error:
   Code: 404 (Not Found)
   Body: Custom section name not found.
@@ -41,14 +43,19 @@
     at async Promise.all (index 0)
     at async Module.UA (http://localhost:4943/index.js:2:98732)
     at async Object.getNames (http://localhost:4943/index.js:2:266156)
-    at async http://localhost:4943/index.js:2:275479'''.strip(), '/index.js'),
-    ('''
+    at async http://localhost:4943/index.js:2:275479""".strip(),
+        "/index.js",
+    ),
+    (
+        """
 Error: Server returned an error:
   Code: 404 (Not Found)
-  Body: Custom section name not found.'''.strip(), '/index.js'),
+  Body: Custom section name not found.""".strip(),
+        "/index.js",
+    ),
 ]
 _CANDID_UI_ERRORS_TO_IGNORE = [
-    ('Failed to load resource: the server responded with a status of 404 (Not Found)', '/read_state'),
+    ("Failed to load resource: the server responded with a status of 404 (Not Found)", "/read_state"),
 ]
 # `page.route` does not support additional function parameters
 _FRONTEND_URL = None
@@ -61,20 +68,24 @@ class _UI(Enum):
 
 def _validate_browsers(browser):
     if browser not in _SUPPORTED_BROWSERS:
-        logging.error(f'Browser {browser} not supported')
+        logging.error(f"Browser {browser} not supported")
         sys.exit(1)
 
     return browser
 
 
 def _get_argument_parser():
-    parser = argparse.ArgumentParser(description='Test the Frontend and Candid UIs')
+    parser = argparse.ArgumentParser(description="Test the Frontend and Candid UIs")
 
-    parser.add_argument('--frontend_url', help='Frontend UI url')
-    parser.add_argument('--candid_url', help='Candid UI url')
+    parser.add_argument("--frontend_url", help="Frontend UI url")
+    parser.add_argument("--candid_url", help="Candid UI url")
 
-    parser.add_argument('--browsers', nargs='+', type=_validate_browsers,
-                        help=f'Test against the specified browsers ({_SUPPORTED_BROWSERS})')
+    parser.add_argument(
+        "--browsers",
+        nargs="+",
+        type=_validate_browsers,
+        help=f"Test against the specified browsers ({_SUPPORTED_BROWSERS})",
+    )
 
     return parser
 
@@ -87,8 +98,8 @@ def _validate_args(args):
         has_err = True
 
     if not args.browsers:
-        logging.error('At least one browser must be specified')
-        logging.error(f'Possible browsers: {_SUPPORTED_BROWSERS}')
+        logging.error("At least one browser must be specified")
+        logging.error(f"Possible browsers: {_SUPPORTED_BROWSERS}")
         has_err = True
 
     if has_err:
@@ -107,26 +118,32 @@ def _get_browser_obj(playwright, browser_name):
 
 
 def _check_console_logs(console_logs):
-    logging.info('Checking console logs')
+    logging.info("Checking console logs")
 
     has_err = False
     for log in console_logs:
-        if log.type not in {'warning', 'error'}:
+        if log.type not in {"warning", "error"}:
             continue
 
         # Skip all `Error with Permissions-Policy header: Unrecognized feature` warnings
-        perm_policy_warn = 'Error with Permissions-Policy header:'
+        perm_policy_warn = "Error with Permissions-Policy header:"
         if perm_policy_warn in log.text:
             logging.warning(f'Skipping Permissions-Policy warning. log.text="{log.text}"')
             continue
 
-        url = log.location.get('url')
+        url = log.location.get("url")
         if not url:
-            raise RuntimeError(f'Cannot find "url" during log parsing (log.type={log.type}, log.text="{log.text}", log.location="{log.location}")')
+            raise RuntimeError(
+                f'Cannot find "url" during log parsing (log.type={log.type}, log.text="{log.text}", log.location="{log.location}")'
+            )
 
-        for actual_text, endpoint in (_CANDID_UI_ERRORS_TO_IGNORE if log.type == 'error' else _CANDID_UI_WARNINGS_TO_IGNORE):
+        for actual_text, endpoint in (
+            _CANDID_UI_ERRORS_TO_IGNORE if log.type == "error" else _CANDID_UI_WARNINGS_TO_IGNORE
+        ):
             if actual_text == log.text.strip() and endpoint in url:
-                logging.warning(f'Found {log.type}, but it was expected (log.type="{actual_text}", endpoint="{endpoint}")')
+                logging.warning(
+                    f'Found {log.type}, but it was expected (log.type="{actual_text}", endpoint="{endpoint}")'
+                )
                 break
         else:
             logging.error(f'Found unexpected console log {log.type}. Text: "{log.text}", url: {url}')
@@ -134,109 +151,131 @@ def _check_console_logs(console_logs):
             has_err = True
 
     if has_err:
-        raise RuntimeError('Console has unexpected warnings and/or errors. Check previous logs')
+        raise RuntimeError("Console has unexpected warnings and/or errors. Check previous logs")
 
-    logging.info('Console logs are ok')
+    logging.info("Console logs are ok")
 
 
 def _click_button(page, button):
     logging.info(f'Clicking button "{button}"')
-    page.get_by_role('button', name=button).click()
+    page.get_by_role("button", name=button).click()
 
 
 def _set_text(page, text, value):
     logging.info(f'Setting text to "{value}"')
-    page.get_by_role('textbox', name=text).fill(value)
+    page.get_by_role("textbox", name=text).fill(value)
 
 
 def _test_frontend_ui_handler(page):
     # Set the name & Click the button
-    name = 'my name'
+    name = "my name"
     logging.info(f'Setting name "{name}"')
-    page.get_by_label('Enter your name:').fill(name)
-    _click_button(page, 'Click Me!')
+    page.get_by_label("Enter your name:").fill(name)
+    _click_button(page, "Click Me!")
 
     # Check if `#greeting` is populated correctly
-    greeting_id = '#greeting'
+    greeting_id = "#greeting"
     timeout_ms = 60000
     greeting_obj = page.wait_for_selector(greeting_id, timeout=timeout_ms)
     if greeting_obj:
         actual_value = greeting_obj.inner_text()
-        expected_value = f'Hello, {name}!'
+        expected_value = f"Hello, {name}!"
         if actual_value == expected_value:
             logging.info(f'"{actual_value}" found in "{greeting_id}"')
         else:
             raise RuntimeError(f'Expected greeting message is "{expected_value}", but found "{actual_value}"')
     else:
-        raise RuntimeError(f'Cannot find {greeting_id} selector')
+        raise RuntimeError(f"Cannot find {greeting_id} selector")
 
 
 def _test_candid_ui_handler(page):
     # Set the text & Click the "Query" button
-    text = 'hello, world'
-    _set_text(page, 'text', text)
-    _click_button(page, 'Query')
+    text = "hello, world"
+    _set_text(page, "text", text)
+    _click_button(page, "Query")
 
     # Check if `#output-list` is populated correctly (after the first click)
-    output_list_id = '#output-list'
+    output_list_id = "#output-list"
     timeout_ms = 60000
     _ = page.wait_for_selector(output_list_id, timeout=timeout_ms)
 
     # Reset the text & Click the "Random" button
-    _set_text(page, 'text', '')
-    _click_button(page, 'Random')
+    _set_text(page, "text", "")
+    _click_button(page, "Random")
     # ~
 
     # Check if `#output-list` is populated correctly  (after the second click)
-    output_list_obj = page.wait_for_selector(output_list_id, timeout=timeout_ms)
-    if output_list_obj:
-        output_list_lines = output_list_obj.inner_text().split('\n')
-        actual_num_lines, expected_num_lines = len(output_list_lines), 4
-        if actual_num_lines != expected_num_lines:
-            err = [f'Expected {expected_num_lines} lines of text but found {actual_num_lines}']
-            err.append('Lines:')
-            err.extend(output_list_lines)
-            raise RuntimeError('\n'.join(err))
-
-        # Extract random text from third line
-        random_text = re.search(r'"([^"]*)"', output_list_lines[2])
-        if not random_text:
-            raise RuntimeError(f'Cannot extract the random text from the third line: {output_list_lines[2]}')
-        random_text = random_text.group(1)
-
-        for i, text_str in enumerate([text, random_text]):
-            l1, l2 = (i * 2), (i * 2 + 1)
-
-            # First output line
-            actual_line, expected_line = output_list_lines[l1], f'› greet("{text_str}")'
-            if actual_line != expected_line:
-                raise RuntimeError(f'Expected {expected_line} line, but found {actual_line} (line {l1})')
-            logging.info(f'"{actual_line}" found in {output_list_id} at position {l1}')
-
-            # Second output line
-            actual_line, expected_line = output_list_lines[l2], f'("Hello, {text_str}!")'
-            if actual_line != expected_line:
-                raise RuntimeError(f'Expected {expected_line} line, but found {actual_line} (line {l2})')
-            logging.info(f'"{actual_line}" found in {output_list_id} at position {l2}')
-
-        logging.info(f'{output_list_id} lines are defined correctly')
-    else:
-        raise RuntimeError(f'Cannot find {output_list_id} selector')
+    #
+    # NOTE: `#output-list` already exists, so `wait_for_selector` won't work as expected.
+    #       We noticed that, especially for `Ubuntu 20.04` and `Webkit`, the two additional lines
+    #       created once the `Random` button was clicked, were not created properly.
+    #
+    #       For this reason there is this simple fallback logic that tries to look at the selector
+    #       for more than once by sleeping for some time.
+    fallback_retries = 10
+    fallback_sleep_sec = 5
+    last_err = None
+    for _ in range(fallback_retries):
+        try:
+            output_list_obj = page.wait_for_selector(output_list_id, timeout=timeout_ms)
+            if not output_list_obj:
+                raise RuntimeError(f"Cannot find {output_list_id} selector")
+
+            output_list_lines = output_list_obj.inner_text().split("\n")
+            actual_num_lines, expected_num_lines = len(output_list_lines), 4
+            if actual_num_lines != expected_num_lines:
+                err = [f"Expected {expected_num_lines} lines of text but found {actual_num_lines}"]
+                err.append("Lines:")
+                err.extend(output_list_lines)
+                raise RuntimeError("\n".join(err))
+
+            # Extract random text from third line
+            random_text = re.search(r'"([^"]*)"', output_list_lines[2])
+            if not random_text:
+                raise RuntimeError(f"Cannot extract the random text from the third line: {output_list_lines[2]}")
+            random_text = random_text.group(1)
+
+            for i, text_str in enumerate([text, random_text]):
+                line1, line2 = (i * 2), (i * 2 + 1)
+
+                # First output line
+                actual_line, expected_line = output_list_lines[line1], f'› greet("{text_str}")'
+                if actual_line != expected_line:
+                    raise RuntimeError(f"Expected {expected_line} line, but found {actual_line} (line {line1})")
+                logging.info(f'"{actual_line}" found in {output_list_id} at position {line1}')
+
+                # Second output line
+                actual_line, expected_line = output_list_lines[line2], f'("Hello, {text_str}!")'
+                if actual_line != expected_line:
+                    raise RuntimeError(f"Expected {expected_line} line, but found {actual_line} (line {line2})")
+                logging.info(f'"{actual_line}" found in {output_list_id} at position {line2}')
+
+            # All good!
+            last_err = None
+            logging.info(f"{output_list_id} lines are defined correctly")
+            break
+        except RuntimeError as run_err:
+            last_err = str(run_err)
+            logging.warning(f"Fallback hit! Sleeping for {fallback_sleep_sec} before continuing")
+            time.sleep(fallback_sleep_sec)
+
+    if last_err:
+        raise RuntimeError(last_err)
 
 
 def _handle_route_for_webkit(route):
-    url = route.request.url.replace('https://', 'http://')
+    url = route.request.url.replace("https://", "http://")
 
     headers = None
-    if any(map(url.endswith, ['.css', '.js', '.svg'])):
+    if any(map(url.endswith, [".css", ".js", ".svg"])):
         global _FRONTEND_URL
         assert _FRONTEND_URL
         headers = {
-            'referer': _FRONTEND_URL,
+            "referer": _FRONTEND_URL,
         }
 
     response = route.fetch(url=url, headers=headers)
-    assert response.status == 200, f'Expected 200 status code, but got {response.status}. Url: {url}'
+    assert response.status == 200, f"Expected 200 status code, but got {response.status}. Url: {url}"
     route.fulfill(response=response)
 
 
@@ -249,7 +288,7 @@ def _test_ui(ui, url, handler, browsers):
             logging.info(f'Checking "{browser_name}" browser')
             browser = _get_browser_obj(playwright, browser_name)
             if not browser:
-                raise RuntimeError(f'Cannot determine browser object for browser {browser_name}')
+                raise RuntimeError(f"Cannot determine browser object for browser {browser_name}")
 
             try:
                 browser = browser.launch(headless=True)
@@ -258,7 +297,7 @@ def _test_ui(ui, url, handler, browsers):
 
                 # Attach a listener to the page's console events
                 console_logs = []
-                page.on('console', lambda msg: console_logs.append(msg))
+                page.on("console", lambda msg: console_logs.append(msg))
 
                 # Webkit forces HTTPS:
                 #   - https://github.com/microsoft/playwright/issues/12975
@@ -266,14 +305,14 @@ def _test_ui(ui, url, handler, browsers):
                 if ui == _UI.FRONTEND and browser_name == _WEBKIT_BROWSER:
                     global _FRONTEND_URL
                     _FRONTEND_URL = url
-                    page.route('**/*', _handle_route_for_webkit)
+                    page.route("**/*", _handle_route_for_webkit)
 
                 page.goto(url)
 
                 handler(page)
                 _check_console_logs(console_logs)
             except Exception as e:
-                logging.error(f'Error: {str(e)}')
+                logging.error(f"Error: {str(e)}")
                 has_err = True
             finally:
                 if context:
@@ -294,9 +333,9 @@ def _main():
     if args.candid_url:
         _test_ui(_UI.CANDID, args.candid_url, _test_candid_ui_handler, args.browsers)
 
-    logging.info('DONE!')
+    logging.info("DONE!")
 
 
-if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
     _main()

From 837b2dad7b3d8a7f30f0b62f10f95efe26d8c82b Mon Sep 17 00:00:00 2001
From: Marco <mmicu.github00@gmail.com>
Date: Thu, 30 Nov 2023 21:53:16 +0100
Subject: [PATCH 10/10] Upgrade to checkout@v4

Co-authored-by: Marcin Nowak-Liebiediew <marcin.liebiediew@dfinity.org>
---
 .github/workflows/e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 32b4623716..b3fc7470fd 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -160,7 +160,7 @@ jobs:
         os: [macos-12, ubuntu-20.04, ubuntu-22.04]
     steps:
       - name: Checking out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setting up Python
         uses: actions/setup-python@v4
         with: