openedx · feanil · Nov 12, 2024 · Nov 12, 2024
diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml
@@ -15,7 +15,7 @@ defaults:
 jobs:
   check-requirements:
     name: Compile requirements
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
 
     steps:
       # Only run remaining steps if there are changes to requirements/**

diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         python-version:
           - "3.11"
-        os: ["ubuntu-24.04"]
+        os: ["ubuntu-22.04"]
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-24.04]
+        os: [ubuntu-22.04]
         python-version:
           - "3.11"
         # 'pinned' is used to install the latest patch version of Django
@@ -126,7 +126,7 @@ jobs:
     if: always()
     needs:
       - check_migrations
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/[email protected]

diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   run-pylint:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -75,7 +75,7 @@ jobs:
     if: always()
     needs:
       - run-pylint
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/[email protected]

diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-24.04]
+        os: [ubuntu-22.04]
         python-version:
           - "3.11"
         node-version: [20]

diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-24.04]
+        os: [ubuntu-22.04]
         python-version:
           - "3.11"
         node-version: [18, 20]

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -15,7 +15,7 @@ concurrency:
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    runs-on: ${{ matrix.os-version }}
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version:
@@ -43,27 +43,22 @@ jobs:
           - "xmodule-with-cms"
         mongo-version:
           - "7.0"
-        os-version:
-          - ubuntu-24.04
 
-        # It's useful to run some subset of the tests on the older version of Ubuntu
-        # so that we don't spend too many resources on this but can find major issues quickly
-        # while we're in a situation where we support two versions.  This section may be commented
-        # out when not in use to easily add/drop future support for any given major dependency.
+        # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore)
+        # This code is left here as an example for future refernce in case we need to reduce the number of shards we're
+        # testing but still have good confidence with older versions of mongo.  We use Mongo 4.4 in the example.
         #
-        # We're testing the older version of Ubuntu and running the xmodule tests since those rely on many
-        # dependent complex libraries and will hopefully catch most issues quickly.
-        include:
-          - shard_name: "xmodule-with-cms"
-            python-version: "3.11"
-            django-version: "pinned"
-            mongo-version: "7.0"
-            os-version: "ubuntu-22.04"
-          - shard_name: "xmodule-with-lms"
-            python-version: "3.11"
-            django-version: "pinned"
-            mongo-version: "7.0"
-            os-version: "ubuntu-22.04"
+        # exclude:
+        #   - mongo-version: "4.4"
+        # include:
+        #   - shard_name: "xmodule-with-cms"
+        #     python-version: "3.11"
+        #     django-version: "pinned"
+        #     mongo-version: "4.4"
+        #   - shard_name: "xmodule-with-lms"
+        #     python-version: "3.11"
+        #     django-version: "pinned"
+        #     mongo-version: "4.4"
 
     steps:
       - name: checkout repo
@@ -95,10 +90,19 @@ jobs:
           activate = 1
           EOF
 
-      - name: Start MongoDB
-        uses: supercharge/[email protected]
-        with:
-          mongodb-version: ${{ matrix.mongo-version }}
+      - name: install mongo version
+        run: |
+          if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then
+            wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add -
+            echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list
+            sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
+          fi
+
+      - name: start mongod server for tests
+        run: |
+          sudo mkdir -p /data/db
+          sudo chmod -R a+rw /data/db
+          mongod &
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -160,7 +164,7 @@ jobs:
           overwrite: true
 
   collect-and-verify:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python
@@ -225,7 +229,7 @@ jobs:
   # https://github.com/orgs/community/discussions/33579
   success:
     name: Unit tests successful
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     if: always()
     needs: [run-tests]
     steps:
@@ -236,7 +240,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     needs: [run-tests]
     steps:
       - uses: actions/checkout@v4
@@ -264,7 +268,7 @@ jobs:
           overwrite: true
 
   merge-artifacts:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     needs: [compile-warnings-report]
     steps:
       - name: Merge Pytest Warnings JSON Artifacts
@@ -284,7 +288,7 @@ jobs:
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     needs: [run-tests]
     strategy:
       matrix:

diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt
@@ -15,7 +15,6 @@
 # Note: Changes to this file will automatically be used by other repos, referencing
 #  this file from Github directly. It does not require packaging in edx-lint.
 
-
 # using LTS django version
 Django<5.0
 
@@ -26,13 +25,6 @@ elasticsearch<7.14.0
 
 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
 
-# Cause: https://github.com/openedx/event-tracking/pull/290
-# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform.
-# We will pin event-tracking to do not break existing installations
-# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586
-# has been resolved and edx-platform is running with pymongo>=4.4.0
-
-
 # Cause: https://github.com/openedx/edx-lint/issues/458
 # This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved.
 pip<24.3
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
@@ -105,6 +105,13 @@ event-tracking==3.0.0
 # https://github.com/openedx/edx-platform/issues/31616
 libsass==0.10.0
 
+# Date: 2024-04-30
+# lxml>=5.0 introduced breaking changes related to system dependencies
+# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean]
+# This constraint can be removed once we upgrade to Python 3.11
+# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35272
+lxml<5.0
+
 # Date: 2018-12-14
 # markdown>=3.4.0 has failures due to internal refactorings which causes the tests to fail
 # pinning the version untill the issue gets resolved in the package itself
@@ -184,3 +191,8 @@ social-auth-app-django<=5.4.1
 # # Date: 2024-10-14
 # # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first.
 # edx-rest-api-client==5.7.1
+
+# Date: 2024-04-24
+# xmlsec==1.3.14 breaking tests or all builds, can be removed once a fix is available
+# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35264
+xmlsec<1.3.14
diff --git a/requirements/edx-sandbox/base.in b/requirements/edx-sandbox/base.in
@@ -2,7 +2,7 @@
 
 chem                                # A helper library for chemistry calculations
 cryptography                        # Implementations of assorted cryptography algorithms
-lxml[html_clean]                    # XML parser
+lxml                                # XML parser
 matplotlib                          # 2D plotting library
 networkx                            # Utilities for creating, manipulating, and studying network graphs
 nltk                                # Natural language processing; used by the chem package

diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt
@@ -26,13 +26,11 @@ joblib==1.4.2
     # via nltk
 kiwisolver==1.4.7
     # via matplotlib
-lxml[html-clean,html_clean]==5.3.0
+lxml==4.9.4
     # via
+    #   -c requirements/edx-sandbox/../constraints.txt
     #   -r requirements/edx-sandbox/base.in
-    #   lxml-html-clean
     #   openedx-calc
-lxml-html-clean==0.3.1
-    # via lxml
 markupsafe==3.0.2
     # via
     #   chem

diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
@@ -708,21 +708,19 @@ loremipsum==1.0.5
     # via ora2
 lti-consumer-xblock==9.11.3
     # via -r requirements/edx/kernel.in
-lxml[html-clean,html_clean]==5.3.0
+lxml==4.9.4
     # via
+    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/kernel.in
     #   edx-i18n-tools
     #   edxval
     #   lti-consumer-xblock
-    #   lxml-html-clean
     #   olxcleaner
     #   openedx-calc
     #   ora2
     #   python3-saml
     #   xblock
     #   xmlsec
-lxml-html-clean==0.3.1
-    # via lxml
 mailsnake==1.6.4
     # via -r requirements/edx/bundled.in
 mako==1.3.6
@@ -1287,8 +1285,10 @@ xblock-utils==4.0.0
     # via
     #   edx-sga
     #   xblock-poll
-xmlsec==1.3.14
-    # via python3-saml
+xmlsec==1.3.13
+    # via
+    #   -c requirements/edx/../constraints.txt
+    #   python3-saml
 xss-utils==0.6.0
     # via -r requirements/edx/kernel.in
 yarl==1.17.0

diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
@@ -1183,26 +1183,21 @@ lti-consumer-xblock==9.11.3
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
-lxml[html-clean]==5.3.0
+lxml==4.9.4
     # via
+    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
     #   edx-i18n-tools
     #   edxval
     #   lti-consumer-xblock
-    #   lxml-html-clean
     #   olxcleaner
     #   openedx-calc
     #   ora2
     #   pyquery
     #   python3-saml
     #   xblock
     #   xmlsec
-lxml-html-clean==0.3.1
-    # via
-    #   -r requirements/edx/doc.txt
-    #   -r requirements/edx/testing.txt
-    #   lxml
 mailsnake==1.6.4
     # via
     #   -r requirements/edx/doc.txt
@@ -2275,8 +2270,9 @@ xblock-utils==4.0.0
     #   -r requirements/edx/testing.txt
     #   edx-sga
     #   xblock-poll
-xmlsec==1.3.14
+xmlsec==1.3.13
     # via
+    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
     #   python3-saml

diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt
@@ -850,23 +850,19 @@ loremipsum==1.0.5
     #   ora2
 lti-consumer-xblock==9.11.3
     # via -r requirements/edx/base.txt
-lxml[html-clean]==5.3.0
+lxml==4.9.4
     # via
+    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/base.txt
     #   edx-i18n-tools
     #   edxval
     #   lti-consumer-xblock
-    #   lxml-html-clean
     #   olxcleaner
     #   openedx-calc
     #   ora2
     #   python3-saml
     #   xblock
     #   xmlsec
-lxml-html-clean==0.3.1
-    # via
-    #   -r requirements/edx/base.txt
-    #   lxml
 mailsnake==1.6.4
     # via -r requirements/edx/base.txt
 mako==1.3.6
@@ -1583,8 +1579,9 @@ xblock-utils==4.0.0
     #   -r requirements/edx/base.txt
     #   edx-sga
     #   xblock-poll
-xmlsec==1.3.14
+xmlsec==1.3.13
     # via
+    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/base.txt
     #   python3-saml
 xss-utils==0.6.0

diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in
@@ -103,7 +103,7 @@ importlib_metadata                  # Used to access entry_points in i18n_api pl
 jsonfield                           # Django model field for validated JSON; used in several apps
 laboratory                          # Library for testing that code refactors/infrastructure changes produce identical results
 importlib_metadata                  # Used to access entry_points in i18n_api plugin
-lxml[html_clean]                    # XML parser
+lxml                                # XML parser
 lti-consumer-xblock>=7.3.0
 mako                                # Primary template language used for server-side page rendering
 Markdown                            # Convert text markup to HTML; used in capa problems, forums, and course wikis