diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index 82298af70a54..048cbe6b006b 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -15,7 +15,7 @@ defaults: jobs: check-requirements: name: Compile requirements - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: # Only run remaining steps if there are changes to requirements/** diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index d989ff9db288..458e00fc6b1f 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -10,7 +10,7 @@ jobs: matrix: python-version: - "3.11" - os: ["ubuntu-24.04"] + os: ["ubuntu-22.04"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 84e334d68872..624caddd5309 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-24.04] + os: [ubuntu-22.04] python-version: - "3.11" # 'pinned' is used to install the latest patch version of Django @@ -126,7 +126,7 @@ jobs: if: always() needs: - check_migrations - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Decide whether the needed jobs succeeded or failed # uses: re-actors/alls-green@v1.2.1 diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 9a654e09e711..8860aced7f92 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -8,7 +8,7 @@ on: jobs: run-pylint: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -75,7 +75,7 @@ jobs: if: always() needs: - run-pylint - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Decide whether the needed jobs succeeded or failed # uses: re-actors/alls-green@v1.2.1 diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 310f9f83bf3d..84610123493c 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-24.04] + os: [ubuntu-22.04] python-version: - "3.11" node-version: [20] diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index e08b2dce8127..4fe66e2a7778 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-24.04] + os: [ubuntu-22.04] python-version: - "3.11" node-version: [18, 20] diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e691e16e47f1..854677b93cff 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: run-tests: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - runs-on: ${{ matrix.os-version }} + runs-on: ubuntu-22.04 strategy: matrix: python-version: @@ -43,27 +43,22 @@ jobs: - "xmodule-with-cms" mongo-version: - "7.0" - os-version: - - ubuntu-24.04 - # It's useful to run some subset of the tests on the older version of Ubuntu - # so that we don't spend too many resources on this but can find major issues quickly - # while we're in a situation where we support two versions. This section may be commented - # out when not in use to easily add/drop future support for any given major dependency. + # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore) + # This code is left here as an example for future refernce in case we need to reduce the number of shards we're + # testing but still have good confidence with older versions of mongo. We use Mongo 4.4 in the example. # - # We're testing the older version of Ubuntu and running the xmodule tests since those rely on many - # dependent complex libraries and will hopefully catch most issues quickly. - include: - - shard_name: "xmodule-with-cms" - python-version: "3.11" - django-version: "pinned" - mongo-version: "7.0" - os-version: "ubuntu-22.04" - - shard_name: "xmodule-with-lms" - python-version: "3.11" - django-version: "pinned" - mongo-version: "7.0" - os-version: "ubuntu-22.04" + # exclude: + # - mongo-version: "4.4" + # include: + # - shard_name: "xmodule-with-cms" + # python-version: "3.11" + # django-version: "pinned" + # mongo-version: "4.4" + # - shard_name: "xmodule-with-lms" + # python-version: "3.11" + # django-version: "pinned" + # mongo-version: "4.4" steps: - name: checkout repo @@ -95,10 +90,19 @@ jobs: activate = 1 EOF - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.11.0 - with: - mongodb-version: ${{ matrix.mongo-version }} + - name: install mongo version + run: | + if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then + wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add - + echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list + sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*" + fi + + - name: start mongod server for tests + run: | + sudo mkdir -p /data/db + sudo chmod -R a+rw /data/db + mongod & - name: Setup Python uses: actions/setup-python@v5 @@ -160,7 +164,7 @@ jobs: overwrite: true collect-and-verify: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Setup Python @@ -225,7 +229,7 @@ jobs: # https://github.com/orgs/community/discussions/33579 success: name: Unit tests successful - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 if: always() needs: [run-tests] steps: @@ -236,7 +240,7 @@ jobs: jobs: ${{ toJSON(needs) }} compile-warnings-report: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [run-tests] steps: - uses: actions/checkout@v4 @@ -264,7 +268,7 @@ jobs: overwrite: true merge-artifacts: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [compile-warnings-report] steps: - name: Merge Pytest Warnings JSON Artifacts @@ -284,7 +288,7 @@ jobs: # Combine and upload coverage reports. coverage: if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [run-tests] strategy: matrix: diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 5188f37250ef..b8166ba67540 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -15,7 +15,6 @@ # Note: Changes to this file will automatically be used by other repos, referencing # this file from Github directly. It does not require packaging in edx-lint. - # using LTS django version Django<5.0 @@ -26,13 +25,6 @@ elasticsearch<7.14.0 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected -# Cause: https://github.com/openedx/event-tracking/pull/290 -# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform. -# We will pin event-tracking to do not break existing installations -# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586 -# has been resolved and edx-platform is running with pymongo>=4.4.0 - - # Cause: https://github.com/openedx/edx-lint/issues/458 # This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved. pip<24.3 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 8348603e93dc..de8760136f00 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -105,6 +105,13 @@ event-tracking==3.0.0 # https://github.com/openedx/edx-platform/issues/31616 libsass==0.10.0 +# Date: 2024-04-30 +# lxml>=5.0 introduced breaking changes related to system dependencies +# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] +# This constraint can be removed once we upgrade to Python 3.11 +# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35272 +lxml<5.0 + # Date: 2018-12-14 # markdown>=3.4.0 has failures due to internal refactorings which causes the tests to fail # pinning the version untill the issue gets resolved in the package itself @@ -184,3 +191,8 @@ social-auth-app-django<=5.4.1 # # Date: 2024-10-14 # # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first. # edx-rest-api-client==5.7.1 + +# Date: 2024-04-24 +# xmlsec==1.3.14 breaking tests or all builds, can be removed once a fix is available +# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35264 +xmlsec<1.3.14 diff --git a/requirements/edx-sandbox/base.in b/requirements/edx-sandbox/base.in index 0c331ce95d62..4c1a61bc2723 100644 --- a/requirements/edx-sandbox/base.in +++ b/requirements/edx-sandbox/base.in @@ -2,7 +2,7 @@ chem # A helper library for chemistry calculations cryptography # Implementations of assorted cryptography algorithms -lxml[html_clean] # XML parser +lxml # XML parser matplotlib # 2D plotting library networkx # Utilities for creating, manipulating, and studying network graphs nltk # Natural language processing; used by the chem package diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 8cf0cbb82904..eb74898596ff 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -26,13 +26,11 @@ joblib==1.4.2 # via nltk kiwisolver==1.4.7 # via matplotlib -lxml[html-clean,html_clean]==5.3.0 +lxml==4.9.4 # via + # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/base.in - # lxml-html-clean # openedx-calc -lxml-html-clean==0.3.1 - # via lxml markupsafe==3.0.2 # via # chem diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1b81540252bc..702af98e25a9 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -708,21 +708,19 @@ loremipsum==1.0.5 # via ora2 lti-consumer-xblock==9.11.3 # via -r requirements/edx/kernel.in -lxml[html-clean,html_clean]==5.3.0 +lxml==4.9.4 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # edx-i18n-tools # edxval # lti-consumer-xblock - # lxml-html-clean # olxcleaner # openedx-calc # ora2 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 - # via lxml mailsnake==1.6.4 # via -r requirements/edx/bundled.in mako==1.3.6 @@ -1287,8 +1285,10 @@ xblock-utils==4.0.0 # via # edx-sga # xblock-poll -xmlsec==1.3.14 - # via python3-saml +xmlsec==1.3.13 + # via + # -c requirements/edx/../constraints.txt + # python3-saml xss-utils==0.6.0 # via -r requirements/edx/kernel.in yarl==1.17.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0143d81709d1..af0dcd8c4a26 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1183,14 +1183,14 @@ lti-consumer-xblock==9.11.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -lxml[html-clean]==5.3.0 +lxml==4.9.4 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-i18n-tools # edxval # lti-consumer-xblock - # lxml-html-clean # olxcleaner # openedx-calc # ora2 @@ -1198,11 +1198,6 @@ lxml[html-clean]==5.3.0 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # lxml mailsnake==1.6.4 # via # -r requirements/edx/doc.txt @@ -2275,8 +2270,9 @@ xblock-utils==4.0.0 # -r requirements/edx/testing.txt # edx-sga # xblock-poll -xmlsec==1.3.14 +xmlsec==1.3.13 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # python3-saml diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 1ed8a41699b7..cb9e53ee83a7 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -850,23 +850,19 @@ loremipsum==1.0.5 # ora2 lti-consumer-xblock==9.11.3 # via -r requirements/edx/base.txt -lxml[html-clean]==5.3.0 +lxml==4.9.4 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-i18n-tools # edxval # lti-consumer-xblock - # lxml-html-clean # olxcleaner # openedx-calc # ora2 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 - # via - # -r requirements/edx/base.txt - # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt mako==1.3.6 @@ -1583,8 +1579,9 @@ xblock-utils==4.0.0 # -r requirements/edx/base.txt # edx-sga # xblock-poll -xmlsec==1.3.14 +xmlsec==1.3.13 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # python3-saml xss-utils==0.6.0 diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 7323c243accf..a5b510742ac7 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -103,7 +103,7 @@ importlib_metadata # Used to access entry_points in i18n_api pl jsonfield # Django model field for validated JSON; used in several apps laboratory # Library for testing that code refactors/infrastructure changes produce identical results importlib_metadata # Used to access entry_points in i18n_api plugin -lxml[html_clean] # XML parser +lxml # XML parser lti-consumer-xblock>=7.3.0 mako # Primary template language used for server-side page rendering Markdown # Convert text markup to HTML; used in capa problems, forums, and course wikis diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 28e358a32cf7..29d923435dc5 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -899,13 +899,13 @@ loremipsum==1.0.5 # ora2 lti-consumer-xblock==9.11.3 # via -r requirements/edx/base.txt -lxml[html-clean]==5.3.0 +lxml==4.9.4 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-i18n-tools # edxval # lti-consumer-xblock - # lxml-html-clean # olxcleaner # openedx-calc # ora2 @@ -913,10 +913,6 @@ lxml[html-clean]==5.3.0 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 - # via - # -r requirements/edx/base.txt - # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt mako==1.3.6 @@ -1682,8 +1678,9 @@ xblock-utils==4.0.0 # -r requirements/edx/base.txt # edx-sga # xblock-poll -xmlsec==1.3.14 +xmlsec==1.3.13 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # python3-saml xss-utils==0.6.0 diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 704baaff2c79..288d660b14be 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -77,8 +77,10 @@ jmespath==1.0.1 # via # boto3 # botocore -lxml==5.3.0 - # via zeep +lxml==4.9.4 + # via + # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt + # zeep more-itertools==10.5.0 # via simple-salesforce newrelic==10.2.0 diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index 4cb3de607db7..da4ce1b0391a 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -116,7 +116,7 @@ jmespath==1.0.1 # -r scripts/user_retirement/requirements/base.txt # boto3 # botocore -lxml==5.3.0 +lxml==4.9.4 # via # -r scripts/user_retirement/requirements/base.txt # zeep