Skip to content

Commit

Permalink
Merge pull request #5896 from xapi-project/feature/py3
Browse files Browse the repository at this point in the history
Python3 update feature merge
  • Loading branch information
liulinC authored Sep 4, 2024
2 parents b1fbddc + 09603bd commit 59664cc
Show file tree
Hide file tree
Showing 173 changed files with 5,293 additions and 8,439 deletions.
1 change: 0 additions & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ jobs:
- name: Make install smoketest
run: |
opam exec -- make install DESTDIR=$(mktemp -d)
opam exec -- make install DESTDIR=$(mktemp -d) BUILD_PY2=NO
- name: Check disk space
run: df -h || true
Expand Down
43 changes: 2 additions & 41 deletions .github/workflows/other.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["2.7", "3.11"]
python-version: ["3.11"]
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand All @@ -39,48 +39,11 @@ jobs:

- uses: pre-commit/[email protected]
name: Run pre-commit checks (no spaces at end of lines, etc)
if: ${{ matrix.python-version != '2.7' }}
with:
extra_args: --all-files --verbose --hook-stage commit
env:
SKIP: no-commit-to-branch

- name: Install dependencies only needed for python 2
if: ${{ matrix.python-version == '2.7' }}
run: pip install enum

- name: Install dependencies only needed for python 3
if: ${{ matrix.python-version != '2.7' }}
run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt

- name: Install common dependencies for Python ${{matrix.python-version}}
run: pip install future mock pytest-coverage pytest-mock

- name: Run Pytest for python 2 and get code coverage
if: ${{ matrix.python-version == '2.7' }}
run: >
pytest
--cov=scripts --cov=ocaml/xcp-rrdd
scripts/ ocaml/xcp-rrdd -vv -rA
--junitxml=.git/pytest${{matrix.python-version}}.xml
--cov-report term-missing
--cov-report xml:.git/coverage${{matrix.python-version}}.xml
env:
PYTHONDEVMODE: yes

- name: Run Pytest for python 3 and get code coverage
if: ${{ matrix.python-version != '2.7' }}
run: >
pytest
--cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
scripts/ ocaml/xcp-rrdd python3/ -vv -rA
--junitxml=.git/pytest${{matrix.python-version}}.xml
--cov-report term-missing
--cov-report xml:.git/coverage${{matrix.python-version}}.xml
env:
PYTHONDEVMODE: yes
PYTHONPATH: "python3:python3/tests/stubs"

- name: Upload coverage report to Coveralls
uses: coverallsapp/github-action@v2
with:
Expand All @@ -90,7 +53,6 @@ jobs:
parallel: true

- uses: dciborow/[email protected]
if: ${{ matrix.python-version != '2.7' }}
with:
reporter: github-pr-review
level: warning
Expand All @@ -99,8 +61,7 @@ jobs:
continue-on-error: true

- name: Run pytype checks
if: ${{ matrix.python-version != '2.7' }}
run: ./pytype_reporter.py
run: pip install pandas pytype toml && ./pytype_reporter.py
env:
PR_NUMBER: ${{ github.event.number }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: XenAPI
path: scripts/examples/python/dist/
path: python3/examples/dist/

build-sdks:
name: Build and upload SDK artifacts
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/setup-xapi-environment/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ runs:
shell: bash
run: sudo apt-get update

- name: Install python2
- name: Install python3
shell: bash
run: sudo apt-get install python2
run: sudo apt-get install python3

- name: Use disk with more space for TMPDIR and XDG_CACHE_HOME
shell: bash
Expand Down
8 changes: 4 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ config.mk
# python packaging
**/__pycache__/
**/*.pyc
scripts/examples/python/setup.py
scripts/examples/python/XenAPI.egg-info/
scripts/examples/python/build/
scripts/examples/python/dist/
python3/examples/setup.py
python3/examples/XenAPI.egg-info/
python3/examples/build/
python3/examples/dist/

# ignore file needed for building the SDK
ocaml/sdk-gen/csharp/XE_SR_ERRORCODES.xml
Expand Down
101 changes: 100 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# pre-commit run -av --hook-stage pre-push
#
default_stages: [commit, push]
default_language_version:
python: python3.11
repos:
# Recommendation for a minimal git pre-commit hook:
# https://github.com/pre-commit/pre-commit-hooks/blob/main/README.md:
Expand All @@ -29,6 +31,95 @@ repos:
- id: check-executables-have-shebangs
exclude: ocaml


# Improve Python formatting incrementally:
# https://dev.to/akaihola/improving-python-code-incrementally-3f7a
#
# darker checks if staged python changes are formatted according using
# the PEP8-aligned black formatter. It also checks if the imports are sorted.
#
# It is a good idea to run this before committing, and it is also run in the
# GitHub Workflow.
#
# Note: darker only checks the changes in files ending in .py!
# Python scripts that don't end in .py should be renamed to have the .py extension
# when moving them to python3/bin.
# (remove the .py extension in the Makefile when installing the file)
#
- repo: https://github.com/akaihola/darker
rev: 1.7.3
hooks:
- id: darker
files: python3/
name: check changes in Python3 tree using darker and isort
args: [--diff, --skip-string-normalization, --isort, -tpy36]
additional_dependencies: [isort]

#
# Run pytest and diff-cover to check that the new /python3 test suite in passes.
# This hook uses a local venv containing the required dependencies. When adding
# new dependencies, they should be added to the additional_dependencies below.
#
- repo: local
hooks:
- id: pytest
files: python3/
name: check that the Python3 test suite in passes
entry: env PYTHONDEVMODE=yes sh -c 'coverage run && coverage xml &&
coverage html && coverage report &&
diff-cover --ignore-whitespace --compare-branch=origin/feature/py3
--show-uncovered --html-report .git/coverage-diff.html
--fail-under 50 .git/coverage3.11.xml'
require_serial: true
pass_filenames: false
language: python
types: [python]
additional_dependencies:
- coverage
- diff-cover
- future
- opentelemetry-api
- opentelemetry-exporter-zipkin-json
- opentelemetry-sdk
- pytest-mock
- mock
- wrapt
- XenAPI


- repo: https://github.com/RobertCraigie/pyright-python
rev: v1.1.372
hooks:
- id: pyright
name: check that python3 tree passes pyright/VSCode check
files: python3/
additional_dependencies:
- mock
- opentelemetry-api
- opentelemetry-exporter-zipkin-json
- opentelemetry-sdk
- pytest
- pyudev
- XenAPI


# Check that pylint passes for the changes in new /python3 code.
- repo: local
hooks:
- id: pylint
files: python3/
stages: [push]
name: check that changes to python3 tree pass pylint
entry: diff-quality --violations=pylint
--ignore-whitespace --compare-branch=origin/feature/py3
pass_filenames: false
language: python
types: [python]
additional_dependencies: [diff-cover, pylint, pytest]


# pre-push hook (it only runs if you install pre-commit as a pre-push hook):
# It can be manually tested using: `pre-commit run -av --hook-stage push`
# Recommendation for a minimal git pre-push hook:
# While using pre-commit yields great results, it
# is "not fast". Therefore only run it pre-push,
Expand All @@ -53,4 +144,12 @@ repos:
# developers have such version installed, it can be configured here:
# language_version: python3.11
require_serial: true
additional_dependencies: [pandas, pytype]
additional_dependencies:
- future
- opentelemetry-api
- opentelemetry-exporter-zipkin-json
- opentelemetry-sdk
- pandas
- pytest
- pytype
files: python3/
7 changes: 2 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,7 @@ test:
trap "kill $${PSTREE_SLEEP_PID}" INT TERM EXIT; \
timeout --foreground $(TEST_TIMEOUT2) \
dune runtest --profile=$(PROFILE) --error-reporting=twice -j $(JOBS)
ifneq ($(PY_TEST), NO)
dune build @runtest-python --profile=$(PROFILE)
endif

stresstest:
dune build @stresstest --profile=$(PROFILE) --no-buffer -j $(JOBS)
Expand Down Expand Up @@ -123,7 +121,7 @@ sdk:
cp -r _build/default/ocaml/sdk-gen/java/autogen/* $(XAPISDK)/java
cp -r _build/default/ocaml/sdk-gen/powershell/autogen/* $(XAPISDK)/powershell
cp -r _build/default/ocaml/sdk-gen/go/autogen/* $(XAPISDK)/go
cp scripts/examples/python/XenAPI/XenAPI.py $(XAPISDK)/python
cp python3/examples/XenAPI/XenAPI.py $(XAPISDK)/python
sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/csharp
sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/powershell

Expand All @@ -138,7 +136,7 @@ sdk-build-java: sdk
cd _build/install/default/xapi/sdk/java && mvn -f xen-api/pom.xml -B clean package install -Drevision=0.0

python:
$(MAKE) -C scripts/examples/python build
$(MAKE) -C python3/examples build

doc-json:
dune exec --profile=$(PROFILE) -- ocaml/idl/json_backend/gen_json.exe -destdir $(XAPIDOC)/jekyll
Expand Down Expand Up @@ -239,7 +237,6 @@ install: build doc sdk doc-json
install -D ./ocaml/xenopsd/scripts/block $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/block
install -D ./ocaml/xenopsd/scripts/xen-backend.rules $(DESTDIR)/$(ETCDIR)/udev/rules.d/xen-backend.rules
install -D ./ocaml/xenopsd/scripts/tap $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/tap
install -D ./ocaml/xenopsd/scripts/qemu-vif-script $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/qemu-vif-script
install -D ./ocaml/xenopsd/scripts/setup-vif-rules $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/setup-vif-rules
install -D ./_build/install/default/bin/pvs-proxy-ovs-setup $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/pvs-proxy-ovs-setup
(cd $(DESTDIR)/$(XENOPSD_LIBEXECDIR) && ln -s pvs-proxy-ovs-setup setup-pvs-proxy-rules)
Expand Down
Loading

0 comments on commit 59664cc

Please sign in to comment.