diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 16e4398d..33a4ba6b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -10,14 +10,16 @@ on: jobs: check-manpage: runs-on: ubuntu-20.04 - name: Check manpage + name: Check docs steps: - uses: actions/checkout@v3 - name: Install build dependencies run: | + sudo add-apt-repository -y universe + sudo add-apt-repository -y ppa:inkscape.dev/stable sudo apt-get update - sudo apt-get -y install libunwind-dev binutils-dev libiberty-dev help2man + sudo apt-get -y install libunwind-dev binutils-dev libiberty-dev help2man inkscape - name: Compile Austin run: | @@ -25,10 +27,10 @@ jobs: ./configure make - - name: Generate manpage - run: bash doc/genman.sh + - name: Generate docs + run: bash doc/gen.sh - - name: Check manpage + - name: Check docs run: git diff -I".* DO NOT MODIFY.*" -I"[.]TH AUSTIN.*" --exit-code src/austin.1 cppcheck-linux: @@ -89,7 +91,7 @@ jobs: run: brew install cppcheck - name: Check source code - run: cppcheck -q -f --error-exitcode=1 --inline-suppr src + run: cppcheck -q -f --error-exitcode=1 --inline-suppr --check-level=exhaustive src codespell: runs-on: ubuntu-20.04 @@ -156,7 +158,7 @@ jobs: sudo apt-get update sudo apt-get -y install \ valgrind \ - python3.{8..12} \ + python3.{8..13} \ python3.10-full python3.10-dev python3.10 -m venv .venv diff --git a/.github/workflows/pre_release.yml b/.github/workflows/pre_release.yml index 82817ff0..ef44102a 100644 --- a/.github/workflows/pre_release.yml +++ b/.github/workflows/pre_release.yml @@ -124,7 +124,7 @@ jobs: sed -i "" "s/$PREV_VERSION/$VERSION/g" src/austin.h echo "::set-output name=version::$VERSION" - gcc-11 -Wall -O3 -Os -o src/austin src/*.c + gcc-12 -Wall -O3 -Os -o src/austin src/*.c pushd src zip -r austin-${VERSION}-mac64.zip austin diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d289ff1..92e7f376 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -127,7 +127,7 @@ jobs: /bin/find . -type f -exec sed -i "s/%VERSION%/$VERSION/g" {} \; ; choco apikey --key ${{ secrets.CHOCO_APIKEY }} --source https://push.chocolatey.org/ choco pack - choco push + choco push --source https://push.chocolatey.org/ popd - name: Upload artifacts to release @@ -186,7 +186,7 @@ jobs: export VERSION=$(cat src/austin.h | sed -n -E "s/^#define VERSION[ ]+\"(.+)\"/\1/p") echo "::set-output name=version::$VERSION" - gcc-11 -Wall -O3 -Os -o src/austin src/*.c + gcc-12 -Wall -O3 -Os -o src/austin src/*.c pushd src zip -r austin-${VERSION}-mac64.zip austin diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8931f7f1..9b109cce 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] env: AUSTIN_TESTS_PYTHON_VERSIONS: ${{ matrix.python-version }} @@ -132,7 +132,7 @@ jobs: - name: Run functional Austin tests (with sudo) run: | ulimit -c unlimited - echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern + sudo echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern sudo -E env PATH="$PATH" .venv/bin/pytest --pastebin=failed -svr a test/functional -k "not austinp" if: always() @@ -228,7 +228,7 @@ jobs: - uses: actions/checkout@v3 - name: Compile Austin - run: gcc-11 -Wall -Werror -O3 -g src/*.c -o src/austin + run: gcc-12 -Wall -Werror -O3 -g src/*.c -o src/austin - uses: actions/upload-artifact@v3 with: @@ -259,7 +259,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] env: AUSTIN_TESTS_PYTHON_VERSIONS: ${{ matrix.python-version }} @@ -385,7 +385,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] env: AUSTIN_TESTS_PYTHON_VERSIONS: ${{ matrix.python-version }} @@ -474,7 +474,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] env: AUSTIN_TESTS_PYTHON_VERSIONS: ${{ matrix.python-version }} diff --git a/ChangeLog b/ChangeLog index ecf3b31d..6b401317 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +2024-10-14 v3.7.0 + + Added support for CPython 3.13. + + Improve support for Python processes running in containers. + + Removed the exclude-empty option. + + Bugfix: fixed a bug with the MOJO binary format that caused the line end + position to wrongly be set to a non-zero value for CPython < 3.11, where line + end information is not actually available. + + 2023-10-04 v3.6.0 Added support for CPython 3.12 diff --git a/README.md b/README.md index f1ac5458..9e6008de 100644 --- a/README.md +++ b/README.md @@ -300,8 +300,6 @@ requires no instrumentation and has practically no impact on the tracee. https://github.com/P403n1x87/austin/wiki/The-MOJO-file-format for more details. -C, --children Attach to child processes. - -e, --exclude-empty Do not output samples of threads with no frame - stacks. -f, --full Produce the full set of metrics (time +mem -mem). -g, --gc Sample the garbage collector state. -h, --heap=n_mb Maximum heap size to allocate to increase sampling @@ -561,7 +559,7 @@ folder in either the SVG, PDF or PNG format # Compatibility -Austin supports Python 3.8 through 3.12, and has been tested on the following +Austin supports Python 3.8 through 3.13, and has been tested on the following platforms and architectures | | | | | diff --git a/configure.ac b/configure.ac index 9d1b0cef..ca99d0bf 100644 --- a/configure.ac +++ b/configure.ac @@ -6,7 +6,7 @@ AC_PREREQ([2.69]) # from scripts.utils import get_current_version_from_changelog as version # print(f"AC_INIT([austin], [{version()}], [https://github.com/p403n1x87/austin/issues])") # ]]] -AC_INIT([austin], [3.6.0], [https://github.com/p403n1x87/austin/issues]) +AC_INIT([austin], [3.7.0], [https://github.com/p403n1x87/austin/issues]) # [[[end]]] AC_CONFIG_SRCDIR([config.h.in]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/cheatsheet.pdf b/doc/cheatsheet.pdf index e6a912ef..e63be2e9 100644 Binary files a/doc/cheatsheet.pdf and b/doc/cheatsheet.pdf differ diff --git a/doc/cheatsheet.png b/doc/cheatsheet.png index b71def1f..8c1bccd3 100644 Binary files a/doc/cheatsheet.png and b/doc/cheatsheet.png differ diff --git a/doc/cheatsheet.svg b/doc/cheatsheet.svg index 1aa90808..99c7b3d4 100644 --- a/doc/cheatsheet.svg +++ b/doc/cheatsheet.svg @@ -6342,7 +6342,7 @@ y="293.48233" x="49.886612" id="tspan9522" - sodipodi:role="line">3.8 | 3.9 | 3.10 | 3.11 | 3.12 + sodipodi:role="line">3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 for version 3.6 + sodipodi:role="line">for version 3.7 src/austin.1 + +VERSION=$(cat src/austin.h | sed -r -n "s/^#define VERSION[ ]+\"([0-9]+[.][0-9]+).*\"/\1/p") + +# Update the version in the SVG file +if [[ $(uname) == "Darwin" ]]; then + sed -E -i '' "s/for version [0-9]+[.][0-9]+/for version $VERSION/g" "doc/cheatsheet.svg" +else + sed -i "s/for version [0-9]+[.][0-9]+/for version $VERSION/g" "doc/cheatsheet.svg" +fi + +inkscape \ + --export-type="png" \ + --export-filename="doc/cheatsheet.png" \ + --export-dpi=192 \ + doc/cheatsheet.svg + +inkscape \ + --export-type="pdf" \ + --export-filename="doc/cheatsheet.pdf" \ + doc/cheatsheet.svg diff --git a/doc/genman.sh b/doc/genman.sh deleted file mode 100755 index c6b9a8fd..00000000 --- a/doc/genman.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -help2man \ - -n "Frame stack sampler for CPython" \ - -i doc/examples.troff \ - src/austin > src/austin.1 diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 311e2584..490cd468 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -14,7 +14,7 @@ from common import download_release from scipy.stats import ttest_ind -VERSIONS = ("3.4.1", "3.5.0", "dev") +VERSIONS = ("3.5.0", "3.6.0", "dev") SCENARIOS = [ *[ ( diff --git a/scripts/build-wheel.py b/scripts/build-wheel.py index 0318660f..788e3d7e 100644 --- a/scripts/build-wheel.py +++ b/scripts/build-wheel.py @@ -21,6 +21,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], "Project-URL": [ "Homepage, https://github.com/P403n1x87/austin", diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b25f684b..c9c51617 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -4,7 +4,7 @@ base: core20 # from scripts.utils import get_current_version_from_changelog as version # print(f"version: '{version()}+git'") # ]]] -version: '3.6.0+git' +version: '3.7.0+git' # [[[end]]] summary: A Python frame stack sampler for CPython description: | diff --git a/src/argparse.c b/src/argparse.c index 2b28f952..36832171 100644 --- a/src/argparse.c +++ b/src/argparse.c @@ -70,7 +70,6 @@ parsed_args_t pargs = { /* timeout */ DEFAULT_INIT_TIMEOUT_MS * 1000, /* attach_pid */ 0, /* where */ 0, - /* exclude_empty */ 0, /* sleepless */ 0, /* format */ (char *) SAMPLE_FORMAT_NORMAL, #ifdef NATIVE @@ -112,7 +111,7 @@ str_to_num(char * str, long * num) { /** * Parse the interval argument. * - * This acceps s, ms and us as units. The result is in microseconds. + * This accepts s, ms and us as units. The result is in microseconds. */ static int parse_interval(char * str, long * num) { @@ -150,7 +149,7 @@ parse_interval(char * str, long * num) { /** * Parse the timeout argument. * - * This acceps s and ms as units. The result is in milliseconds. + * This accepts s and ms as units. The result is in milliseconds. */ static int parse_timeout(char * str, long * num) { @@ -219,10 +218,6 @@ static struct argp_option options[] = { "timeout", 't', "n_ms", 0, "Start up wait time in milliseconds (default is 100). Accepted units: s, ms." }, - { - "exclude-empty",'e', NULL, 0, - "Do not output samples of threads with no frame stacks." - }, { "sleepless", 's', NULL, 0, "Suppress idle samples to estimate CPU time." @@ -339,10 +334,6 @@ parse_opt (int key, char *arg, struct argp_state *state) pargs.binary = 1; break; - case 'e': - pargs.exclude_empty = 1; - break; - case 's': pargs.sleepless = 1; break; @@ -566,8 +557,6 @@ print(";") " https://github.com/P403n1x87/austin/wiki/The-MOJO-file-format\n" " for more details.\n" " -C, --children Attach to child processes.\n" -" -e, --exclude-empty Do not output samples of threads with no frame\n" -" stacks.\n" " -f, --full Produce the full set of metrics (time +mem -mem).\n" " -g, --gc Sample the garbage collector state.\n" " -h, --heap=n_mb Maximum heap size to allocate to increase sampling\n" @@ -602,12 +591,11 @@ for line in check_output(["src/austin", "--usage"]).decode().strip().splitlines( print(f'"{line}\\n"') print(";") ]]]*/ -"Usage: austin [-bCefgmPs?V] [-h n_mb] [-i n_us] [-o FILE] [-p PID] [-t n_ms]\n" -" [-w PID] [-x n_sec] [--binary] [--children] [--exclude-empty]\n" -" [--full] [--gc] [--heap=n_mb] [--interval=n_us] [--memory]\n" -" [--output=FILE] [--pid=PID] [--pipe] [--sleepless] [--timeout=n_ms]\n" -" [--where=PID] [--exposure=n_sec] [--help] [--usage] [--version]\n" -" command [ARG...]\n" +"Usage: austin [-bCfgmPs?V] [-h n_mb] [-i n_us] [-o FILE] [-p PID] [-t n_ms]\n" +" [-w PID] [-x n_sec] [--binary] [--children] [--full] [--gc]\n" +" [--heap=n_mb] [--interval=n_us] [--memory] [--output=FILE]\n" +" [--pid=PID] [--pipe] [--sleepless] [--timeout=n_ms] [--where=PID]\n" +" [--exposure=n_sec] [--help] [--usage] [--version] command [ARG...]\n" ; /*[[[end]]]*/ @@ -686,10 +674,6 @@ cb(const char opt, const char * arg) { pargs.binary = 1; break; - case 'e': - pargs.exclude_empty = 1; - break; - case 's': pargs.sleepless = 1; break; diff --git a/src/argparse.h b/src/argparse.h index cfa791c4..4beb8474 100644 --- a/src/argparse.h +++ b/src/argparse.h @@ -35,7 +35,6 @@ typedef struct { ctime_t timeout; pid_t attach_pid; int where; - int exclude_empty; int sleepless; char * format; #ifdef NATIVE diff --git a/src/austin.1 b/src/austin.1 index 12941579..9882016e 100644 --- a/src/austin.1 +++ b/src/austin.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.13. -.TH AUSTIN "1" "October 2023" "austin 3.6.0" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. +.TH AUSTIN "1" "October 2024" "austin 3.7.0" "User Commands" .SH NAME austin \- Frame stack sampler for CPython .SH SYNOPSIS @@ -18,10 +18,6 @@ for more details. \fB\-C\fR, \fB\-\-children\fR Attach to child processes. .TP -\fB\-e\fR, \fB\-\-exclude\-empty\fR -Do not output samples of threads with no frame -stacks. -.TP \fB\-f\fR, \fB\-\-full\fR Produce the full set of metrics (time +mem \fB\-mem\fR). .TP diff --git a/src/austin.c b/src/austin.c index 62ea9b59..db37aa44 100644 --- a/src/austin.c +++ b/src/austin.c @@ -293,7 +293,7 @@ int main(int argc, char ** argv) { logger_init(); if (!pargs.pipe) - log_header(); + log_header(); // cppcheck-suppress [unknownMacro] if (exec_arg <= 0 && pargs.attach_pid == 0) { _msg(MCMDLINE); diff --git a/src/austin.h b/src/austin.h index bfd860e9..29d0b3ba 100644 --- a/src/austin.h +++ b/src/austin.h @@ -34,7 +34,7 @@ from scripts.utils import get_current_version_from_changelog as version print(f'#define VERSION "{version()}"') ]]] */ -#define VERSION "3.6.0" +#define VERSION "3.7.0" // [[[end]]] #endif diff --git a/src/frame.h b/src/frame.h index c92ad212..fe82c237 100644 --- a/src/frame.h +++ b/src/frame.h @@ -168,7 +168,7 @@ _frame_from_code_raddr(py_proc_t * py_proc, void * code_raddr, int lasti, python ssize_t len = 0; unsigned int lineno = V_FIELD(unsigned int, code, py_code, o_firstlineno); - unsigned int line_end = lineno; + unsigned int line_end = 0; unsigned int column = 0; unsigned int column_end = 0; diff --git a/src/linux/common.h b/src/linux/common.h index a97cd541..8c9ecb54 100644 --- a/src/linux/common.h +++ b/src/linux/common.h @@ -26,9 +26,11 @@ #include #include #include +#include #include #include "../error.h" +#include "../hints.h" #include "../stats.h" @@ -109,3 +111,24 @@ _procfs(pid_t pid, char * file) { return fp; } + + +// ---------------------------------------------------------------------------- +static inline char * +proc_root(pid_t pid, char * file) { + if (file[0] != '/') { + log_e("File path is not absolute"); // GCOV_EXCL_START + return NULL; // GCOV_EXCL_STOP + } + + char * proc_root = calloc(1, strlen(file) + 24); + if (!isvalid(proc_root)) + return NULL; // GCOV_EXCL_LINE + + if (sprintf(proc_root, "/proc/%d/root%s", pid, file) < 0) { + free(proc_root); // GCOV_EXCL_START + return NULL; // GCOV_EXCL_STOP + } + + return proc_root; +} diff --git a/src/linux/py_proc.h b/src/linux/py_proc.h index c5ea90cc..852d8d00 100644 --- a/src/linux/py_proc.h +++ b/src/linux/py_proc.h @@ -455,16 +455,16 @@ _py_proc__parse_maps_file(py_proc_t * self) { // The first memory map of the executable if (!isvalid(pd->maps[MAP_BIN].path) && strcmp(pd->exe_path, pathname) == 0) { map = &(pd->maps[MAP_BIN]); - map->path = strndup(pathname, strlen(pathname)); + map->path = proc_root(self->pid, pathname); if (!isvalid(map->path)) { - log_ie("Cannot duplicate path name"); + log_e("Cannot get proc root path for %s", pathname); // GCOV_EXCL_START set_error(EPROC); - FAIL; + FAIL; // GCOV_EXCL_STOP } - map->file_size = _file_size(pathname); + map->file_size = _file_size(map->path); map->base = (void *) lower; map->size = upper - lower; - map->has_symbols = success(_py_proc__analyze_elf(self, pathname, (void *) lower)); + map->has_symbols = success(_py_proc__analyze_elf(self, map->path, (void *) lower)); if (map->has_symbols) { map->bss_base = self->map.bss.base; map->bss_size = self->map.bss.size; @@ -479,13 +479,13 @@ _py_proc__parse_maps_file(py_proc_t * self) { int has_symbols = success(_py_proc__analyze_elf(self, pathname, (void *) lower)); if (has_symbols) { map = &(pd->maps[MAP_LIBSYM]); - map->path = strndup(pathname, strlen(pathname)); + map->path = proc_root(self->pid, pathname); if (!isvalid(map->path)) { - log_ie("Cannot duplicate path name"); + log_e("Cannot get proc root path for %s", pathname); // GCOV_EXCL_START set_error(EPROC); - FAIL; + FAIL; // GCOV_EXCL_STOP } - map->file_size = _file_size(pathname); + map->file_size = _file_size(map->path); map->base = (void *) lower; map->size = upper - lower; map->has_symbols = TRUE; @@ -503,13 +503,13 @@ _py_proc__parse_maps_file(py_proc_t * self) { unsigned int v; if (sscanf(needle, "libpython%u.%u", &v, &v) == 2) { map = &(pd->maps[MAP_LIBNEEDLE]); - map->path = needle_path = strndup(pathname, strlen(pathname)); + map->path = needle_path = proc_root(self->pid, pathname); if (!isvalid(map->path)) { - log_ie("Cannot duplicate path name"); + log_e("Cannot get proc root path for %s", pathname); // GCOV_EXCL_START set_error(EPROC); - FAIL; + FAIL; // GCOV_EXCL_STOP } - map->file_size = _file_size(pathname); + map->file_size = _file_size(map->path); map->base = (void *) lower; map->size = upper - lower; map->has_symbols = FALSE; diff --git a/src/py_proc.c b/src/py_proc.c index 33141677..559b111b 100644 --- a/src/py_proc.c +++ b/src/py_proc.c @@ -110,17 +110,14 @@ _get_version_from_executable(char * binary, int * major, int * minor, int * patc #endif fp = _popen(cmd, "r"); - if (!isvalid(fp)) { - set_error(EPROC); + if (!isvalid(fp)) FAIL; - } while (fgets(version, sizeof(version) - 1, fp) != NULL) { if (sscanf(version, "Python %d.%d.%d", major, minor, patch) == 3) SUCCESS; } - set_error(EPROC); FAIL; } /* _get_version_from_executable */ @@ -129,13 +126,15 @@ _get_version_from_filename(char * filename, const char * needle, int * major, in #if defined PL_LINUX /* LINUX */ char * base = filename; char * end = base + strlen(base); + size_t needle_len = strlen(needle); while (base < end) { base = strstr(base, needle); if (!isvalid(base)) { break; } - if (sscanf(base + strlen(needle), "%u.%u", major, minor) == 2) { + base += needle_len; + if (sscanf(base, "%u.%u", major, minor) == 2) { SUCCESS; } } @@ -143,23 +142,18 @@ _get_version_from_filename(char * filename, const char * needle, int * major, in #elif defined PL_WIN /* WIN */ // Assume the library path is of the form *.python3[0-9]+[.]dll int n = strlen(filename); - if (n < 10) { - set_error(EPROC); + if (n < 10) FAIL; - } char * p = filename + n - 1; while (*(p--) != 'n' && p > filename); p++; *major = *(p++) - '0'; - if (*major != 3) { - set_error(EPROC); + if (*major != 3) FAIL; - } - if (sscanf(p,"%d.dll", minor) == 1) { + if (sscanf(p,"%d.dll", minor) == 1) SUCCESS; - } #elif defined PL_MACOS /* MAC */ char * ver_needle = strstr(filename, "3."); @@ -169,7 +163,6 @@ _get_version_from_filename(char * filename, const char * needle, int * major, in #endif - set_error(EPROC); FAIL; } /* _get_version_from_filename */ @@ -242,6 +235,30 @@ _py_proc__infer_python_version(py_proc_t * self) { int major = 0, minor = 0, patch = 0; + // Starting with Python 3.13 we can use the PyRuntime structure + if (isvalid(self->symbols[DYNSYM_RUNTIME])) { + _Py_DebugOffsets py_d; + if (fail(py_proc__get_type(self, self->symbols[DYNSYM_RUNTIME], py_d))) { + log_e("Cannot copy PyRuntimeState structure from remote address"); + FAIL; + } + + if (0 == memcmp(py_d.v3_13.cookie, _Py_Debug_Cookie, sizeof(py_d.v3_13.cookie))) { + uint64_t version = py_d.v3_13.version; + major = (version>>24) & 0xFF; + minor = (version>>16) & 0xFF; + patch = (version>>8) & 0xFF; + + log_d("Python version (from debug offsets): %d.%d.%d", major, minor, patch); + + self->py_v = get_version_descriptor(major, minor, patch); + + init_version_descriptor(self->py_v, &py_d); + + SUCCESS; + } + } + // Starting with Python 3.11 we can rely on the Py_Version symbol if (isvalid(self->symbols[DYNSYM_HEX_VERSION])) { unsigned long py_version = 0; @@ -329,17 +346,14 @@ _py_proc__check_interp_state(py_proc_t * self, void * raddr) { V_DESC(self->py_v); - PyInterpreterState is; - PyThreadState tstate_head; - - if (py_proc__get_type(self, raddr, is)) { + if (py_proc__copy_v(self, is, raddr, self->is)) { log_ie("Cannot get remote interpreter state"); FAIL; } + log_d("Interpreter state buffer %p", self->is); + void * tstate_head_addr = V_FIELD_PTR(void *, self->is, py_is, o_tstate_head); - void * tstate_head_addr = V_FIELD(void *, is, py_is, o_tstate_head); - - if (fail(py_proc__get_type(self, tstate_head_addr, tstate_head))) { + if (fail(py_proc__copy_v(self, thread, tstate_head_addr, self->ts))) { log_e( "Cannot copy PyThreadState head at %p from PyInterpreterState instance", tstate_head_addr @@ -349,7 +363,7 @@ _py_proc__check_interp_state(py_proc_t * self, void * raddr) { log_t("PyThreadState head loaded @ %p", V_FIELD(void *, is, py_is, o_tstate_head)); - if (V_FIELD(void*, tstate_head, py_thread, o_interp) != raddr) { + if (V_FIELD_PTR(void*, self->ts, py_thread, o_interp) != raddr) { log_d("PyThreadState head does not point to interpreter state"); set_error(EPROC); FAIL; @@ -365,7 +379,7 @@ _py_proc__check_interp_state(py_proc_t * self, void * raddr) { raddr, V_FIELD(void *, is, py_is, o_tstate_head) ); - raddr_t thread_raddr = {self->proc_ref, V_FIELD(void *, is, py_is, o_tstate_head)}; + raddr_t thread_raddr = {self->proc_ref, V_FIELD_PTR(void *, self->is, py_is, o_tstate_head)}; py_thread_t thread; if (fail(py_thread__fill_from_raddr(&thread, &thread_raddr, self))) { @@ -496,7 +510,6 @@ _py_proc__deref_interp_head(py_proc_t * self) { void * interp_head_raddr = NULL; - _PyRuntimeState py_runtime; void * runtime_addr = self->symbols[DYNSYM_RUNTIME]; #if defined PL_LINUX const size_t size = getpagesize(); @@ -517,7 +530,7 @@ _py_proc__deref_interp_head(py_proc_t * self) { #endif for (void * current_addr = lower; current_addr <= upper; current_addr += sizeof(void *)) { - if (py_proc__get_type(self, current_addr, py_runtime)) { + if (py_proc__copy_v(self, runtime, current_addr, self->rs)) { log_d( "Cannot copy runtime state structure from remote address %p", current_addr @@ -525,7 +538,7 @@ _py_proc__deref_interp_head(py_proc_t * self) { continue; } - interp_head_raddr = V_FIELD(void *, py_runtime, py_runtime, o_interp_head); + interp_head_raddr = V_FIELD_PTR(void *, self->rs, py_runtime, o_interp_head); if (V_MAX(3, 8)) { self->gc_state_raddr = current_addr + py_v->py_runtime.o_gc; log_d("GC runtime state @ %p", self->gc_state_raddr); @@ -567,6 +580,46 @@ _py_proc__get_current_thread_state_raddr(py_proc_t * self) { return (void *) -1; } +// ---------------------------------------------------------------------------- +static void +_py_proc__free_local_buffers(py_proc_t * self) { + sfree(self->is); + sfree(self->ts); + sfree(self->rs); +} + +// ---------------------------------------------------------------------------- +#define LOCAL_ALLOC(dest, src, name) { \ + self->dest = calloc(1, self->py_v->py_##src.size); \ + if (!isvalid(self->dest)) { \ + log_e("Cannot allocate memory for " #name); \ + goto error; \ + } \ +} + +static int +_py_proc__init_local_buffers(py_proc_t * self) { + if (!isvalid(self)) { + set_error(EPROC); + FAIL; + } + + LOCAL_ALLOC(rs, runtime, "PyRuntimeState"); + LOCAL_ALLOC(is, is, "PyInterpreterState"); + LOCAL_ALLOC(ts, thread, "PyThreadState"); + + log_d("Local buffers initialised"); + + SUCCESS; + +error: + set_error(ENOMEM); + + _py_proc__free_local_buffers(self); + + FAIL; +} + // ---------------------------------------------------------------------------- static int _py_proc__find_interpreter_state(py_proc_t * self) { @@ -582,6 +635,9 @@ _py_proc__find_interpreter_state(py_proc_t * self) { if (fail(_py_proc__infer_python_version(self))) FAIL; + if (fail(_py_proc__init_local_buffers(self))) + FAIL; + if (self->sym_loaded || isvalid(self->map.runtime.base)) { // Try to resolve the symbols or the runtime section, if we have them @@ -710,6 +766,11 @@ py_proc_new(int child) { py_proc->child = child; py_proc->gc_state_raddr = NULL; + py_proc->py_v = NULL; + + py_proc->is = NULL; + py_proc->ts = NULL; + py_proc->rs = NULL; _prehash_symbols(); @@ -958,12 +1019,11 @@ _py_proc__find_current_thread_offset(py_proc_t * self, void * thread_raddr) { V_DESC(self->py_v); void * interp_head_raddr; - _PyRuntimeState py_runtime; - if (py_proc__get_type(self, self->symbols[DYNSYM_RUNTIME], py_runtime)) + if (py_proc__copy_v(self, runtime, self->symbols[DYNSYM_RUNTIME], self->rs)) FAIL; - interp_head_raddr = V_FIELD(void *, py_runtime, py_runtime, o_interp_head); + interp_head_raddr = V_FIELD_PTR(void *, self->rs, py_runtime, o_interp_head); // Search offset of current thread in _PyRuntimeState structure PyInterpreterState is; @@ -1207,15 +1267,13 @@ py_proc__sample(py_proc_t * self) { V_DESC(self->py_v); - PyInterpreterState is; - do { - if (fail(py_proc__get_type(self, current_interp, is))) { + if (fail(py_proc__copy_v(self, is, current_interp, self->is))) { log_ie("Failed to get interpreter state while sampling"); FAIL; } - void * tstate_head = V_FIELD(void *, is, py_is, o_tstate_head); + void * tstate_head = V_FIELD_PTR(void *, self->is, py_is, o_tstate_head); if (!isvalid(tstate_head)) // Maybe the interpreter state is in an invalid state. We'll try again // unless there is a fatal error. @@ -1230,7 +1288,7 @@ py_proc__sample(py_proc_t * self) { time_delta = gettime() - self->timestamp; #endif - int result = _py_proc__sample_interpreter(self, &is, time_delta); + int result = _py_proc__sample_interpreter(self, self->is, time_delta); #ifdef NATIVE if (fail(_py_proc__resume_threads(self, &raddr))) { @@ -1241,7 +1299,7 @@ py_proc__sample(py_proc_t * self) { if (fail(result)) FAIL; - } while (isvalid(current_interp = V_FIELD(void *, is, py_is, o_next))); + } while (isvalid(current_interp = V_FIELD_PTR(void *, self->is, py_is, o_next))); #ifdef NATIVE self->timestamp = gettime(); @@ -1328,6 +1386,8 @@ py_proc__destroy(py_proc_t * self) { hash_table__destroy(self->base_table); #endif + _py_proc__free_local_buffers(self); + sfree(self->bin_path); sfree(self->lib_path); sfree(self->extra); diff --git a/src/py_proc.h b/src/py_proc.h index 56afdd75..cf20d2b1 100644 --- a/src/py_proc.h +++ b/src/py_proc.h @@ -99,6 +99,11 @@ typedef struct { hash_table_t * base_table; #endif + // Local buffers + _PyRuntimeState * rs; + PyInterpreterState * is; + PyThreadState * ts; + // Platform-dependent fields proc_extra_info * extra; } py_proc_t; @@ -208,6 +213,19 @@ py_proc__sample(py_proc_t *); */ #define py_proc__get_type(self, raddr, dt) (py_proc__memcpy(self, raddr, sizeof(dt), &dt)) +/** + * Make a local copy of a remote structure. + * + * @param self the process object. + * @param type the type of the structure. + * @param raddr the remote address of the structure. + * @param dest the destination address. + * + * @return 0 on success. + */ +#define py_proc__copy_v(self, type, raddr, dest) (py_proc__memcpy(self, raddr, py_v->py_##type.size, dest)) + + /** * Log the Python interpreter version * @param self the process object. diff --git a/src/py_proc_list.c b/src/py_proc_list.c index 7fc682a7..88fcbb21 100644 --- a/src/py_proc_list.c +++ b/src/py_proc_list.c @@ -176,7 +176,7 @@ py_proc_list__sample(py_proc_list_t * self) { for (py_proc_item_t * item = self->first; item != NULL; /* item = item->next */) { log_t("Sampling process with PID %d", item->py_proc->pid); stopwatch_start(); - if (fail(py_proc__sample(item->py_proc))) { + if (!isvalid(item->py_proc->py_v) || fail(py_proc__sample(item->py_proc))) { py_proc__wait(item->py_proc); py_proc_item_t * next = item->next; _py_proc_list__remove(self, item); diff --git a/src/py_thread.c b/src/py_thread.c index 196e997b..256ffbd2 100644 --- a/src/py_thread.c +++ b/src/py_thread.c @@ -424,7 +424,7 @@ static inline int _py_thread__unwind_iframe_stack(py_thread_t * self, void * iframe_raddr) { int invalid = FALSE; void * curr = iframe_raddr; - + while (isvalid(curr)) { if (fail(_py_thread__push_iframe(self, &curr))) { log_d("Failed to retrieve iframe #%d", stack_pointer()); @@ -457,8 +457,6 @@ static inline int _py_thread__unwind_cframe_stack(py_thread_t * self) { PyCFrame cframe; - int invalid = FALSE; - _py_thread__read_stack(self); stack_reset(); @@ -470,11 +468,7 @@ _py_thread__unwind_cframe_stack(py_thread_t * self) { FAIL; } - invalid = fail(_py_thread__unwind_iframe_stack(self, V_FIELD(void *, cframe, py_cframe, o_current_frame))); - if (invalid) - return invalid; - - return invalid; + return fail(_py_thread__unwind_iframe_stack(self, V_FIELD(void *, cframe, py_cframe, o_current_frame))); } @@ -900,10 +894,6 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t if (self->invalid) return; - if (pargs.exclude_empty && stack_is_empty()) - // Skip if thread has no frames and we want to exclude empty threads - return; - if (mem_delta == 0 && time_delta == 0) return; @@ -954,7 +944,13 @@ py_thread__emit_collapsed_stack(py_thread_t * self, int64_t interp_id, ctime_t t V_DESC(self->proc->py_v); if (isvalid(self->top_frame)) { - if (V_MIN(3, 11)) { + if (V_MIN(3, 13)) { + if (fail(_py_thread__unwind_iframe_stack(self, self->top_frame))) { + emit_invalid_frame(); + error = TRUE; + } + } + else if (V_MIN(3, 11)) { if (fail(_py_thread__unwind_cframe_stack(self))) { emit_invalid_frame(); error = TRUE; diff --git a/src/python/runtime.h b/src/python/runtime.h index 9b4147a1..3f24e8a0 100644 --- a/src/python/runtime.h +++ b/src/python/runtime.h @@ -156,4 +156,151 @@ typedef union { _PyRuntimeState3_12 v3_12; } _PyRuntimeState; +// ---------------------------------------------------------------------------- +// Starting with CPython 3.13, we can retrieve the offsets from a dedicated +// data structure + +#define _Py_Debug_Cookie "xdebugpy" + +typedef struct _Py_DebugOffsets3_13 { + char cookie[8]; // _Py_Debug_Cookie + uint64_t version; + uint64_t free_threaded; + // Runtime state offset; + struct _runtime_state { + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state { + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state{ + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame { + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; + } interpreter_frame; + + // Code object offset; + struct _code_object { + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; + } code_object; + + // PyObject offset; + struct _pyobject { + uint64_t size; + uint64_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + uint64_t size; + uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } tuple_object; + + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t size; + uint64_t state; + uint64_t length; + uint64_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc { + uint64_t size; + uint64_t collecting; + } gc; +} _Py_DebugOffsets3_13; + + +typedef union { + _Py_DebugOffsets3_13 v3_13; +} _Py_DebugOffsets; + + #endif diff --git a/src/python/string.h b/src/python/string.h index 8d768ff9..c1a2c903 100644 --- a/src/python/string.h +++ b/src/python/string.h @@ -120,14 +120,4 @@ typedef struct { char ob_sval[1]; } PyBytesObject; - -// ---- stringobject.h -------------------------------------------------------- - -typedef struct { - PyObject_VAR_HEAD - long ob_shash; - int ob_sstate; - char ob_sval[1]; -} PyStringObject; /* From Python 2.7 */ - #endif diff --git a/src/version.h b/src/version.h index c4153362..d6aefb64 100644 --- a/src/version.h +++ b/src/version.h @@ -140,16 +140,6 @@ typedef struct { } py_thread_v; -typedef struct { - int version; -} py_unicode_v; - - -typedef struct { - int version; -} py_bytes_v; - - typedef struct { ssize_t size; @@ -280,14 +270,6 @@ typedef struct { offsetof(s, _status) \ } -#define PY_UNICODE(n) { \ - n \ -} - -#define PY_BYTES(n) { \ - n \ -} - #define PY_RUNTIME(s) { \ sizeof(s), \ offsetof(s, interpreters.head), \ @@ -390,6 +372,10 @@ python_v python_v3_12 = { PY_IFRAME_312 (_PyInterpreterFrame3_12), }; +// ---- Python 3.13 ----------------------------------------------------------- + +python_v python_v3_13; + // ---------------------------------------------------------------------------- static inline python_v * get_version_descriptor(int major, int minor, int patch) { @@ -428,6 +414,9 @@ get_version_descriptor(int major, int minor, int patch) { // 3.12 case 12: py_v = &python_v3_12; break; + // 3.13+ + case 13: py_v = &python_v3_13; break; + default: py_v = LATEST_VERSION; UNSUPPORTED_VERSION; } @@ -440,6 +429,74 @@ get_version_descriptor(int major, int minor, int patch) { return py_v; } +// ---------------------------------------------------------------------------- + +#define V_ASSIGN(ver, dst, src) {py_v->py_##dst = py_d->v##ver.src;log_d("py_%s = %ld", #dst, py_v->py_##dst);} + +#define PY_CODE_313(v) { \ + V_ASSIGN(v, code.size, code_object.size) \ + V_ASSIGN(v, code.o_filename, code_object.filename) \ + V_ASSIGN(v, code.o_name, code_object.name) \ + V_ASSIGN(v, code.o_lnotab, code_object.linetable) \ + V_ASSIGN(v, code.o_firstlineno, code_object.firstlineno) \ + V_ASSIGN(v, code.o_code, code_object.co_code_adaptive) \ + V_ASSIGN(v, code.o_qualname, code_object.qualname) \ +} + +#define PY_IFRAME_313(v) { \ + V_ASSIGN(v, iframe.size, interpreter_frame.size) \ + V_ASSIGN(v, iframe.o_previous, interpreter_frame.previous) \ + V_ASSIGN(v, iframe.o_code, interpreter_frame.executable) \ + V_ASSIGN(v, iframe.o_prev_instr, interpreter_frame.instr_ptr) \ + V_ASSIGN(v, iframe.o_owner, interpreter_frame.owner) \ +} + +#define PY_THREAD_313(v) { \ + V_ASSIGN(v, thread.size, thread_state.size) \ + V_ASSIGN(v, thread.o_prev, thread_state.prev) \ + V_ASSIGN(v, thread.o_next, thread_state.next) \ + V_ASSIGN(v, thread.o_interp, thread_state.interp) \ + V_ASSIGN(v, thread.o_frame, thread_state.current_frame) \ + V_ASSIGN(v, thread.o_thread_id, thread_state.thread_id) \ + V_ASSIGN(v, thread.o_native_thread_id, thread_state.native_thread_id) \ + V_ASSIGN(v, thread.o_stack, thread_state.datastack_chunk) \ + V_ASSIGN(v, thread.o_status, thread_state.status) \ +} + +#define PY_RUNTIME_313(v) { \ + V_ASSIGN(v, runtime.size, runtime_state.size) \ + V_ASSIGN(v, runtime.o_interp_head, runtime_state.interpreters_head) \ +} + +#define PY_IS_313(v) { \ + V_ASSIGN(v, is.size, interpreter_state.size) \ + V_ASSIGN(v, is.o_next, interpreter_state.next) \ + V_ASSIGN(v, is.o_tstate_head, interpreter_state.threads_head) \ + V_ASSIGN(v, is.o_id, interpreter_state.id) \ + V_ASSIGN(v, is.o_gc, interpreter_state.gc) \ + V_ASSIGN(v, is.o_gil_state, interpreter_state.ceval_gil) \ +} + +#define PY_GC_313(v) { \ + V_ASSIGN(v, gc.size, gc.size) \ + V_ASSIGN(v, gc.o_collecting, gc.collecting) \ +} + +// ---------------------------------------------------------------------------- +static void +init_version_descriptor(python_v * py_v, _Py_DebugOffsets* py_d) { + switch (py_v->minor) { + case 13: + PY_CODE_313(3_13); + PY_IFRAME_313(3_13); + PY_THREAD_313(3_13); + PY_RUNTIME_313(3_13); + PY_IS_313(3_13); + PY_GC_313(3_13); + } +} + + #endif // PY_PROC_C #endif diff --git a/test/__init__.py b/test/__init__.py index adea0185..abe0312e 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,7 +1,8 @@ import os import platform -PY3_LATEST = 12 +PY3_EARLIEST = 8 +PY3_LATEST = 13 try: REQUESTED_PYTHON_VERSIONS = [ @@ -15,9 +16,9 @@ match platform.system(): case "Darwin": PYTHON_VERSIONS = REQUESTED_PYTHON_VERSIONS or [ - (3, _) for _ in range(8, PY3_LATEST + 1) + (3, _) for _ in range(PY3_EARLIEST, PY3_LATEST + 1) ] case _: PYTHON_VERSIONS = REQUESTED_PYTHON_VERSIONS or [ - (3, _) for _ in range(8, PY3_LATEST + 1) + (3, _) for _ in range(PY3_EARLIEST, PY3_LATEST + 1) ] diff --git a/test/cunit/__init__.py b/test/cunit/__init__.py index 10d18e97..45752a25 100644 --- a/test/cunit/__init__.py +++ b/test/cunit/__init__.py @@ -254,9 +254,14 @@ def visit_Decl(self, node: c_ast.Node) -> None: args = ( [ - (_.name, c_void_p if isinstance(_.type, c_ast.PtrDecl) else c_long) - if hasattr(_, "name") - else None + ( + ( + _.name, + c_void_p if isinstance(_.type, c_ast.PtrDecl) else c_long, + ) + if hasattr(_, "name") + else None + ) for _ in node.type.args.params ] if node.type.args is not None diff --git a/test/functional/test_mojo.py b/test/functional/test_mojo.py index b2720b8f..02eaeb7e 100644 --- a/test/functional/test_mojo.py +++ b/test/functional/test_mojo.py @@ -21,13 +21,9 @@ # along with this program. If not, see . from pathlib import Path -from test.utils import allpythons -from test.utils import austin -from test.utils import python -from test.utils import target +from test.utils import allpythons, austin, python, target -from austin.format.mojo import MojoFile -from austin.format.mojo import MojoFrame +from austin.format.mojo import MojoFile, MojoFrame @allpythons(min=(3, 11)) @@ -56,3 +52,27 @@ def strip(f): ("lazy", 6, 6, 9, 19), ("", 17, 17, 5, 17), } + + +@allpythons(max=(3, 10)) +def test_mojo_no_column_data(py, tmp_path: Path): + """ + Test that no other location information is present apart from the line + number for Python versions prior to 3.11. + """ + datafile = tmp_path / "test_mojo_column.austin" + + result = austin( + "-i", "100", "-o", str(datafile), *python(py), target("column.py"), mojo=True + ) + assert result.returncode == 0, result.stderr or result.stdout + + def strip(f): + return + + with datafile.open("rb") as f: + assert { + (e.line_end, e.column, e.column_end) + for e in MojoFile(f).parse() + if isinstance(e, MojoFrame) + } == {(0, 0, 0)}