Merge pull request #27 from parca-dev/new_build_toolchain

Implement a new toolchain

.dockerignore

@@ -0,0 +1,6 @@
+out
+src
+tables
+vendored
+jitdump
+.devbox

.envrc

@@ -0,0 +1,7 @@
+# Automatically sets up your devbox environment whenever you cd into this
+# directory via our direnv integration:
+
+eval "$(devbox generate direnv --print-envrc)"
+
+# check out https://www.jetpack.io/devbox/docs/ide_configuration/direnv/
+# for more details

.gitattributes

@@ -0,0 +1,3 @@
+vendored/ linguist-vendored
+out/ linguist-generated
+tables/ linguist-generated

.github/workflows/ci.yml

@@ -0,0 +1,16 @@
+name: CI
+
+on: push
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: run
+        uses: dagger/dagger-for-github@v5
+        with:
+          verb: call
+          module: ci
+          args: run --src "."

.gitignore

@@ -1,2 +1 @@
-out/basic-cpp-jit
-out/basic-cpp-jit-no-fp
+.devbox

Dockerfile

@@ -0,0 +1,12 @@
+FROM jetpackio/devbox:latest
+
+# Installing your devbox project.
+WORKDIR /code
+USER ${DEVBOX_USER}:${DEVBOX_USER}
+
+COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.json devbox.json
+COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.lock devbox.lock
+
+RUN devbox run -- echo "Installed Packages."
+
+CMD ["devbox", "shell"]

Makefile

@@ -1,107 +1,61 @@
-EH_FRAME_BIN = ../dist/eh-frame
-
-all: lint build
-
-lint:
-	clang-format -i src/*.cpp
-	go fmt src/*.go
-
-build:
-	gcc src/basic-cpp.cpp -o out/x86/basic-cpp -g
-	gcc src/basic-cpp-plt.cpp -o out/x86/basic-cpp-plt
-	gcc src/basic-cpp.cpp -o out/x86/basic-cpp-no-fp -fomit-frame-pointer
-	gcc src/basic-cpp.cpp -o out/x86/basic-cpp-no-fp-with-debuginfo -fomit-frame-pointer -g
-	gcc src/basic-cpp-plt.cpp -o out/x86/basic-cpp-plt-pie -pie -fPIE
-	gcc src/basic-cpp-plt.cpp -o out/x86/basic-cpp-plt-hardened -pie -fPIE -fstack-protector-all -D_FORTIFY_SOURCE=2 -Wl,-z,now -Wl,-z,relro -O2
-	gcc src/basic-cpp-multithreaded-no-fp.cpp -o out/x86/basic-cpp-multithreaded-no-fp -fomit-frame-pointer
-	# The JIT code has frame pointers.
-	gcc src/basic-cpp-jit.cpp -o out/x86/basic-cpp-jit -g
-	gcc src/basic-cpp-jit.cpp -o out/x86/basic-cpp-jit-no-fp -fomit-frame-pointer -g
-	# Go code.
-	go build -o out/x86/basic-go src/main.go
-
-	gcc src/basic-cpp.cpp -fno-PIE -no-pie -o out/arm64/basic-cpp -g
-	gcc src/basic-cpp-plt.cpp -fno-PIE -no-pie -o out/arm64/basic-cpp-plt
-	gcc src/basic-cpp.cpp -fno-PIE -no-pie -o out/arm64/basic-cpp-no-fp -fomit-frame-pointer
-	gcc src/basic-cpp.cpp -fno-PIE -no-pie -o out/arm64/basic-cpp-no-fp-with-debuginfo -fomit-frame-pointer -g
-	gcc src/basic-cpp-plt.cpp -o out/arm64/basic-cpp-plt-pie -pie -fPIE
-	gcc src/basic-cpp-plt.cpp -o out/arm64/basic-cpp-plt-hardened -pie -fPIE -fstack-protector-all -D_FORTIFY_SOURCE=2 -Wl,-z,now -Wl,-z,relro -O2
-
-	# The JIT code has frame pointers but isn't for Arm64
-	# TODO(sylfrena): Port JIT toy binaries for arm64
-	# gcc src/basic-cpp-jit.cpp -o out/arm64/basic-cpp-jit -g
-	# gcc src/basic-cpp-jit.cpp -o out/arm64/basic-cpp-jit-no-fp -fomit-frame-pointer -g
-
-	# Go code.
-	go build -o out/arm64/basic-go src/main.go
-
-validate:
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp > tables/x86/ours_basic-cpp.txt
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp-plt > tables/x86/ours_basic-cpp-plt.txt
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp-no-fp > tables/x86/ours_basic-cpp-no-fp.txt
-
-	$(EH_FRAME_BIN) --executable vendored/x86/libc.so.6 > tables/x86/ours_libc_so_6.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/libpython3.10.so.1.0 > tables/x86/ours_libpython3.10.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/systemd > tables/x86/ours_systemd.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/parca-agent > tables/x86/ours_parca-agent.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/ruby > tables/x86/ours_ruby.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/libruby > tables/x86/ours_libruby.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/redpanda > tables/x86/ours_redpanda.txt
-
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp > tables/arm64/ours_basic-cpp.txt
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp-plt > tables/arm64/ours_basic-cpp-plt.txt
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp-no-fp > tables/arm64/ours_basic-cpp-no-fp.txt
-
-	# $(EH_FRAME_BIN) --executable vendored/arm64/libc.so.6 > tables/arm64/ours_libc_so_6.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/libpython3.10.so.1.0 > tables/arm64/ours_libpython3.10.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/systemd > tables/arm64/ours_systemd.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/parca-agent > tables/arm64/ours_parca-agent.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/ruby > tables/arm64/ours_ruby.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/libruby > tables/arm64/ours_libruby.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/redpanda > tables/arm64/ours_redpanda.txt
-
-
-validate-final:
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp --final > final_tables/x86/ours_basic-cpp.txt
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp-plt --final > final_tables/x86/ours_basic-cpp-plt.txt
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp-no-fp --final > final_tables/x86/ours_basic-cpp-no-fp.txt
-
-	$(EH_FRAME_BIN) --executable vendored/x86/libc.so.6 --final > final_tables/x86/ours_libc_so_6.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/libpython3.10.so.1.0 --compact > final_tables/x86/ours_libpython3.10.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/systemd --final  > final_tables/x86/ours_systemd.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/parca-agent --final > final_tables/x86/ours_parca-agent.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/ruby --final  > final_tables/x86/ours_ruby.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/libruby --final  > final_tables/x86/ours_libruby.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/redpanda --final > final_tables/x86/ours_redpanda.txt
-
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp --final > final_tables/arm64/ours_basic-cpp.txt
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp-plt --final > final_tables/arm64/ours_basic-cpp-plt.txt
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp-no-fp --final > final_tables/arm64/ours_basic-cpp-no-fp.txt
-	$(EH_FRAME_BIN) --executable vendored/arm64/libc.so.6 --final > final_tables/arm64/ours_libc_so_6.txt
-
-validate-compact:
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp --compact > compact_tables/x86/ours_basic-cpp.txt
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp-plt --compact > compact_tables/x86/ours_basic-cpp-plt.txt
-	$(EH_FRAME_BIN) --executable out/x86/basic-cpp-no-fp --compact > compact_tables/x86/ours_basic-cpp-no-fp.txt
-
-	$(EH_FRAME_BIN) --executable vendored/x86/libc.so.6 --compact > compact_tables/x86/ours_libc_so_6.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/libpython3.10.so.1.0 --compact > compact_tables/x86/ours_libpython3.10.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/systemd --compact  > compact_tables/x86/ours_systemd.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/parca-agent --compact > compact_tables/x86/ours_parca-agent.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/ruby --compact  > compact_tables/x86/ours_ruby.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/libruby --compact  > compact_tables/x86/ours_libruby.txt
-	$(EH_FRAME_BIN) --executable vendored/x86/redpanda --compact > compact_tables/x86/ours_redpanda.txt
-
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp --compact > compact_tables/arm64/ours_basic-cpp.txt
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp-plt --compact > compact_tables/arm64/ours_basic-cpp-plt.txt
-	$(EH_FRAME_BIN) --executable out/arm64/basic-cpp-no-fp --compact > compact_tables/arm64/ours_basic-cpp-no-fp.txt
-	$(EH_FRAME_BIN) --executable vendored/arm64/libc.so.6 --compact > compact_tables/arm64/ours_libc_so_6.txt
-
-	# $(EH_FRAME_BIN) --executable vendored/arm64/libpython3.10.so.1.0 --compact > compact_tables/arm64/ours_libpython3.10.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/systemd --compact  > compact_tables/arm64/ours_systemd.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/parca-agent --compact > compact_tables/arm64/ours_parca-agent.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/ruby --compact  > compact_tables/arm64/ours_ruby.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/libruby --compact  > compact_tables/arm64/ours_libruby.txt
-	# $(EH_FRAME_BIN) --executable vendored/arm64/redpanda --compact > compact_tables/arm64/ours_redpanda.txt
-
-all: validate
+# This file is just a porcelaine for devbox commands.
+# Devbox is a tool to manage development environment dependencies.
+# See https://www.jetpack.io/devbox for more information.
+
+# You can install devbox with the following command:
+# make bootstrap
+# It also installs direnv to automatically load devbox environment.
+
+# If you don't want to use devbox, you can use a containerized version of devbox
+# with all the necessary dependencies. To do so, you need to install docker (or podman, etc)
+# and run the targets with DOCKER=1 prefix. For example:
+# make DOCKER=1 build
+
+bootstrap:
+	curl -fsSL https://get.jetpack.io/devbox | bash
+	curl -sfL https://direnv.net/install.sh | bash
+
+.PHONY: format
+ifndef DOCKER
+format: check_devbox
+	devbox run format
+else
+format: $(DOCKER_BUILDER)
+	$(call docker_builder_devbox,run format)
+endif
+
+.PHONY: build
+ifndef DOCKER
+build: check_devbox
+	devbox run build
+else
+build: $(DOCKER_BUILDER)
+	$(call docker_builder_devbox,run build)
+endif
+
+.PHONY: generate
+ifndef DOCKER
+generate: check_devbox
+	devbox run generate
+else
+generate: $(DOCKER_BUILDER)
+	$(call docker_builder_devbox,run generate)
+endif
+
+check_%:
+	@command -v $* >/dev/null || (echo "missing required tool $*" ; false)
+
+CMD_DOCKER ?= docker
+DOCKER_BUILDER ?= parca-dev/testdata-builder
+
+.PHONY: $(DOCKER_BUILDER)
+$(DOCKER_BUILDER): Dockerfile | check_$(CMD_DOCKER)
+	$(CMD_DOCKER) build -t $(DOCKER_BUILDER):latest .
+
+define docker_builder_devbox
+	$(CMD_DOCKER) run --rm \
+	-w /code \
+	-v $(PWD):/code \
+	--entrypoint devbox \
+	$(DOCKER_BUILDER) $(1)
+endef

README.md

@@ -1,14 +1,26 @@
+[![Apache 2 License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)
+[![Built with Devbox](https://jetpack.io/img/devbox/shield_moon.svg)](https://jetpack.io/devbox/docs/contributor-quickstart/)
+[![Built with Mage](https://magefile.org/badge.svg)](https://magefile.org)
+![CI](https://github.com/parca-dev/testdata/workflows/ci/badge.svg?branch=main)
+![CI](https://github.com/parca-dev/testdata/actions/workflows/ci.yml/badge.svg)
+
 This repo has some small custom binaries and vendored executables we use for testing some of Parca Agent's features. This lives in a separate repository as we we wanted to commit executables and unwind tables that can be very large.
 
 ## Structure
 
-- `src/` contains the source for our custom programs
-- `out/` has the produced executables for our custom programs
-- `vendored/` hosts prebuilt executables
-- `tables/` contains a textual representation of the unwind tables. the names follow the following format `"<producer>_<the executable name>.txt"`. `<producer>` is the name of the tool that produced the table, "ours" for our implementation 
-- `compact_tables/` as above, but using the compact unwind format we use in BPF
-- `final_tables/` similar to compact tables, but after any processing. These are the ones that would be loaded in the BPF maps
+- `tables/default` contains a textual representation of the unwind tables. the names follow the following format `"<producer>_<the executable name>.txt"`. `<producer>` is the name of the tool that produced the table, "ours" for our implementation.
+- `tables/compact` as above, but using the compact unwind format we use in BPF.
+- `tables/tables` similar to compact tables, but after any processing. These are the ones that would be loaded in the BPF maps.
+- `jitdump` contains the jitdump files for certain executables. These are used to test the JIT dump parser.
+- `out/` has the produced executables for our custom programs.
+- `src/` contains the source for our custom programs.
+- `vendored/` hosts prebuilt executables.
+
+
+## Building
+
+To build the executables, run `mage build:all` or `make build`. This will build the executables and place them in `out/`.
+
+## Adding new tables
 
-## TODO
-- Improve Makefile
-- Don't use the system compiler / tools
+To add a new table, run `mage generate:all` or `make generate`. This will build the tables and place them in `tables/default/`,`tables/compact/` and `final_tables/`.