diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml new file mode 100644 index 0000000..3bd513d --- /dev/null +++ b/.github/workflows/build-package.yml @@ -0,0 +1,132 @@ +name: build package + +on: + pull_request: + branches: + - main + workflow_dispatch: + inputs: + openwrt_version: + description: "OpenWrt version" + type: string + required: true + default: "23.05.3" + openwrt_arch: + description: "OpenWrt arch" + type: string + required: true + default: "mips_24kc" + openwrt_target: + description: "OpenWrt target" + type: string + required: true + default: "ath79" + openwrt_subtarget: + description: "OpenWrt subtarget" + type: string + required: true + default: "generic" + openwrt_vermagic: + description: "OpenWrt vermagic" + type: string + required: true + default: "auto" + +jobs: + build: + name: "v${{ matrix.build_env.tag }} - ${{ matrix.build_env.pkgarch }} :: ${{ matrix.build_env.target }}/${{ matrix.build_env.subtarget }} openwrt build" + runs-on: ubuntu-22.04 + strategy: + matrix: + build_env: + - tag: ${{ inputs.openwrt_version || vars.DEFAULT_OPENWRT_VERSION }} + pkgarch: ${{ inputs.openwrt_arch || vars.DEFAULT_OPENWRT_ARCH }} + target: ${{ inputs.openwrt_target || vars.DEFAULT_OPENWRT_TARGET }} + subtarget: ${{ inputs.openwrt_subtarget || vars.DEFAULT_OPENWRT_SUBTARGET }} + vermagic: ${{ inputs.openwrt_vermagic || vars.DEFAULT_OPENWRT_VERMAGIC }} + + env: + OPENWRT_RELEASE: ${{ matrix.build_env.tag }} + OPENWRT_ARCH: ${{ matrix.build_env.pkgarch }} + OPENWRT_TARGET: ${{ matrix.build_env.target }} + OPENWRT_SUBTARGET: ${{ matrix.build_env.subtarget }} + OPENWRT_VERMAGIC: ${{ matrix.build_env.vermagic }} + + steps: + - name: checkout openwrt-loki-exporter + uses: actions/checkout@v4 + with: + path: openwrt-loki-exporter + fetch-depth: 0 + + - name: checkout openwrt + uses: actions/checkout@v4 + with: + path: openwrt + repository: openwrt/openwrt + ref: v${{ matrix.build_env.tag }} + fetch-depth: 0 + + - name: restore cached toolchain + id: restore-toolchain-cache + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: false + key: ${{ runner.os }}-openwrt-cache-toolchain-${{ matrix.build_env.tag }}-${{ matrix.build_env.pkgarch }}-${{ matrix.build_env.target }}-${{ matrix.build_env.subtarget }} + restore-keys: | + ${{ runner.os }}-openwrt-cache-toolchain-${{ matrix.build_env.pkgarch }}- + path: | + openwrt/.config + openwrt/.config.old + openwrt/feeds.conf + openwrt/bin/** + openwrt/build_dir/** + openwrt/dl/** + openwrt/feeds/** + openwrt/package/** + openwrt/staging_dir/** + openwrt/tmp/** + + - name: building openwrt toolchain + if: steps.restore-toolchain-cache.outputs.cache-hit != 'true' + id: build-toolchain + run: | + set -x + cd openwrt-loki-exporter + make show-env + time -p make build-toolchain + make purge-circular-symlinks + + - name: save toolchain cache + if: steps.restore-toolchain-cache.outputs.cache-hit != 'true' + id: save-toolchain-cache + uses: actions/cache/save@v4 + with: + key: ${{ runner.os }}-openwrt-cache-toolchain-${{ matrix.build_env.tag }}-${{ matrix.build_env.pkgarch }}-${{ matrix.build_env.target }}-${{ matrix.build_env.subtarget }} + path: | + openwrt/.config + openwrt/.config.old + openwrt/feeds.conf + openwrt/bin/** + openwrt/build_dir/** + openwrt/dl/** + openwrt/feeds/** + openwrt/package/** + openwrt/staging_dir/** + openwrt/tmp/** + + - name: build package + id: build-package + run: | + set -x + cd openwrt-loki-exporter + make show-env + make export-env >> $GITHUB_OUTPUT + time -p make package + make prepare-artifacts + + - name: upload artifacts + uses: actions/upload-artifact@v4 + with: + name: loki-exporter-${{ steps.build-package.outputs.version_str }}_v${{ matrix.build_env.tag }}_${{ matrix.build_env.pkgarch }}_${{ matrix.build_env.target }}_${{ matrix.build_env.subtarget }} + path: loki_exporter_artifacts/* diff --git a/.github/workflows/build-toolchain-cache.yml b/.github/workflows/build-toolchain-cache.yml new file mode 100644 index 0000000..9e7e31c --- /dev/null +++ b/.github/workflows/build-toolchain-cache.yml @@ -0,0 +1,91 @@ +name: build toolchain cache + +on: + workflow_dispatch: + inputs: + openwrt_version: + description: "OpenWrt version" + type: string + required: true + default: "23.05.3" + openwrt_arch: + description: "OpenWrt arch" + type: string + required: true + default: "mips_24kc" + openwrt_target: + description: "OpenWrt target" + type: string + required: true + default: "ath79" + openwrt_subtarget: + description: "OpenWrt subtarget" + type: string + required: true + default: "generic" + openwrt_vermagic: + description: "OpenWrt vermagic" + type: string + required: true + default: "auto" + +jobs: + build: + name: "v${{ matrix.build_env.tag }} - ${{ matrix.build_env.pkgarch }} :: ${{ matrix.build_env.target }}/${{ matrix.build_env.subtarget }} openwrt build" + runs-on: ubuntu-22.04 + strategy: + matrix: + build_env: + - tag: ${{ inputs.openwrt_version }} + pkgarch: ${{ inputs.openwrt_arch }} + target: ${{ inputs.openwrt_target }} + subtarget: ${{ inputs.openwrt_subtarget }} + vermagic: ${{ inputs.openwrt_vermagic }} + + env: + OPENWRT_RELEASE: ${{ matrix.build_env.tag }} + OPENWRT_ARCH: ${{ matrix.build_env.pkgarch }} + OPENWRT_TARGET: ${{ matrix.build_env.target }} + OPENWRT_SUBTARGET: ${{ matrix.build_env.subtarget }} + OPENWRT_VERMAGIC: ${{ matrix.build_env.vermagic }} + + steps: + - name: checkout openwrt-loki-exporter + uses: actions/checkout@v4 + with: + path: openwrt-loki-exporter + fetch-depth: 0 + + - name: checkout openwrt + uses: actions/checkout@v4 + with: + path: openwrt + repository: openwrt/openwrt + ref: v${{ matrix.build_env.tag }} + fetch-depth: 0 + + - name: building openwrt toolchain + id: build-toolchain + run: | + set -x + cd openwrt-loki-exporter + make show-env + time -p make build-toolchain + make purge-circular-symlinks + + - name: save toolchain cache + id: save-toolchain-cache + uses: actions/cache/save@v4 + with: + key: ${{ runner.os }}-openwrt-cache-toolchain-${{ matrix.build_env.tag }}-${{ matrix.build_env.pkgarch }}-${{ matrix.build_env.target }}-${{ matrix.build_env.subtarget }} + path: | + openwrt/.config + openwrt/.config.old + openwrt/feeds.conf + openwrt/bin/** + openwrt/build_dir/** + openwrt/dl/** + openwrt/feeds/** + openwrt/package/** + openwrt/staging_dir/** + openwrt/tmp/** diff --git a/.gitignore b/.gitignore index b64daaf..e0e9e17 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__ create-test-env run-test-exporter-onetime tests/*.log.state +loki-exporter/ diff --git a/Makefile b/Makefile index ce1e010..ec8a5ef 100644 --- a/Makefile +++ b/Makefile @@ -2,19 +2,55 @@ TOPDIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) SELF := $(abspath $(lastword $(MAKEFILE_LIST))) +UPPERDIR := $(realpath $(TOPDIR)/../) + +OPENWRT_SRCDIR ?= $(UPPERDIR)/openwrt +LOKI_EXPORTER_SRCDIR ?= $(TOPDIR) +LOKI_EXPORTER_DSTDIR ?= $(UPPERDIR)/loki_exporter_artifacts + +OPENWRT_RELEASE ?= 23.05.3 +OPENWRT_ARCH ?= mips_24kc +OPENWRT_TARGET ?= ath79 +OPENWRT_SUBTARGET ?= generic +OPENWRT_VERMAGIC ?= auto + +OPENWRT_ROOT_URL ?= https://downloads.openwrt.org/releases +OPENWRT_BASE_URL ?= $(OPENWRT_ROOT_URL)/$(OPENWRT_RELEASE)/targets/$(OPENWRT_TARGET)/$(OPENWRT_SUBTARGET) +OPENWRT_MANIFEST ?= $(OPENWRT_BASE_URL)/openwrt-$(OPENWRT_RELEASE)-$(OPENWRT_TARGET)-$(OPENWRT_SUBTARGET).manifest + +ifndef OPENWRT_VERMAGIC +_NEED_VERMAGIC=1 +endif + +ifeq ($(OPENWRT_VERMAGIC), auto) +_NEED_VERMAGIC=1 +endif + +ifeq ($(_NEED_VERMAGIC), 1) +OPENWRT_VERMAGIC := $(shell curl -fs $(OPENWRT_MANIFEST) | grep -- "^kernel" | sed -e "s,.*\-,,") +endif GITHUB_RUN_ID ?= 0 +GITHUB_SHA ?= $(shell git rev-parse --short HEAD) +VERSION_STR ?= $(shell git describe --tags --long --dirty) -DATE := $(shell date +"%Y%m%d") +DATE := $(shell date +"%Y%m%d") VERSION := $(shell git describe --tags --always --match='v[0-9]*' | cut -d '-' -f 1 | tr -d 'v') RELEASE := $(shell git describe --tags --always --match='v[0-9]*' --long | cut -d '-' -f 2) -BUILD := $(shell git describe --tags --long --always --dirty)-$(DATE)-$(GITHUB_RUN_ID) +BUILD := $(shell git describe --tags --long --always --dirty)-$(DATE)-$(GITHUB_RUN_ID) SHOW_ENV_VARS = \ VERSION \ RELEASE \ + GITHUB_SHA \ GITHUB_RUN_ID \ - BUILD + VERSION_STR \ + BUILD \ + OPENWRT_RELEASE \ + OPENWRT_ARCH \ + OPENWRT_TARGET \ + OPENWRT_SUBTARGET \ + OPENWRT_VERMAGIC help: ## Show help message (list targets) @awk 'BEGIN {FS = ":.*##"; printf "\nTargets:\n"} /^[$$()% 0-9a-zA-Z_-]+:.*?##/ {printf " \033[36m%-25s\033[0m %s\n", $$1, $$2}' $(SELF) @@ -23,11 +59,20 @@ show-var-%: @{ \ escaped_v="$(subst ",\",$($*))" ; \ if [ -n "$$escaped_v" ]; then v="$$escaped_v"; else v="(undefined)"; fi; \ - printf "%-13s %s\n" "$*" "$$v"; \ + printf "%-20s %s\n" "$*" "$$v"; \ } show-env: $(addprefix show-var-, $(SHOW_ENV_VARS)) ## Show environment details +export-var-%: + @{ \ + escaped_v="$(subst ",\",$($*))" ; \ + if [ -n "$$escaped_v" ]; then v="$$escaped_v"; else v="(undefined)"; fi; \ + printf "%s=%s\n" "$*" "$$v"; \ + } + +export-env: $(addprefix export-var-, $(SHOW_ENV_VARS)) ## Export environment + results: mkdir -p results @@ -124,6 +169,92 @@ compare-logs: | results save-logs: | results docker logs tests-loki-1 >results/loki.log 2>&1 +$(OPENWRT_SRCDIR): + @{ \ + set -ex ; \ + git clone https://github.com/openwrt/openwrt.git $@ ; \ + cd $@ ; \ + git checkout v$(OPENWRT_RELEASE) ; \ + } + +$(OPENWRT_SRCDIR)/feeds.conf: | $(OPENWRT_SRCDIR) + @{ \ + set -ex ; \ + curl -fsL $(OPENWRT_BASE_URL)/feeds.buildinfo | tee $@ ; \ + } + +$(OPENWRT_SRCDIR)/.config: | $(OPENWRT_SRCDIR) + @{ \ + set -ex ; \ + curl -fsL $(OPENWRT_BASE_URL)/config.buildinfo > $@ ; \ + } + +.PHONY: build-toolchain +build-toolchain: $(OPENWRT_SRCDIR)/feeds.conf $(OPENWRT_SRCDIR)/.config ## Build OpenWrt toolchain + @{ \ + set -ex ; \ + cd $(OPENWRT_SRCDIR) ; \ + time -p ./scripts/feeds update ; \ + time -p ./scripts/feeds install -a ; \ + time -p make defconfig ; \ + time -p make tools/install -i -j $(NPROC) ; \ + time -p make toolchain/install -i -j $(NPROC) ; \ + } + +# TODO: this should not be required but actions/cache/save@v4 could not handle circular symlinks with error like this: +# Warning: ELOOP: too many symbolic links encountered, stat '/home/runner/work/amneziawg-openwrt/amneziawg-openwrt/openwrt/staging_dir/toolchain-mips_24kc_gcc-11.2.0_musl/initial/lib/lib' +# Warning: Cache save failed. +.PHONY: purge-circular-symlinks +purge-circular-symlinks: + @{ \ + set -ex ; \ + cd $(OPENWRT_SRCDIR) ; \ + export LC_ALL=C ; \ + for deadlink in $$(find . -follow -type l -printf "" 2>&1 | sed -e "s/find: '\(.*\)': Too many levels of symbolic links.*/\1/"); do \ + echo "deleting dead link: $${deadlink}" ; \ + rm -f "$${deadlink}" ; \ + done ; \ + } + +loki-exporter: loki_exporter.sh loki_exporter.init loki_exporter.conf + mkdir -p $(TOPDIR)/$@ + sed \ + -e "s,%% PKG_VERSION %%,$(VERSION),g" \ + -e "s,%% PKG_RELEASE %%,$(RELEASE),g" \ + -e "s,%% BUILD_ID %%,$(BUILD),g" \ + < $(TOPDIR)/Makefile.package > $(TOPDIR)/$@/Makefile + mkdir -p $(TOPDIR)/$@/files + for f in loki_exporter.init loki_exporter.conf; do \ + install -m 644 $(TOPDIR)/$${f} $(TOPDIR)/$@/files/ ; \ + done + install -m 755 $(TOPDIR)/loki_exporter.sh $(TOPDIR)/$@/files/loki_exporter.sh + +.PHONY: package +package: loki-exporter ## Build OpenWRT package + @{ \ + set -ex ; \ + cd $(OPENWRT_SRCDIR) ; \ + echo "src-link loki_exporter $(LOKI_EXPORTER_SRCDIR)" > feeds.conf ; \ + ./scripts/feeds update ; \ + ./scripts/feeds install -a ; \ + mv .config.old .config ; \ + echo "CONFIG_PACKAGE_loki-exporter=y" >> .config ; \ + make defconfig ; \ + make V=s package/loki-exporter/clean ; \ + make V=s package/loki-exporter/download ; \ + make V=s package/loki-exporter/prepare ; \ + make V=s package/loki-exporter/compile ; \ + } + +.PHONY: prepare-artifacts +prepare-artifacts: ## Save loki-exporter artifacts (.ipk packages) + @{ \ + set -ex ; \ + cd $(OPENWRT_SRCDIR) ; \ + mkdir -p $(LOKI_EXPORTER_DSTDIR) ; \ + cp bin/packages/$(OPENWRT_ARCH)/loki_exporter/loki-exporter_*.ipk $(LOKI_EXPORTER_DSTDIR)/ ; \ + } + .PHONY: clean clean: delete-test-env ## Clean-up find $(TOPDIR)/ -type f -name "*.pyc" -delete @@ -133,3 +264,4 @@ clean: delete-test-env ## Clean-up rm -f $(TOPDIR)/run-test-exporter-onetime find $(TOPDIR)/tests/ -type f -name "*.log.state" -delete rm -rf $(TOPDIR)/results + rm -rf $(TOPDIR)/loki-exporter diff --git a/Makefile.package b/Makefile.package new file mode 100644 index 0000000..64c3771 --- /dev/null +++ b/Makefile.package @@ -0,0 +1,48 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=loki-exporter +PKG_VERSION:=%% PKG_VERSION %% +PKG_RELEASE:=%% PKG_RELEASE %% +PKG_LICENSE:=MIT +PKG_MAINTAINER:=Andrei Belov + +include $(INCLUDE_DIR)/package.mk + +define Package/loki-exporter + SECTION:=net + CATEGORY:=Network + TITLE:=loki-exporter + DEPENDS:=+curl + PKGARCH:=all + URL:=https://github.com/defanator/openwrt-loki-exporter +endef + +define Package/loki-exporter/description +Tiny service for exporting system log to remote Loki instance, +written in pure shell. + +Build ID: %% BUILD_ID %% + +endef + +define Build/Prepare +endef + +define Build/Configure +endef + +define Build/Compile +endef + +define Package/loki-exporter/install + $(INSTALL_DIR) $(1)/usr/bin + $(INSTALL_BIN) ./files/loki_exporter.sh $(1)/usr/bin/loki_exporter + + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/loki_exporter.init $(1)/etc/init.d/loki_exporter + + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/loki_exporter.conf $(1)/etc/config/loki_exporter +endef + +$(eval $(call BuildPackage,loki-exporter)) diff --git a/loki_exporter.config b/loki_exporter.conf similarity index 100% rename from loki_exporter.config rename to loki_exporter.conf diff --git a/loki_exporter.service b/loki_exporter.init similarity index 100% rename from loki_exporter.service rename to loki_exporter.init