From 557d095336f1a67f97168263ce17ac5422808239 Mon Sep 17 00:00:00 2001 From: cytopia Date: Fri, 4 Dec 2020 12:40:02 +0100 Subject: [PATCH 01/14] Add mypy linting --- .github/workflows/linting.yml | 1 + Makefile | 27 ++++++-- bin/vhost-gen | 116 ++++++++++++++++++++++++++++------ setup.cfg | 40 ++++++++++++ 4 files changed, 162 insertions(+), 22 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index af19ab2..0d4da90 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -22,6 +22,7 @@ jobs: - pycodestyle - pydocstyle - black + - mypy name: "[ ${{ matrix.target }} ]" steps: diff --git a/Makefile b/Makefile index 0b09a45..e5acc0e 100644 --- a/Makefile +++ b/Makefile @@ -36,16 +36,35 @@ help: # Lint Targets # ------------------------------------------------------------------------------------------------- -lint: pycodestyle pydocstyle black +lint: pycodestyle pydocstyle black mypy +.PHONY: pycodestyle pycodestyle: - docker run --rm -v $(PWD):/data cytopia/pycodestyle --show-source --show-pep8 $(BINPATH)$(BINNAME) + @echo "# -------------------------------------------------------------------- #" + @echo "# Check pycodestyle" + @echo "# -------------------------------------------------------------------- #" + docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/pycodestyle --show-source --show-pep8 $(BINPATH)$(BINNAME) +.PHONY: pydocstyle pydocstyle: - docker run --rm -v $(PWD):/data cytopia/pydocstyle $(BINPATH)$(BINNAME) + @echo "# -------------------------------------------------------------------- #" + @echo "# Check pydocstyle" + @echo "# -------------------------------------------------------------------- #" + docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/pydocstyle $(BINPATH)$(BINNAME) +.PHONY: black black: - docker run --rm -v ${PWD}:/data cytopia/black -l 100 --check --diff $(BINPATH)$(BINNAME) + @echo "# -------------------------------------------------------------------- #" + @echo "# Check black" + @echo "# -------------------------------------------------------------------- #" + docker run --rm $$(tty -s && echo "-it" || echo) -v ${PWD}:/data cytopia/black -l 100 --check --diff $(BINPATH)$(BINNAME) + +.PHONY: mypy +mypy: + @echo "# -------------------------------------------------------------------- #" + @echo "# Check mypy" + @echo "# -------------------------------------------------------------------- #" + docker run --rm $$(tty -s && echo "-it" || echo) -v ${PWD}:/data cytopia/mypy --config-file setup.cfg $(BINPATH)$(BINNAME) # ------------------------------------------------------------------------------------------------- diff --git a/bin/vhost-gen b/bin/vhost-gen index 373e2fb..904bacb 100755 --- a/bin/vhost-gen +++ b/bin/vhost-gen @@ -22,6 +22,9 @@ import getopt import itertools import yaml +if os.environ.get("MYPY_CHECK", False): + from typing import Optional, Iterator, List, Dict, Any, Callable, Tuple, Union + from types import CodeType ############################################################ # Globals @@ -76,6 +79,7 @@ TEMPLATES = {"apache22": "apache22.yml", "apache24": "apache24.yml", "nginx": "n def print_help(): + # type: () -> None """Show program help.""" print( """ @@ -134,6 +138,7 @@ Misc arguments: def print_version(): + # type: () -> None """Show program version.""" print("vhost-gen v1.0.1 (2020-09-29)") print("cytopia ") @@ -147,6 +152,7 @@ def print_version(): def str_replace(string, replacer): + # type: (str, Dict[str, Any]) -> str """Generic string replace.""" # Replace all 'keys' with 'values' @@ -157,12 +163,14 @@ def str_replace(string, replacer): def str_indent(text, amount, char=" "): + # type: (str, int, str) -> str """Indent every newline inside str by specified value.""" padding = amount * char return "".join(padding + line for line in text.splitlines(True)) def to_str(string): + # type: (Optional[str]) -> str """Dummy string retriever.""" if string is None: return "" @@ -170,6 +178,7 @@ def to_str(string): def load_yaml(path): + # type: (str) -> Tuple[bool, Dict[str, Any], str] """Wrapper to load yaml file safely.""" try: @@ -186,11 +195,13 @@ def load_yaml(path): def merge_yaml(yaml1, yaml2): + # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] """Merge two yaml strings. The secondary takes precedence.""" return dict(itertools.chain(yaml1.items(), yaml2.items())) def symlink(src, dst, force=False): + # type: (str, str, bool) -> Tuple[bool, Optional[str]] """ Wrapper function to create a symlink with the addition of being able to overwrite an already existing file. @@ -219,6 +230,7 @@ def symlink(src, dst, force=False): def parse_args(argv): + # type: (List[str]) -> Tuple[Any, ...] """Parse command line arguments.""" # Config location, can be overwritten with -c @@ -226,8 +238,8 @@ def parse_args(argv): l_template_dir = TEMPLATE_DIR o_template_dir = None save = None - path = None - name = None + docroot = None + name = "" proxy = None mode = None location = None @@ -258,7 +270,7 @@ def parse_args(argv): l_config_path = arg # Vhost document root path elif opt == "-p": - path = arg + docroot = arg # Vhost reverse proxy (ADDR:PORT) elif opt == "-r": proxy = arg @@ -287,7 +299,7 @@ def parse_args(argv): l_config_path, l_template_dir, o_template_dir, - path, + docroot, proxy, mode, location, @@ -299,6 +311,7 @@ def parse_args(argv): def validate_args_req(name, docroot, proxy, mode, location): + # type: (Optional[str], Optional[str], Optional[str], str, Optional[str]) -> None """Validate required arguments.""" # Validate required command line options are set if docroot is None and proxy is None: @@ -361,6 +374,7 @@ def validate_args_req(name, docroot, proxy, mode, location): def validate_args_opt(config_path, tpl_dir): + # type: (str, str) -> None """Validate optional arguments.""" if not os.path.isfile(config_path): @@ -396,7 +410,8 @@ def validate_args_opt(config_path, tpl_dir): ############################################################ -def validate_config(config): +def validate_config(config, verbose): + # type: (Dict[str, Any], bool) -> None """Validate some important keys in config dict.""" # Validate server type @@ -423,6 +438,7 @@ def validate_config(config): def vhost_get_port(config, ssl): + # type: (Dict[str, Any], bool) -> str """Get listen port.""" if ssl: if config["server"] == "nginx": @@ -433,6 +449,7 @@ def vhost_get_port(config, ssl): def vhost_get_http_proto(config, ssl): + # type: (Dict[str, Any], bool) -> str """Get HTTP protocol. Only relevant for Apache 2.4/Nginx and SSL.""" if config["server"] == "apache24": @@ -448,6 +465,7 @@ def vhost_get_http_proto(config, ssl): def vhost_get_default_server(config, default): + # type: (Dict[str, Any], bool) -> str """ Get vhost default directive which makes it the default vhost. @@ -471,6 +489,7 @@ def vhost_get_default_server(config, default): def vhost_get_server_name(config, server_name, default): + # type: (Dict[str, Any], str, bool) -> str """Get server name.""" # Nginx uses: "server_name _;" as the default @@ -485,6 +504,7 @@ def vhost_get_server_name(config, server_name, default): def vhost_get_access_log(config, server_name): + # type: (Dict[str, Any], str) -> str """Get access log directive.""" if config["vhost"]["log"]["access"]["stdout"]: return STDOUT_ACCESS @@ -496,6 +516,7 @@ def vhost_get_access_log(config, server_name): def vhost_get_error_log(config, server_name): + # type: (Dict[str, Any], str) -> str """Get error log directive.""" if config["vhost"]["log"]["error"]["stderr"]: return STDERR_ERROR @@ -512,6 +533,7 @@ def vhost_get_error_log(config, server_name): def vhost_get_vhost_docroot(config, template, docroot, proxy): + # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str]) -> str """Get document root directive.""" if proxy is not None: return "" @@ -526,14 +548,21 @@ def vhost_get_vhost_docroot(config, template, docroot, proxy): def vhost_get_vhost_rproxy(template, proxy, location): + # type: (Dict[str, Any], Optional[str], Optional[str]) -> str """Get reverse proxy definition.""" if proxy is not None: + match = re.search("^.*://(.+):[0-9]+", str(proxy)) + if not match: + print("[ERR] Wrong proxy format", file=sys.stderr) + sys.exit(1) + + proxy_addr = match.group(1) return str_replace( template["vhost_type"]["rproxy"], { "__LOCATION__": location, "__PROXY_PROTO__": re.sub("://.*$", "", proxy), - "__PROXY_ADDR__": re.search("^.*://(.+):[0-9]+", proxy).group(1), + "__PROXY_ADDR__": proxy_addr, "__PROXY_PORT__": re.sub("^.*:", "", proxy), }, ) @@ -546,6 +575,7 @@ def vhost_get_vhost_rproxy(template, proxy, location): def vhost_get_vhost_ssl(config, template, server_name): + # type: (Dict[str, Any], Dict[str, Any], str) -> str """Get ssl definition.""" return str_replace( template["features"]["ssl"], @@ -560,17 +590,19 @@ def vhost_get_vhost_ssl(config, template, server_name): def vhost_get_vhost_redir(config, template, server_name): + # type: (Dict[str, Any], Dict[str, Any], str) -> str """Get redirect to ssl definition.""" return str_replace( template["features"]["redirect"], { - "__VHOST_NAME__": vhost_get_server_name(config, server_name, 0), + "__VHOST_NAME__": vhost_get_server_name(config, server_name, False), "__SSL_PORT__": to_str(config["vhost"]["ssl_port"]), }, ) def vhost_get_ssl_crt_path(config, server_name): + # type: (Dict[str, Any], str) -> str """Get ssl crt path""" prefix = to_str(config["vhost"]["name"]["prefix"]) @@ -582,6 +614,7 @@ def vhost_get_ssl_crt_path(config, server_name): def vhost_get_ssl_key_path(config, server_name): + # type: (Dict[str, Any], str) -> str """Get ssl key path""" prefix = to_str(config["vhost"]["name"]["prefix"]) suffix = to_str(config["vhost"]["name"]["suffix"]) @@ -592,26 +625,29 @@ def vhost_get_ssl_key_path(config, server_name): def vhost_get_docroot_path(config, docroot, proxy): + # type: (Dict[str, Any], Optional[str], Optional[str]) -> str """Get path of document root.""" if proxy is not None: return "" suffix = to_str(config["vhost"]["docroot"]["suffix"]) - path = os.path.join(docroot, suffix) + path = os.path.join(str(docroot), suffix) return path def vhost_get_index(config): + # type: (Dict[str, Any]) -> str """Get index.""" if "index" in config["vhost"] and config["vhost"]["index"]: elem = config["vhost"]["index"] else: - elem = DEFAULT_CONFIG["vhost"]["index"] + elem = DEFAULT_CONFIG["vhost"]["index"] # type: ignore return " ".join(elem) def vhost_get_php_fpm(config, template, docroot, proxy): + # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str]) -> str """Get PHP FPM directive. If using reverse proxy, PHP-FPM will be disabled.""" if proxy is not None: return "" @@ -632,6 +668,7 @@ def vhost_get_php_fpm(config, template, docroot, proxy): def vhost_get_aliases(config, template): + # type: (Dict[str, Any], Dict[str, Any]) -> str """Get virtual host alias directives.""" aliases = [] for item in config["vhost"]["alias"]: @@ -659,6 +696,7 @@ def vhost_get_aliases(config, template): def vhost_get_denies(config, template): + # type: (Dict[str, Any], Dict[str, Any]) -> str """Get virtual host deny alias directives.""" denies = [] for item in config["vhost"]["deny"]: @@ -670,6 +708,7 @@ def vhost_get_denies(config, template): def vhost_get_server_status(config, template): + # type: (Dict[str, Any], Dict[str, Any]) -> str """Get virtual host server status directivs.""" status = "" if config["vhost"]["server_status"]["enable"]: @@ -679,6 +718,7 @@ def vhost_get_server_status(config, template): def vhost_get_custom_section(config): + # type: (Dict[str, Any]) -> str """Get virtual host custom directives.""" return to_str(config["custom"]) @@ -688,7 +728,16 @@ def vhost_get_custom_section(config): ############################################################ -def get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default): +def get_vhost_plain( + config, # type: Dict[str, Any] + tpl, # type: Dict[str, Any] + docroot, # type: Optional[str] + proxy, # type: str + location, # type: Optional[str] + server_name, # type: str + default, # type: bool +): + # type: (...) -> str """Get plain vhost""" return str_replace( tpl["vhost"], @@ -716,7 +765,16 @@ def get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default) ) -def get_vhost_ssl(config, tpl, docroot, proxy, location, server_name, default): +def get_vhost_ssl( + config, # type: Dict[str, Any] + tpl, # type: Dict[str, Any] + docroot, # type: Optional[str] + proxy, # type: str + location, # type: Optional[str] + server_name, # type: str + default, # type: bool +): + # type: (...) -> str """Get ssl vhost""" return str_replace( tpl["vhost"], @@ -744,7 +802,15 @@ def get_vhost_ssl(config, tpl, docroot, proxy, location, server_name, default): ) -def get_vhost_redir(config, tpl, docroot, proxy, server_name, default): +def get_vhost_redir( + config, # type: Dict[str, Any] + tpl, # type: Dict[str, Any] + docroot, # type: Optional[str] + proxy, # type: str + server_name, # type: str + default, # type: bool +): + # type: (...) -> str """Get redirect to ssl vhost""" return str_replace( tpl["vhost"], @@ -770,7 +836,17 @@ def get_vhost_redir(config, tpl, docroot, proxy, server_name, default): ) -def get_vhost(config, tpl, docroot, proxy, mode, location, server_name, default): +def get_vhost( + config, # type: Dict[str, Any] + tpl, # type: Dict[str, Any] + docroot, # type: Optional[str] + proxy, # type: str + mode, # type: str + location, # type: Optional[str] + server_name, # type: str + default, # type: bool +): + # type: (...) -> str """Create the vhost.""" if mode == "ssl": @@ -794,6 +870,7 @@ def get_vhost(config, tpl, docroot, proxy, mode, location, server_name, default) def load_config(config_path): + # type: (str) -> Tuple[bool, Dict[str, Any], str] """Load config and merge with defaults in case not found or something is missing.""" # Load configuration file @@ -812,6 +889,7 @@ def load_config(config_path): def load_template(template_dir, o_template_dir, server): + # type: (str, str, str) -> Tuple[bool, Dict[str, Any], str] """Load global and optional template file and merge them.""" # Load global template file @@ -854,6 +932,7 @@ def load_template(template_dir, o_template_dir, server): def apply_log_settings(config): + # type: (Dict[str, Any]) -> Tuple[bool, Optional[str]] """ This function will apply various settings for the log defines, including creating the directory itself as well as handling log file output (access @@ -888,6 +967,7 @@ def apply_log_settings(config): def main(argv): + # type: (List[Any]) -> None """Main entrypoint.""" # Get command line arguments @@ -911,15 +991,15 @@ def main(argv): validate_args_opt(config_path, tpl_dir) # Load config - succ, config, err = load_config(config_path) + succ, config, err_config = load_config(config_path) if not succ: - print("[ERR] Error loading config", err, file=sys.stderr) + print("[ERR] Error loading config", err_config, file=sys.stderr) sys.exit(1) # Load template - succ, template, err = load_template(tpl_dir, o_tpl_dir, config["server"]) + succ, template, err_tpl = load_template(tpl_dir, o_tpl_dir, config["server"]) if not succ: - print("[ERR] Error loading template", err, file=sys.stderr) + print("[ERR] Error loading template", err_tpl, file=sys.stderr) sys.exit(1) # Validate configuration file @@ -957,7 +1037,7 @@ def main(argv): outfile.write(vhost) # Apply settings for logging (symlinks, mkdir) only in save mode - succ, err = apply_log_settings(config) + succ, err_apply = apply_log_settings(config) if not succ: print(err, file=sys.stderr) sys.exit(1) diff --git a/setup.cfg b/setup.cfg index 4e2a1b6..b5ce024 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,43 @@ universal=1 [pycodestyle] max-line-length = 100 + +[mypy] +# Display +show_error_context = True +show_column_numbers = True +show_error_codes = True +pretty = True +color_output = True +error_summary = True + +# Meta +warn_unused_configs = True +incremental = False +show_traceback = True + +# Mode +strict_optional = True +show_none_errors = True + +# Allow +disallow_any_expr = False +disallow_any_explicit = False +disallow_any_decorated = False + +# Deny +disallow_any_unimported = True +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_no_return = True +warn_return_any = True +warn_unreachable = True +allow_untyped_globals = False +allow_redefinition = False From daf22dbe9ab871708708b8cb1a6702597eca2638 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sat, 5 Dec 2020 08:37:55 +0100 Subject: [PATCH 02/14] Add more CI checks --- .github/workflows/code-black.yml | 30 ++++++++++++ .github/workflows/code-mypy.yml | 30 ++++++++++++ .github/workflows/code-pycodestyle.yml | 30 ++++++++++++ .github/workflows/code-pydocstyle.yml | 30 ++++++++++++ .github/workflows/linting.yml | 6 +-- Makefile | 65 +++++++++++++++++++++----- 6 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/code-black.yml create mode 100644 .github/workflows/code-mypy.yml create mode 100644 .github/workflows/code-pycodestyle.yml create mode 100644 .github/workflows/code-pydocstyle.yml diff --git a/.github/workflows/code-black.yml b/.github/workflows/code-black.yml new file mode 100644 index 0000000..a019ca3 --- /dev/null +++ b/.github/workflows/code-black.yml @@ -0,0 +1,30 @@ +--- + +### +### Code style +### + +name: black +on: + pull_request: + push: + branches: + - master + tags: + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + target: + - black + name: "[ ${{ matrix.target }} ]" + steps: + - name: Checkout repository + uses: actions/checkout@master + + - name: "${{ matrix.target }}" + run: | + make _code-${{ matrix.target }} diff --git a/.github/workflows/code-mypy.yml b/.github/workflows/code-mypy.yml new file mode 100644 index 0000000..93b7c9f --- /dev/null +++ b/.github/workflows/code-mypy.yml @@ -0,0 +1,30 @@ +--- + +### +### Code style +### + +name: mypy +on: + pull_request: + push: + branches: + - master + tags: + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + target: + - mypy + name: "[ ${{ matrix.target }} ]" + steps: + - name: Checkout repository + uses: actions/checkout@master + + - name: "${{ matrix.target }}" + run: | + make _code-${{ matrix.target }} diff --git a/.github/workflows/code-pycodestyle.yml b/.github/workflows/code-pycodestyle.yml new file mode 100644 index 0000000..91a0e14 --- /dev/null +++ b/.github/workflows/code-pycodestyle.yml @@ -0,0 +1,30 @@ +--- + +### +### Code style +### + +name: pycode +on: + pull_request: + push: + branches: + - master + tags: + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + target: + - pycodestyle + name: "[ ${{ matrix.target }} ]" + steps: + - name: Checkout repository + uses: actions/checkout@master + + - name: "${{ matrix.target }}" + run: | + make _code-${{ matrix.target }} diff --git a/.github/workflows/code-pydocstyle.yml b/.github/workflows/code-pydocstyle.yml new file mode 100644 index 0000000..a16f7cc --- /dev/null +++ b/.github/workflows/code-pydocstyle.yml @@ -0,0 +1,30 @@ +--- + +### +### Code style +### + +name: pydoc +on: + pull_request: + push: + branches: + - master + tags: + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + target: + - pydocstyle + name: "[ ${{ matrix.target }} ]" + steps: + - name: Checkout repository + uses: actions/checkout@master + + - name: "${{ matrix.target }}" + run: | + make _code-${{ matrix.target }} diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 0d4da90..cc96c47 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -19,10 +19,8 @@ jobs: fail-fast: False matrix: target: - - pycodestyle - - pydocstyle - - black - - mypy + - _lint-files + - _lint-version name: "[ ${{ matrix.target }} ]" steps: diff --git a/Makefile b/Makefile index e5acc0e..ab1640e 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ endif # ------------------------------------------------------------------------------------------------- # Default configuration # ------------------------------------------------------------------------------------------------- -.PHONY: help lint pycodestyle pydocstyle black dist sdist bdist build checkbuild deploy autoformat clean +.PHONY: help lint code dist sdist bdist build checkbuild deploy autoformat clean VERSION = 2.7 @@ -20,7 +20,8 @@ TPLDIR = templates # Default Target # ------------------------------------------------------------------------------------------------- help: - @echo "lint Lint source code" + @echo "lint Lint repository" + @echo "code Lint source code" @echo "test Test source code" @echo "autoformat Autoformat code according to Python black" @echo "install Install (requires sudo or root)" @@ -33,34 +34,74 @@ help: # ------------------------------------------------------------------------------------------------- -# Lint Targets +# Lint Repository Targets # ------------------------------------------------------------------------------------------------- +lint: _lint-files +lint: _lint-version -lint: pycodestyle pydocstyle black mypy +.PHONY: _lint-files +_lint-files: + @echo "# --------------------------------------------------------------------" + @echo "# Lint files" + @echo "# -------------------------------------------------------------------- #" + @docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/file-lint:$(FL_VERSION) file-cr --text --ignore '$(FL_IGNORES)' --path . + @docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/file-lint:$(FL_VERSION) file-crlf --text --ignore '$(FL_IGNORES)' --path . + @docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/file-lint:$(FL_VERSION) file-trailing-single-newline --text --ignore '$(FL_IGNORES)' --path . + @docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/file-lint:$(FL_VERSION) file-trailing-space --text --ignore '$(FL_IGNORES)' --path . + @docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/file-lint:$(FL_VERSION) file-utf8 --text --ignore '$(FL_IGNORES)' --path . + @docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/file-lint:$(FL_VERSION) file-utf8-bom --text --ignore '$(FL_IGNORES)' --path . + +.PHONY: _lint-version +_lint-version: + @echo "# -------------------------------------------------------------------- #" + @echo "# Check version config" + @echo "# -------------------------------------------------------------------- #" + @VERSION_VHOSTGEN=$$( grep -E '^VERSION = "v?[.0-9]+(-\w+)?"' $(BINPATH)$(BINNAME) | awk -F'"' '{print $$2}' || true ); \ + VERSION_SETUP=$$( grep version= setup.py | awk -F'"' '{print $$2}' || true ); \ + if [ "$${VERSION_VHOSTGEN}" != "$${VERSION_SETUP}" ]; then \ + echo "[ERROR] Version mismatch"; \ + echo "bin/vhost-gen: $${VERSION_VHOSTGEN}"; \ + echo "setup.py: $${VERSION_SETUP}"; \ + exit 1; \ + else \ + echo "[OK] Version match"; \ + echo "bin/vhost-gen: $${VERSION_VHOSTGEN}"; \ + echo "setup.py: $${VERSION_SETUP}"; \ + exit 0; \ + fi \ + + +# ------------------------------------------------------------------------------------------------- +# Lint Code Targets +# ------------------------------------------------------------------------------------------------- +code: _code-pycodestyle +code: _code-pydocstyle +code: _code-black +code: _code-mypy -.PHONY: pycodestyle -pycodestyle: +.PHONY: _code-pycodestyle +_code-pycodestyle: @echo "# -------------------------------------------------------------------- #" @echo "# Check pycodestyle" @echo "# -------------------------------------------------------------------- #" docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/pycodestyle --show-source --show-pep8 $(BINPATH)$(BINNAME) -.PHONY: pydocstyle -pydocstyle: +.PHONY: _code-pydocstyle +_code-pydocstyle: @echo "# -------------------------------------------------------------------- #" @echo "# Check pydocstyle" @echo "# -------------------------------------------------------------------- #" docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/pydocstyle $(BINPATH)$(BINNAME) -.PHONY: black -black: +.PHONY: _code-black +_code-black: @echo "# -------------------------------------------------------------------- #" @echo "# Check black" @echo "# -------------------------------------------------------------------- #" docker run --rm $$(tty -s && echo "-it" || echo) -v ${PWD}:/data cytopia/black -l 100 --check --diff $(BINPATH)$(BINNAME) -.PHONY: mypy -mypy: +.PHONY: _code-mypy +_code-mypy: @echo "# -------------------------------------------------------------------- #" @echo "# Check mypy" @echo "# -------------------------------------------------------------------- #" From 4fb7d8b9f571c9d8c947fc208e49af8bfcd0187b Mon Sep 17 00:00:00 2001 From: cytopia Date: Sat, 5 Dec 2020 11:01:51 +0100 Subject: [PATCH 03/14] Integrate logging --- README.md | 6 +- bin/vhost-gen | 339 ++++++++++++++++++++++++++++++++------------------ setup.py | 10 +- 3 files changed, 225 insertions(+), 130 deletions(-) diff --git a/README.md b/README.md index a04ec37..3ad4e8a 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ If you are not satisfied with the `Allow from all` permissions, simply rewrite t #### Available command line options ```bash -Usage: vhost-gen -p|r -n [-l -m -c -t -o -d -s -v] +Usage: vhost-gen -p|r -n [-l -c -t -o -d -s -v] vhost-gen --help vhost-gen --version @@ -263,7 +263,7 @@ Optional arguments: If not set, the default location is /etc/vhost-gen/templates/ If vhost template files are not found in this directory, the program will abort. - -o Path to local vhost template directory. + -o Path to local override vhost template directory. This is used as a secondary template directory and definitions found here will be merged with the ones found in the global template directory. Note, definitions in local vhost teplate directory take precedence over @@ -274,7 +274,7 @@ Optional arguments: Apache does not have any specialities, the first vhost takes precedence. -s If specified, the generated vhost will be saved in the location found in conf.yml. If not specified, vhost will be printed to stdout. - -v Be verbose. + -v Be verbose. Use twice for debug output. Misc arguments: --help Show this help. diff --git a/bin/vhost-gen b/bin/vhost-gen index 904bacb..7fc8ac8 100755 --- a/bin/vhost-gen +++ b/bin/vhost-gen @@ -30,6 +30,11 @@ if os.environ.get("MYPY_CHECK", False): # Globals ############################################################ +APPNAME = "vhost-gen" +APPREPO = "https://github.com/devilbox/vhost-gen" +VERSION = "1.0.2" +RELDATE = "2020-12-05" + # Default paths CONFIG_PATH = "/etc/vhost-gen/conf.yml" TEMPLATE_DIR = "/etc/vhost-gen/templates" @@ -78,6 +83,28 @@ TEMPLATES = {"apache22": "apache22.yml", "apache24": "apache24.yml", "nginx": "n ############################################################ +# Log info to stderr during execution based on verbosity +# level: +# 0: error +# 1: warning +# 2: info +# 3: debug +def log(level, message, verbosity): + # type: (int, str, int) -> None + """Log based on current verbosity.""" + + stime = time.strftime("%Y-%m-%d %H:%M:%S") + + if level == 0: + print("vhost-gen: \x1b[31;21m[ERROR] (%s): %s\x1b[0m" % (stime, message), file=sys.stderr) + elif level == 1: + print("vhost-gen: \x1b[33;21m[WARN] (%s): %s\x1b[0m" % (stime, message), file=sys.stderr) + elif level == 2 and verbosity >= 1: + print("vhost-gen: [INFO] (%s): %s" % (stime, message), file=sys.stderr) + elif level == 3 and verbosity >= 2: + print("vhost-gen: [DEBUG] (%s): %s" % (stime, message), file=sys.stderr) + + def print_help(): # type: () -> None """Show program help.""" @@ -117,7 +144,7 @@ Optional arguments: If not set, the default location is /etc/vhost-gen/templates/ If vhost template files are not found in this directory, the program will abort. - -o Path to local vhost template directory. + -o Path to local override vhost template directory. This is used as a secondary template directory and definitions found here will be merged with the ones found in the global template directory. Note, definitions in local vhost teplate directory take precedence over @@ -128,7 +155,7 @@ Optional arguments: Apache does not have any specialities, the first vhost takes precedence. -s If specified, the generated vhost will be saved in the location found in conf.yml. If not specified, vhost will be printed to stdout. - -v Be verbose. + -v Be verbose. Use twice for debug output. Misc arguments: --help Show this help. @@ -140,9 +167,9 @@ Misc arguments: def print_version(): # type: () -> None """Show program version.""" - print("vhost-gen v1.0.1 (2020-09-29)") - print("cytopia ") - print("https://github.com/devilbox/vhost-gen") + print("%s: Version v%s (%s) by cytopia" % (APPNAME, VERSION, RELDATE)) + print("") + print("") print("The MIT License (MIT)") @@ -244,14 +271,14 @@ def parse_args(argv): mode = None location = None default = False - verbose = False + verbose = 0 # Define command line options try: opts, argv = getopt.getopt(argv, "vm:c:p:r:l:n:t:o:ds", ["version", "help"]) except getopt.GetoptError as err: - print("[ERR]", str(err), file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, str(err), 1) + log(0, "Type --help for help", 1) sys.exit(2) # Get command line options @@ -264,7 +291,7 @@ def parse_args(argv): sys.exit() # Verbose elif opt == "-v": - verbose = True + verbose = verbose + 1 # Config file overwrite elif opt == "-c": l_config_path = arg @@ -310,98 +337,108 @@ def parse_args(argv): ) -def validate_args_req(name, docroot, proxy, mode, location): - # type: (Optional[str], Optional[str], Optional[str], str, Optional[str]) -> None +def validate_args_req(name, docroot, proxy, mode, location, verbose): + # type: (Optional[str], Optional[str], Optional[str], str, Optional[str], int) -> None """Validate required arguments.""" # Validate required command line options are set if docroot is None and proxy is None: - print("[ERR] -p or -r is required", file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "-p or -r is required", verbose) + log(0, "Type --help for help", verbose) sys.exit(1) if docroot is not None and proxy is not None: - print("[ERR] -p and -r are mutually exclusive", file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "-p and -r are mutually exclusive", verbose) + log(0, "Type --help for help", verbose) sys.exit(1) # Check proxy string if proxy is not None: if location is None: - print("[ERR] When specifying -r, -l is also required.", file=sys.stderr) + log(0, "When specifying -r, -l is also required.", verbose) + log(0, "Type --help for help", verbose) sys.exit(1) # Regex: HOSTNAME/IP:PORT regex = re.compile("(^http(s)?://[-_.a-zA-Z0-9]+:[0-9]+$)", re.IGNORECASE) if not regex.match(proxy): - print( - "[ERR] Invalid proxy argument string: '%s', should be: %s or %s." - % (proxy, "http(s)://HOST:PORT", "http(s)://IP:PORT"), - file=sys.stderr, + log( + 0, + "Invalid proxy argument string: '{}', should be {} or {}.".format( + proxy, "http(s)://HOST:PORT", "http(s)://IP:PORT" + ), + verbose, ) + log(0, "Type --help for help", verbose) sys.exit(1) port = int(re.sub("^.*:", "", proxy)) if port < 1 or port > 65535: - print( - "[ERR] Invalid reverse proxy port range: '%d', should between 1 and 65535" % (port), - file=sys.stderr, + log( + 0, + "Invalid reverse proxy range: '{}', should be between 1 and 65535".format(port), + verbose, ) + log(0, "Type --help for help", verbose) sys.exit(1) # Check mode string if mode is not None: if mode not in ("plain", "ssl", "both", "redir"): - print( - "[ERR] Invalid -m mode string: '%s', should be: %s, %s, %s or %s" - % (mode, "plain", "ssl", "both", "redir"), - file=sys.stderr, + log( + 0, + "Invalid -m mode string: '{}', should be: {}, {}, {} or {}".format( + mode, "plain", "ssl", "both", "redir" + ), + verbose, ) + log(0, "Type --help for help", verbose) sys.exit(1) # Check normal server settings if docroot is not None: if location is not None: - print("[WARN] -l is ignored when using normal vhost (-p)", file=sys.stderr) + log(1, "-l is ignored when using normal vhost (-p)", verbose) if name is None: - print("[ERR] -n is required", file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "-n is required", verbose) + log(0, "Type --help for help", verbose) sys.exit(1) regex = re.compile("(^[-_.a-zA-Z0-9]+$)", re.IGNORECASE) if not regex.match(name): - print("[ERR] Invalid name:", name, file=sys.stderr) + log(0, "Invalid name: {}".format(name), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) -def validate_args_opt(config_path, tpl_dir): - # type: (str, str) -> None +def validate_args_opt(config_path, tpl_dir, verbose): + # type: (str, str, int) -> None """Validate optional arguments.""" if not os.path.isfile(config_path): - print("[WARN] Config file not found:", config_path, file=sys.stderr) + log(1, "Config file not found: {}".format(config_path), verbose) if not os.path.isdir(tpl_dir): - print("[ERR] Template path does not exist:", tpl_dir, file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "Template path does not exist: {}".format(tpl_dir), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) # Validate global templates tpl_file = os.path.join(tpl_dir, TEMPLATES["apache22"]) if not os.path.isfile(tpl_file): - print("[ERR] Apache 2.2 template file does not exist:", tpl_file, file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "Apache 2.2 template file does not exist: {}".format(tpl_file), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) tpl_file = os.path.join(tpl_dir, TEMPLATES["apache24"]) if not os.path.isfile(tpl_file): - print("[ERR] Apache 2.4 template file does not exist:", tpl_file, file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "Apache 2.4 template file does not exist: {}".format(tpl_file), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) tpl_file = os.path.join(tpl_dir, TEMPLATES["nginx"]) if not os.path.isfile(tpl_file): - print("[ERR] Nginx template file does not exist:", tpl_file, file=sys.stderr) - print("Type --help for help", file=sys.stderr) + log(0, "Nginx template file does not exist: {}".format(tpl_file), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) @@ -417,8 +454,8 @@ def validate_config(config, verbose): # Validate server type valid_hosts = list(TEMPLATES.keys()) if config["server"] not in valid_hosts: - print("[ERR] httpd.server must be 'apache22', 'apache24' or 'nginx'", file=sys.stderr) - print("[ERR] Your configuration is:", config["server"], file=sys.stderr) + log(0, "httpd.server must be 'apache22', 'apache24' or 'nginx'", verbose) + log(0, "Your configuration is: " + config["server"], verbose) sys.exit(1) @@ -532,8 +569,8 @@ def vhost_get_error_log(config, server_name): ############################################################ -def vhost_get_vhost_docroot(config, template, docroot, proxy): - # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str]) -> str +def vhost_get_vhost_docroot(config, template, docroot, proxy, verbose): + # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str], int) -> str """Get document root directive.""" if proxy is not None: return "" @@ -547,13 +584,14 @@ def vhost_get_vhost_docroot(config, template, docroot, proxy): ) -def vhost_get_vhost_rproxy(template, proxy, location): - # type: (Dict[str, Any], Optional[str], Optional[str]) -> str +def vhost_get_vhost_rproxy(template, proxy, location, verbose): + # type: (Dict[str, Any], Optional[str], Optional[str], int) -> str """Get reverse proxy definition.""" if proxy is not None: match = re.search("^.*://(.+):[0-9]+", str(proxy)) if not match: - print("[ERR] Wrong proxy format", file=sys.stderr) + log(0, "Wrong proxy format", verbose) + log(0, "Type --help for help", verbose) sys.exit(1) proxy_addr = match.group(1) @@ -574,8 +612,8 @@ def vhost_get_vhost_rproxy(template, proxy, location): ############################################################ -def vhost_get_vhost_ssl(config, template, server_name): - # type: (Dict[str, Any], Dict[str, Any], str) -> str +def vhost_get_vhost_ssl(config, template, server_name, verbose): + # type: (Dict[str, Any], Dict[str, Any], str, int) -> str """Get ssl definition.""" return str_replace( template["features"]["ssl"], @@ -589,8 +627,8 @@ def vhost_get_vhost_ssl(config, template, server_name): ) -def vhost_get_vhost_redir(config, template, server_name): - # type: (Dict[str, Any], Dict[str, Any], str) -> str +def vhost_get_vhost_redir(config, template, server_name, verbose): + # type: (Dict[str, Any], Dict[str, Any], str, int) -> str """Get redirect to ssl definition.""" return str_replace( template["features"]["redirect"], @@ -695,32 +733,50 @@ def vhost_get_aliases(config, template): return os.linesep.join(aliases) -def vhost_get_denies(config, template): - # type: (Dict[str, Any], Dict[str, Any]) -> str +def vhost_get_denies(config, template, verbose): + # type: (Dict[str, Any], Dict[str, Any], int) -> str """Get virtual host deny alias directives.""" denies = [] for item in config["vhost"]["deny"]: + log(3, "[denies] Retrieving cfg 'deny': {}".format(item), verbose) + + replacer = to_str(item["alias"]) + log(3, "[denies] Replacing: __REGEX__ -> {}".format(replacer), verbose) denies.append( - str_replace(template["features"]["deny"], {"__REGEX__": to_str(item["alias"])}) + str_replace(template["features"]["deny"], {"__REGEX__": replacer}) ) # Join by OS independent newlines - return os.linesep.join(denies) + deny = os.linesep.join(denies) + log(3, "[denies] Returning: {}".format(deny), verbose) + return deny -def vhost_get_server_status(config, template): - # type: (Dict[str, Any], Dict[str, Any]) -> str +def vhost_get_server_status(config, template, verbose): + # type: (Dict[str, Any], Dict[str, Any], int) -> str """Get virtual host server status directivs.""" status = "" - if config["vhost"]["server_status"]["enable"]: + enable = config["vhost"]["server_status"]["enable"] + + log(3, "[status] Retrieving cfg 'enable': {}".format(enable), verbose) + if enable: status = template["features"]["server_status"] + log(3, "[status] Retrieving tpl 'server_status': {}".format(status), verbose) - return str_replace(status, {"__REGEX__": to_str(config["vhost"]["server_status"]["alias"])}) + replacer = to_str(config["vhost"]["server_status"]["alias"]) + log(3, "[status] Replacing: __REGEX__ -> {}".format(replacer), verbose) + sstatus = str_replace(status, {"__REGEX__": replacer}) + log(3, "[status] Returning: {}".format(sstatus), verbose) + return sstatus -def vhost_get_custom_section(config): - # type: (Dict[str, Any]) -> str + +def vhost_get_custom_section(config, verbose): + # type: (Dict[str, Any], int) -> str """Get virtual host custom directives.""" - return to_str(config["custom"]) + custom = to_str(config["custom"]) + log(3, "[custom] Retrieving cfg 'custom': {}".format(custom), verbose) + log(3, "[custom] Returning': {}".format(custom), verbose) + return custom ############################################################ @@ -736,6 +792,7 @@ def get_vhost_plain( location, # type: Optional[str] server_name, # type: str default, # type: bool + verbose, # type: int ): # type: (...) -> str """Get plain vhost""" @@ -748,9 +805,11 @@ def get_vhost_plain( "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), "__VHOST_NAME__": vhost_get_server_name(config, server_name, default), "__VHOST_DOCROOT__": str_indent( - vhost_get_vhost_docroot(config, tpl, docroot, proxy), 4 + vhost_get_vhost_docroot(config, tpl, docroot, proxy, verbose), 4 + ), + "__VHOST_RPROXY__": str_indent( + vhost_get_vhost_rproxy(tpl, proxy, location, verbose), 4 ), - "__VHOST_RPROXY__": str_indent(vhost_get_vhost_rproxy(tpl, proxy, location), 4), "__REDIRECT__": "", "__SSL__": "", "__INDEX__": vhost_get_index(config), @@ -758,9 +817,9 @@ def get_vhost_plain( "__ERROR_LOG__": vhost_get_error_log(config, server_name), "__PHP_FPM__": str_indent(vhost_get_php_fpm(config, tpl, docroot, proxy), 4), "__ALIASES__": str_indent(vhost_get_aliases(config, tpl), 4), - "__DENIES__": str_indent(vhost_get_denies(config, tpl), 4), - "__SERVER_STATUS__": str_indent(vhost_get_server_status(config, tpl), 4), - "__CUSTOM__": str_indent(vhost_get_custom_section(config), 4), + "__DENIES__": str_indent(vhost_get_denies(config, tpl, verbose), 4), + "__SERVER_STATUS__": str_indent(vhost_get_server_status(config, tpl, verbose), 4), + "__CUSTOM__": str_indent(vhost_get_custom_section(config, verbose), 4), }, ) @@ -773,6 +832,7 @@ def get_vhost_ssl( location, # type: Optional[str] server_name, # type: str default, # type: bool + verbose, # type: int ): # type: (...) -> str """Get ssl vhost""" @@ -785,19 +845,21 @@ def get_vhost_ssl( "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), "__VHOST_NAME__": vhost_get_server_name(config, server_name, default), "__VHOST_DOCROOT__": str_indent( - vhost_get_vhost_docroot(config, tpl, docroot, proxy), 4 + vhost_get_vhost_docroot(config, tpl, docroot, proxy, verbose), 4 + ), + "__VHOST_RPROXY__": str_indent( + vhost_get_vhost_rproxy(tpl, proxy, location, verbose), 4 ), - "__VHOST_RPROXY__": str_indent(vhost_get_vhost_rproxy(tpl, proxy, location), 4), "__REDIRECT__": "", - "__SSL__": str_indent(vhost_get_vhost_ssl(config, tpl, server_name), 4), + "__SSL__": str_indent(vhost_get_vhost_ssl(config, tpl, server_name, verbose), 4), "__INDEX__": vhost_get_index(config), "__ACCESS_LOG__": vhost_get_access_log(config, server_name + "_ssl"), "__ERROR_LOG__": vhost_get_error_log(config, server_name + "_ssl"), "__PHP_FPM__": str_indent(vhost_get_php_fpm(config, tpl, docroot, proxy), 4), "__ALIASES__": str_indent(vhost_get_aliases(config, tpl), 4), - "__DENIES__": str_indent(vhost_get_denies(config, tpl), 4), - "__SERVER_STATUS__": str_indent(vhost_get_server_status(config, tpl), 4), - "__CUSTOM__": str_indent(vhost_get_custom_section(config), 4), + "__DENIES__": str_indent(vhost_get_denies(config, tpl, verbose), 4), + "__SERVER_STATUS__": str_indent(vhost_get_server_status(config, tpl, verbose), 4), + "__CUSTOM__": str_indent(vhost_get_custom_section(config, verbose), 4), }, ) @@ -809,6 +871,7 @@ def get_vhost_redir( proxy, # type: str server_name, # type: str default, # type: bool + verbose, # type: int ): # type: (...) -> str """Get redirect to ssl vhost""" @@ -822,7 +885,7 @@ def get_vhost_redir( "__VHOST_NAME__": vhost_get_server_name(config, server_name, default), "__VHOST_DOCROOT__": "", "__VHOST_RPROXY__": "", - "__REDIRECT__": str_indent(vhost_get_vhost_redir(config, tpl, server_name), 4), + "__REDIRECT__": str_indent(vhost_get_vhost_redir(config, tpl, server_name, verbose), 4), "__SSL__": "", "__INDEX__": "", "__ACCESS_LOG__": vhost_get_access_log(config, server_name), @@ -845,23 +908,28 @@ def get_vhost( location, # type: Optional[str] server_name, # type: str default, # type: bool + verbose, # type: int ): # type: (...) -> str """Create the vhost.""" if mode == "ssl": - return get_vhost_ssl(config, tpl, docroot, proxy, location, server_name, default) + log(2, "Creating vhost type: https-only (ssl)", verbose) + return get_vhost_ssl(config, tpl, docroot, proxy, location, server_name, default, verbose) if mode == "both": + log(2, "Creating vhost type: https and http (both)", verbose) return get_vhost_ssl( - config, tpl, docroot, proxy, location, server_name, default - ) + get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default) + config, tpl, docroot, proxy, location, server_name, default, verbose + ) + get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default, verbose) if mode == "redir": + log(2, "Creating vhost type: https and redirect http to https (redir)", verbose) return get_vhost_ssl( - config, tpl, docroot, proxy, location, server_name, default - ) + get_vhost_redir(config, tpl, docroot, proxy, server_name, default) + config, tpl, docroot, proxy, location, server_name, default, verbose + ) + get_vhost_redir(config, tpl, docroot, proxy, server_name, default, verbose) - return get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default) + log(2, "Creating vhost type: http-only (plain)", verbose) + return get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default, verbose) ############################################################ @@ -869,17 +937,19 @@ def get_vhost( ############################################################ -def load_config(config_path): - # type: (str) -> Tuple[bool, Dict[str, Any], str] +def load_config(config_path, verbose): + # type: (str, int) -> Tuple[bool, Dict[str, Any], str] """Load config and merge with defaults in case not found or something is missing.""" # Load configuration file + log(2, "Loading configuration file (-c): {}".format(config_path), verbose) if os.path.isfile(config_path): succ, config, err = load_yaml(config_path) if not succ: return (False, dict(), err) else: - print("[WARN] config file not found", config_path, file=sys.stderr) + log(1, "Config file not found in: {}".format(config_path), verbose) + log(1, "Using default config settings", verbose) config = dict() # Merge config settings with program defaults (config takes precedence over defaults) @@ -888,27 +958,46 @@ def load_config(config_path): return (True, config, "") -def load_template(template_dir, o_template_dir, server): - # type: (str, str, str) -> Tuple[bool, Dict[str, Any], str] +def load_template(template_dir, o_template_dir, server, verbose): + # type: (str, str, str, int) -> Tuple[bool, Dict[str, Any], str] """Load global and optional template file and merge them.""" # Load global template file + log( + 2, + "Loading vhost template (global) (-t): {}".format( + os.path.join(template_dir, TEMPLATES[server]) + ), + verbose, + ) succ, template, err = load_yaml(os.path.join(template_dir, TEMPLATES[server])) if not succ: return ( False, dict(), - "(global template: " + os.path.join(template_dir, TEMPLATES[server]) + "): " + err, + "(global vhost template: " + + os.path.join(template_dir, TEMPLATES[server]) + + "): " + + err, ) # Load optional template file (if specified file and merge it) if o_template_dir is not None: + log( + 2, + "Loading vhost template (override) (-o): {}".format( + os.path.join(o_template_dir, TEMPLATES[server]) + ), + verbose, + ) # Template dir exists, but no file was added, give the user a warning, but do not abort if not os.path.isfile(os.path.join(o_template_dir, TEMPLATES[server])): - print( - "[WARN] override template not found: ", - os.path.join(o_template_dir, TEMPLATES[server]), - file=sys.stderr, + log( + 1, + "Override Vhost template not found: {}".format( + os.path.join(o_template_dir, TEMPLATES[server]) + ), + verbose, ) else: succ, template2, err = load_yaml(os.path.join(o_template_dir, TEMPLATES[server])) @@ -916,7 +1005,7 @@ def load_template(template_dir, o_template_dir, server): return ( False, dict(), - "(override template: " + "(override vhost template: " + os.path.join(o_template_dir, TEMPLATES[server]) + "): " + err, @@ -931,8 +1020,8 @@ def load_template(template_dir, o_template_dir, server): ############################################################ -def apply_log_settings(config): - # type: (Dict[str, Any]) -> Tuple[bool, Optional[str]] +def apply_log_settings(config, verbose): + # type: (Dict[str, Any], int) -> Tuple[bool, Optional[str]] """ This function will apply various settings for the log defines, including creating the directory itself as well as handling log file output (access @@ -940,24 +1029,32 @@ def apply_log_settings(config): """ # Symlink stdout to access logfile if config["vhost"]["log"]["access"]["stdout"]: + log(2, "Log setting: stdout -> /dev/stdout", verbose) succ, err = symlink("/dev/stdout", STDOUT_ACCESS, force=True) if not succ: return (False, err) # Symlink stderr to error logfile if config["vhost"]["log"]["error"]["stderr"]: + log(2, "Log setting: stderr -> /dev/stderr", verbose) succ, err = symlink("/dev/stderr", STDERR_ERROR, force=True) if not succ: return (False, err) # Create log dir if config["vhost"]["log"]["dir"]["create"]: + log( + 2, + "Log setting: dir -> {}".format(os.path.isdir(config["vhost"]["log"]["dir"]["path"])), + verbose, + ) if not os.path.isdir(config["vhost"]["log"]["dir"]["path"]): try: os.makedirs(config["vhost"]["log"]["dir"]["path"]) except OSError as err: return (False, "[ERR] Cannot create directory: " + str(err)) + log(2, "Log setting: Not specified", verbose) return (True, None) @@ -987,49 +1084,46 @@ def main(argv): # Validate command line arguments This will abort the program on error # This will abort the program on error - validate_args_req(name, docroot, proxy, mode, location) - validate_args_opt(config_path, tpl_dir) + validate_args_req(name, docroot, proxy, mode, location, verbose) + validate_args_opt(config_path, tpl_dir, verbose) # Load config - succ, config, err_config = load_config(config_path) + succ, config, err_config = load_config(config_path, verbose) if not succ: - print("[ERR] Error loading config", err_config, file=sys.stderr) + log(0, "Error loading config: {}".format(err_config), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) # Load template - succ, template, err_tpl = load_template(tpl_dir, o_tpl_dir, config["server"]) + succ, template, err_tpl = load_template(tpl_dir, o_tpl_dir, config["server"], verbose) if not succ: - print("[ERR] Error loading template", err_tpl, file=sys.stderr) + log(0, "Error loading tempalte: {}".format(err_tpl), verbose) + log(0, "Type --help for help", verbose) sys.exit(1) # Validate configuration file # This will abort the program on error - validate_config(config) + validate_config(config, verbose) # Retrieve fully build vhost - vhost = get_vhost(config, template, docroot, proxy, mode, location, name, default) - - if verbose: - print( - "vhostgen: [%s] Adding: %s" - % ( - time.strftime("%Y-%m-%d %H:%M:%S"), - to_str(config["vhost"]["name"]["prefix"]) - + name - + to_str(config["vhost"]["name"]["suffix"]), - ) - ) + vhost = get_vhost(config, template, docroot, proxy, mode, location, name, default, verbose) + + log( + 2, + "Using vhost name: {}".format( + to_str(config["vhost"]["name"]["prefix"]) + + name + + to_str(config["vhost"]["name"]["suffix"]) + ), + verbose, + ) if save: if not os.path.isdir(config["conf_dir"]): - print("[ERR] output conf_dir does not exist:", config["conf_dir"], file=sys.stderr) + log(0, "conf_dir does not exist: " + config["conf_dir"], verbose) sys.exit(1) if not os.access(config["conf_dir"], os.W_OK): - print( - "[ERR] directory does not have write permissions", - config["conf_dir"], - file=sys.stderr, - ) + log(0, "conf_dir does not have write permissions: " + config["conf_dir"], verbose) sys.exit(1) vhost_path = os.path.join(config["conf_dir"], name + ".conf") @@ -1037,10 +1131,11 @@ def main(argv): outfile.write(vhost) # Apply settings for logging (symlinks, mkdir) only in save mode - succ, err_apply = apply_log_settings(config) + succ, err_apply = apply_log_settings(config, verbose) if not succ: - print(err, file=sys.stderr) + log(0, str(err_apply), verbose) sys.exit(1) + log(2, "Vhost config written to: " + vhost_path, verbose) else: print(vhost) diff --git a/setup.py b/setup.py index dbe7c0a..1cfaf46 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="vhost-gen", - version="1.0.1", + version="1.0.2", description="Configurable vHost generator for Apache 2.2, Apache 2.4 and Nginx.", license="MIT", long_description=long_description, @@ -19,10 +19,9 @@ "bin/vhost-gen" ], classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable + # https://pypi.org/classifiers/ + # + # How mature is this project 'Development Status :: 5 - Production/Stable', # Indicate who your project is intended for 'Intended Audience :: Developers', @@ -35,6 +34,7 @@ "License :: OSI Approved :: MIT License", # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. + "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", # How does it run From cc187dcfa5aec5660a1dac28a72147deb619b7cb Mon Sep 17 00:00:00 2001 From: cytopia Date: Sat, 5 Dec 2020 11:04:26 +0100 Subject: [PATCH 04/14] Format according to Python Black --- bin/vhost-gen | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/vhost-gen b/bin/vhost-gen index 7fc8ac8..06fc0e5 100755 --- a/bin/vhost-gen +++ b/bin/vhost-gen @@ -742,9 +742,7 @@ def vhost_get_denies(config, template, verbose): replacer = to_str(item["alias"]) log(3, "[denies] Replacing: __REGEX__ -> {}".format(replacer), verbose) - denies.append( - str_replace(template["features"]["deny"], {"__REGEX__": replacer}) - ) + denies.append(str_replace(template["features"]["deny"], {"__REGEX__": replacer})) # Join by OS independent newlines deny = os.linesep.join(denies) log(3, "[denies] Returning: {}".format(deny), verbose) From 8eabfe8671a345e96d2986cdc701b6c06081b104 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sat, 5 Dec 2020 11:06:59 +0100 Subject: [PATCH 05/14] Fix file-lint --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index ab1640e..3ed3665 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ BINNAME = vhost-gen CONFIG = conf.yml TPLDIR = templates +FL_VERSION = 0.3 +FL_IGNORES = .git/,.github/,*.egg-info,.mypy_cache/ # ------------------------------------------------------------------------------------------------- # Default Target From 3ed41d1b994ab32ac95d1c0239d6655484ca690c Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:01:48 +0100 Subject: [PATCH 06/14] Add more logging --- bin/vhost-gen | 205 ++++++++++++++++++++++++-------------------------- 1 file changed, 100 insertions(+), 105 deletions(-) diff --git a/bin/vhost-gen b/bin/vhost-gen index 06fc0e5..31fdd7f 100755 --- a/bin/vhost-gen +++ b/bin/vhost-gen @@ -9,10 +9,9 @@ vHost creator for Apache 2.2, Apache 2.4 and Nginx. """ -############################################################ +# -------------------------------------------------------------------------------------------------- # Imports -############################################################ - +# -------------------------------------------------------------------------------------------------- from __future__ import print_function import os import sys @@ -23,13 +22,12 @@ import itertools import yaml if os.environ.get("MYPY_CHECK", False): - from typing import Optional, Iterator, List, Dict, Any, Callable, Tuple, Union - from types import CodeType + from typing import Optional, List, Dict, Any, Tuple -############################################################ -# Globals -############################################################ +# -------------------------------------------------------------------------------------------------- +# Globals +# -------------------------------------------------------------------------------------------------- APPNAME = "vhost-gen" APPREPO = "https://github.com/devilbox/vhost-gen" VERSION = "1.0.2" @@ -78,11 +76,9 @@ DEFAULT_CONFIG = { TEMPLATES = {"apache22": "apache22.yml", "apache24": "apache24.yml", "nginx": "nginx.yml"} -############################################################ +# -------------------------------------------------------------------------------------------------- # System Functions -############################################################ - - +# -------------------------------------------------------------------------------------------------- # Log info to stderr during execution based on verbosity # level: # 0: error @@ -173,11 +169,9 @@ def print_version(): print("The MIT License (MIT)") -############################################################ +# -------------------------------------------------------------------------------------------------- # Wrapper Functions -############################################################ - - +# -------------------------------------------------------------------------------------------------- def str_replace(string, replacer): # type: (str, Dict[str, Any]) -> str """Generic string replace.""" @@ -251,11 +245,9 @@ def symlink(src, dst, force=False): return (True, None) -############################################################ +# -------------------------------------------------------------------------------------------------- # Argument Functions -############################################################ - - +# -------------------------------------------------------------------------------------------------- def parse_args(argv): # type: (List[str]) -> Tuple[Any, ...] """Parse command line arguments.""" @@ -442,11 +434,9 @@ def validate_args_opt(config_path, tpl_dir, verbose): sys.exit(1) -############################################################ +# -------------------------------------------------------------------------------------------------- # Config File Functions -############################################################ - - +# -------------------------------------------------------------------------------------------------- def validate_config(config, verbose): # type: (Dict[str, Any], bool) -> None """Validate some important keys in config dict.""" @@ -469,11 +459,9 @@ def validate_config(config, verbose): # sys.exit(1) -############################################################ +# -------------------------------------------------------------------------------------------------- # Get vHost Skeleton placeholders -############################################################ - - +# -------------------------------------------------------------------------------------------------- def vhost_get_port(config, ssl): # type: (Dict[str, Any], bool) -> str """Get listen port.""" @@ -564,11 +552,9 @@ def vhost_get_error_log(config, server_name): return path -############################################################ +# -------------------------------------------------------------------------------------------------- # Get vHost Type (normal or reverse proxy -############################################################ - - +# -------------------------------------------------------------------------------------------------- def vhost_get_vhost_docroot(config, template, docroot, proxy, verbose): # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str], int) -> str """Get document root directive.""" @@ -578,8 +564,8 @@ def vhost_get_vhost_docroot(config, template, docroot, proxy, verbose): return str_replace( template["vhost_type"]["docroot"], { - "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), - "__INDEX__": vhost_get_index(config), + "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy, verbose), + "__INDEX__": vhost_get_index(config, verbose), }, ) @@ -607,11 +593,9 @@ def vhost_get_vhost_rproxy(template, proxy, location, verbose): return "" -############################################################ +# -------------------------------------------------------------------------------------------------- # Get vHost Features -############################################################ - - +# -------------------------------------------------------------------------------------------------- def vhost_get_vhost_ssl(config, template, server_name, verbose): # type: (Dict[str, Any], Dict[str, Any], str, int) -> str """Get ssl definition.""" @@ -619,7 +603,7 @@ def vhost_get_vhost_ssl(config, template, server_name, verbose): template["features"]["ssl"], { "__SSL_PATH_CRT__": to_str(vhost_get_ssl_crt_path(config, server_name)), - "__SSL_PATH_KEY__": to_str(vhost_get_ssl_key_path(config, server_name)), + "__SSL_PATH_KEY__": to_str(vhost_get_ssl_key_path(config, server_name, verbose)), "__SSL_PROTOCOLS__": to_str(config["vhost"]["ssl"]["protocols"]), "__SSL_HONOR_CIPHER_ORDER__": to_str(config["vhost"]["ssl"]["honor_cipher_order"]), "__SSL_CIPHERS__": to_str(config["vhost"]["ssl"]["ciphers"]), @@ -627,8 +611,8 @@ def vhost_get_vhost_ssl(config, template, server_name, verbose): ) -def vhost_get_vhost_redir(config, template, server_name, verbose): - # type: (Dict[str, Any], Dict[str, Any], str, int) -> str +def vhost_get_vhost_redir(config, template, server_name): + # type: (Dict[str, Any], Dict[str, Any], str) -> str """Get redirect to ssl definition.""" return str_replace( template["features"]["redirect"], @@ -651,65 +635,87 @@ def vhost_get_ssl_crt_path(config, server_name): return os.path.join(path, name) -def vhost_get_ssl_key_path(config, server_name): - # type: (Dict[str, Any], str) -> str +def vhost_get_ssl_key_path(config, server_name, verbose): + # type: (Dict[str, Any], str, int) -> str """Get ssl key path""" prefix = to_str(config["vhost"]["name"]["prefix"]) suffix = to_str(config["vhost"]["name"]["suffix"]) name = prefix + server_name + suffix + ".key" + log(3, "[ssl] Retrieving cfg 'ssl' name: {}".format(name), verbose) path = to_str(config["vhost"]["ssl"]["dir_crt"]) - return os.path.join(path, name) + log(3, "[ssl] Retrieving cfg 'ssl' path: {}".format(path), verbose) + + output = os.path.join(path, name) + log(3, "[ssl] Returning: {}".format(output), verbose) + return output -def vhost_get_docroot_path(config, docroot, proxy): - # type: (Dict[str, Any], Optional[str], Optional[str]) -> str +def vhost_get_docroot_path(config, docroot, proxy, verbose): + # type: (Dict[str, Any], Optional[str], Optional[str], int) -> str """Get path of document root.""" if proxy is not None: + log(3, "[docroot] Retrieving cfg 'docroot': None, we're using Proxy", verbose) return "" suffix = to_str(config["vhost"]["docroot"]["suffix"]) - path = os.path.join(str(docroot), suffix) - return path + output = os.path.join(str(docroot), suffix) + log(3, "[docroot] Retrieving cfg 'docroot': {}".format(output), verbose) + log(3, "[docroot] Returning: {}".format(output), verbose) + return output -def vhost_get_index(config): - # type: (Dict[str, Any]) -> str +def vhost_get_index(config, verbose): + # type: (Dict[str, Any], int) -> str """Get index.""" if "index" in config["vhost"] and config["vhost"]["index"]: elem = config["vhost"]["index"] + log(3, "[index] Retrieving cfg 'index': {}".format(elem), verbose) else: elem = DEFAULT_CONFIG["vhost"]["index"] # type: ignore + log(3, "[index] Retrieving default 'index': {}".format(elem), verbose) + output = " ".join(elem) + log(3, "[index] Returning: {}".format(output), verbose) return " ".join(elem) -def vhost_get_php_fpm(config, template, docroot, proxy): - # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str]) -> str +def vhost_get_php_fpm(config, template, docroot, proxy, verbose): + # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[str], int) -> str """Get PHP FPM directive. If using reverse proxy, PHP-FPM will be disabled.""" if proxy is not None: + log(3, "[php_fpm] Retrieving cfg 'php_fpm': None, we're using Proxy", verbose) return "" # Get PHP-FPM php_fpm = "" if config["vhost"]["php_fpm"]["enable"]: + log( + 3, + "[php_fpm] Retrieving cfg 'php_fpm': {}:{}".format( + config["vhost"]["php_fpm"]["address"], config["vhost"]["php_fpm"]["port"] + ), + verbose, + ) php_fpm = str_replace( template["features"]["php_fpm"], { "__PHP_ADDR__": to_str(config["vhost"]["php_fpm"]["address"]), "__PHP_PORT__": to_str(config["vhost"]["php_fpm"]["port"]), "__PHP_TIMEOUT__": to_str(config["vhost"]["php_fpm"]["timeout"]), - "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), + "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy, verbose), }, ) + log(3, "[php_fpm] Returning: {}".format(php_fpm), verbose) return php_fpm -def vhost_get_aliases(config, template): - # type: (Dict[str, Any], Dict[str, Any]) -> str +def vhost_get_aliases(config, template, verbose): + # type: (Dict[str, Any], Dict[str, Any], int) -> str """Get virtual host alias directives.""" aliases = [] for item in config["vhost"]["alias"]: + log(3, "[alias] Retrieving cfg 'alias': {}".format(item["alias"]), verbose) # Add optional xdomain request if enabled xdomain_request = "" if "xdomain_request" in item: @@ -730,7 +736,9 @@ def vhost_get_aliases(config, template): ) ) # Join by OS independent newlines - return os.linesep.join(aliases) + output = os.linesep.join(aliases) + log(3, "[alias] Returning: {}".format(output), verbose) + return output def vhost_get_denies(config, template, verbose): @@ -738,15 +746,14 @@ def vhost_get_denies(config, template, verbose): """Get virtual host deny alias directives.""" denies = [] for item in config["vhost"]["deny"]: - log(3, "[denies] Retrieving cfg 'deny': {}".format(item), verbose) - - replacer = to_str(item["alias"]) - log(3, "[denies] Replacing: __REGEX__ -> {}".format(replacer), verbose) - denies.append(str_replace(template["features"]["deny"], {"__REGEX__": replacer})) + log(3, "[denies] Retrieving cfg 'deny': {}".format(item["alias"]), verbose) + denies.append( + str_replace(template["features"]["deny"], {"__REGEX__": to_str(item["alias"])}) + ) # Join by OS independent newlines - deny = os.linesep.join(denies) - log(3, "[denies] Returning: {}".format(deny), verbose) - return deny + output = os.linesep.join(denies) + log(3, "[denies] Returning: {}".format(output), verbose) + return output def vhost_get_server_status(config, template, verbose): @@ -755,33 +762,28 @@ def vhost_get_server_status(config, template, verbose): status = "" enable = config["vhost"]["server_status"]["enable"] - log(3, "[status] Retrieving cfg 'enable': {}".format(enable), verbose) + log(3, "[status] Retrieving cfg 'enable': {}".format(enable), verbose) if enable: status = template["features"]["server_status"] - log(3, "[status] Retrieving tpl 'server_status': {}".format(status), verbose) + log(3, "[status] Retrieving tpl 'server_status': {}".format(status), verbose) - replacer = to_str(config["vhost"]["server_status"]["alias"]) - log(3, "[status] Replacing: __REGEX__ -> {}".format(replacer), verbose) - sstatus = str_replace(status, {"__REGEX__": replacer}) - - log(3, "[status] Returning: {}".format(sstatus), verbose) - return sstatus + output = str_replace(status, {"__REGEX__": to_str(config["vhost"]["server_status"]["alias"])}) + log(3, "[status] Returning: {}".format(output), verbose) + return output def vhost_get_custom_section(config, verbose): # type: (Dict[str, Any], int) -> str """Get virtual host custom directives.""" - custom = to_str(config["custom"]) - log(3, "[custom] Retrieving cfg 'custom': {}".format(custom), verbose) - log(3, "[custom] Returning': {}".format(custom), verbose) - return custom + output = to_str(config["custom"]) + log(3, "[custom] Retrieving cfg 'custom': {}".format(output), verbose) + log(3, "[custom] Returning': {}".format(output), verbose) + return output -############################################################ +# -------------------------------------------------------------------------------------------------- # vHost create -############################################################ - - +# -------------------------------------------------------------------------------------------------- def get_vhost_plain( config, # type: Dict[str, Any] tpl, # type: Dict[str, Any] @@ -800,7 +802,7 @@ def get_vhost_plain( "__PORT__": vhost_get_port(config, False), "__HTTP_PROTO__": vhost_get_http_proto(config, False), "__DEFAULT_VHOST__": vhost_get_default_server(config, default), - "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), + "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy, verbose), "__VHOST_NAME__": vhost_get_server_name(config, server_name, default), "__VHOST_DOCROOT__": str_indent( vhost_get_vhost_docroot(config, tpl, docroot, proxy, verbose), 4 @@ -810,11 +812,11 @@ def get_vhost_plain( ), "__REDIRECT__": "", "__SSL__": "", - "__INDEX__": vhost_get_index(config), + "__INDEX__": vhost_get_index(config, verbose), "__ACCESS_LOG__": vhost_get_access_log(config, server_name), "__ERROR_LOG__": vhost_get_error_log(config, server_name), - "__PHP_FPM__": str_indent(vhost_get_php_fpm(config, tpl, docroot, proxy), 4), - "__ALIASES__": str_indent(vhost_get_aliases(config, tpl), 4), + "__PHP_FPM__": str_indent(vhost_get_php_fpm(config, tpl, docroot, proxy, verbose), 4), + "__ALIASES__": str_indent(vhost_get_aliases(config, tpl, verbose), 4), "__DENIES__": str_indent(vhost_get_denies(config, tpl, verbose), 4), "__SERVER_STATUS__": str_indent(vhost_get_server_status(config, tpl, verbose), 4), "__CUSTOM__": str_indent(vhost_get_custom_section(config, verbose), 4), @@ -840,7 +842,7 @@ def get_vhost_ssl( "__PORT__": vhost_get_port(config, True), "__HTTP_PROTO__": vhost_get_http_proto(config, True), "__DEFAULT_VHOST__": vhost_get_default_server(config, default), - "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), + "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy, verbose), "__VHOST_NAME__": vhost_get_server_name(config, server_name, default), "__VHOST_DOCROOT__": str_indent( vhost_get_vhost_docroot(config, tpl, docroot, proxy, verbose), 4 @@ -850,11 +852,11 @@ def get_vhost_ssl( ), "__REDIRECT__": "", "__SSL__": str_indent(vhost_get_vhost_ssl(config, tpl, server_name, verbose), 4), - "__INDEX__": vhost_get_index(config), + "__INDEX__": vhost_get_index(config, verbose), "__ACCESS_LOG__": vhost_get_access_log(config, server_name + "_ssl"), "__ERROR_LOG__": vhost_get_error_log(config, server_name + "_ssl"), - "__PHP_FPM__": str_indent(vhost_get_php_fpm(config, tpl, docroot, proxy), 4), - "__ALIASES__": str_indent(vhost_get_aliases(config, tpl), 4), + "__PHP_FPM__": str_indent(vhost_get_php_fpm(config, tpl, docroot, proxy, verbose), 4), + "__ALIASES__": str_indent(vhost_get_aliases(config, tpl, verbose), 4), "__DENIES__": str_indent(vhost_get_denies(config, tpl, verbose), 4), "__SERVER_STATUS__": str_indent(vhost_get_server_status(config, tpl, verbose), 4), "__CUSTOM__": str_indent(vhost_get_custom_section(config, verbose), 4), @@ -879,11 +881,11 @@ def get_vhost_redir( "__PORT__": vhost_get_port(config, False), "__HTTP_PROTO__": vhost_get_http_proto(config, False), "__DEFAULT_VHOST__": vhost_get_default_server(config, default), - "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy), + "__DOCUMENT_ROOT__": vhost_get_docroot_path(config, docroot, proxy, verbose), "__VHOST_NAME__": vhost_get_server_name(config, server_name, default), "__VHOST_DOCROOT__": "", "__VHOST_RPROXY__": "", - "__REDIRECT__": str_indent(vhost_get_vhost_redir(config, tpl, server_name, verbose), 4), + "__REDIRECT__": str_indent(vhost_get_vhost_redir(config, tpl, server_name), 4), "__SSL__": "", "__INDEX__": "", "__ACCESS_LOG__": vhost_get_access_log(config, server_name), @@ -930,11 +932,9 @@ def get_vhost( return get_vhost_plain(config, tpl, docroot, proxy, location, server_name, default, verbose) -############################################################ +# -------------------------------------------------------------------------------------------------- # Load configs and templates -############################################################ - - +# -------------------------------------------------------------------------------------------------- def load_config(config_path, verbose): # type: (str, int) -> Tuple[bool, Dict[str, Any], str] """Load config and merge with defaults in case not found or something is missing.""" @@ -1013,11 +1013,9 @@ def load_template(template_dir, o_template_dir, server, verbose): return (True, template, "") -############################################################ +# -------------------------------------------------------------------------------------------------- # Post actions -############################################################ - - +# -------------------------------------------------------------------------------------------------- def apply_log_settings(config, verbose): # type: (Dict[str, Any], int) -> Tuple[bool, Optional[str]] """ @@ -1056,11 +1054,9 @@ def apply_log_settings(config, verbose): return (True, None) -############################################################ +# -------------------------------------------------------------------------------------------------- # Main Function -############################################################ - - +# -------------------------------------------------------------------------------------------------- def main(argv): # type: (List[Any]) -> None """Main entrypoint.""" @@ -1138,9 +1134,8 @@ def main(argv): print(vhost) -############################################################ +# -------------------------------------------------------------------------------------------------- # Main Entry Point -############################################################ - +# -------------------------------------------------------------------------------------------------- if __name__ == "__main__": main(sys.argv[1:]) From 18991fc251d1771219870b047e253daf0a6ad013 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:02:11 +0100 Subject: [PATCH 07/14] Add pylint --- .github/workflows/code-pylint.yml | 30 ++++++++++++++++++++++++++++++ Makefile | 8 ++++++++ setup.cfg | 18 ++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 .github/workflows/code-pylint.yml diff --git a/.github/workflows/code-pylint.yml b/.github/workflows/code-pylint.yml new file mode 100644 index 0000000..51d93ea --- /dev/null +++ b/.github/workflows/code-pylint.yml @@ -0,0 +1,30 @@ +--- + +### +### Code style +### + +name: mypy +on: + pull_request: + push: + branches: + - master + tags: + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + target: + - pylint + name: "[ ${{ matrix.target }} ]" + steps: + - name: Checkout repository + uses: actions/checkout@master + + - name: "${{ matrix.target }}" + run: | + make _code-${{ matrix.target }} diff --git a/Makefile b/Makefile index 3ed3665..992af68 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,7 @@ _lint-version: # ------------------------------------------------------------------------------------------------- code: _code-pycodestyle code: _code-pydocstyle +code: _code-pylint code: _code-black code: _code-mypy @@ -95,6 +96,13 @@ _code-pydocstyle: @echo "# -------------------------------------------------------------------- #" docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/pydocstyle $(BINPATH)$(BINNAME) +.PHONY: _code-pylint +_code-pylint: + @echo "# -------------------------------------------------------------------- #" + @echo "# Check pylint" + @echo "# -------------------------------------------------------------------- #" + docker run --rm $$(tty -s && echo "-it" || echo) -v $(PWD):/data cytopia/pylint --rcfile=setup.cfg $(BINPATH)$(BINNAME) + .PHONY: _code-black _code-black: @echo "# -------------------------------------------------------------------- #" diff --git a/setup.cfg b/setup.cfg index b5ce024..2de752c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,9 +4,27 @@ description-file = README.md [bdist_wheel] universal=1 +[pydocstyle] +convention = google + [pycodestyle] max-line-length = 100 +[flake8] +max-line-length = 100 + +[pylint] +# useless-object-inheritance: don't lint useless-object-inheritance to stary Python2/3 compatible +# bad-continuation: let Python Black take care of this +# unidiomatic-typecheck: Need to check if int or bool and this doesnt work with isinstance() +disable = useless-object-inheritance, bad-continuation, unidiomatic-typecheck, import-error +module-rgx=[-a-z]+ +max-branches = 15 +max-args = 9 +max-locals = 21 +max-module-lines = 2000 +ignore-imports=yes + [mypy] # Display show_error_context = True From 39e9e12e6b3bcec90e97c5335b0d8114327b53ae Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:02:36 +0100 Subject: [PATCH 08/14] Clean repo --- .pylintrc | 425 ------------------------------------------------------ 1 file changed, 425 deletions(-) delete mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 8a96888..0000000 --- a/.pylintrc +++ /dev/null @@ -1,425 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=8 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=15 - -# Maximum number of locals for function / method body -max-locals=19 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception From fe11d5503b12388c209906b22fabdcc900ddb5ec Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:03:03 +0100 Subject: [PATCH 09/14] Update release date --- bin/vhost-gen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/vhost-gen b/bin/vhost-gen index 31fdd7f..88e037f 100755 --- a/bin/vhost-gen +++ b/bin/vhost-gen @@ -31,7 +31,7 @@ if os.environ.get("MYPY_CHECK", False): APPNAME = "vhost-gen" APPREPO = "https://github.com/devilbox/vhost-gen" VERSION = "1.0.2" -RELDATE = "2020-12-05" +RELDATE = "2020-12-06" # Default paths CONFIG_PATH = "/etc/vhost-gen/conf.yml" From 388c3870486212cefff8ed0b3ef062b28394f437 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:12:59 +0100 Subject: [PATCH 10/14] Soucrce cleanup --- bin/vhost-gen | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/bin/vhost-gen b/bin/vhost-gen index 88e037f..ea4268e 100755 --- a/bin/vhost-gen +++ b/bin/vhost-gen @@ -449,16 +449,6 @@ def validate_config(config, verbose): sys.exit(1) -# # Validate if log dir can be created -# log_dir = config['vhost']['log']['dir']['path'] -# if config['vhost']['log']['dir']['create']: -# if not os.path.isdir(log_dir): -# if not os.access(os.path.dirname(log_dir), os.W_OK): -# print('[ERR] log directory does not exist and cannot be created:', log_dir, -# file=sys.stderr) -# sys.exit(1) - - # -------------------------------------------------------------------------------------------------- # Get vHost Skeleton placeholders # -------------------------------------------------------------------------------------------------- @@ -602,7 +592,7 @@ def vhost_get_vhost_ssl(config, template, server_name, verbose): return str_replace( template["features"]["ssl"], { - "__SSL_PATH_CRT__": to_str(vhost_get_ssl_crt_path(config, server_name)), + "__SSL_PATH_CRT__": to_str(vhost_get_ssl_crt_path(config, server_name, verbose)), "__SSL_PATH_KEY__": to_str(vhost_get_ssl_key_path(config, server_name, verbose)), "__SSL_PROTOCOLS__": to_str(config["vhost"]["ssl"]["protocols"]), "__SSL_HONOR_CIPHER_ORDER__": to_str(config["vhost"]["ssl"]["honor_cipher_order"]), @@ -623,16 +613,20 @@ def vhost_get_vhost_redir(config, template, server_name): ) -def vhost_get_ssl_crt_path(config, server_name): - # type: (Dict[str, Any], str) -> str +def vhost_get_ssl_crt_path(config, server_name, verbose): + # type: (Dict[str, Any], str, int) -> str """Get ssl crt path""" - prefix = to_str(config["vhost"]["name"]["prefix"]) suffix = to_str(config["vhost"]["name"]["suffix"]) name = prefix + server_name + suffix + ".crt" + log(3, "[ssl-crt] Retrieving cfg 'ssl' name: {}".format(name), verbose) path = to_str(config["vhost"]["ssl"]["dir_crt"]) - return os.path.join(path, name) + log(3, "[ssl-crt] Retrieving cfg 'ssl' path: {}".format(path), verbose) + + output = os.path.join(path, name) + log(3, "[ssl-crt] Returning: {}".format(output), verbose) + return output def vhost_get_ssl_key_path(config, server_name, verbose): @@ -641,13 +635,13 @@ def vhost_get_ssl_key_path(config, server_name, verbose): prefix = to_str(config["vhost"]["name"]["prefix"]) suffix = to_str(config["vhost"]["name"]["suffix"]) name = prefix + server_name + suffix + ".key" - log(3, "[ssl] Retrieving cfg 'ssl' name: {}".format(name), verbose) + log(3, "[ssl-key] Retrieving cfg 'ssl' name: {}".format(name), verbose) path = to_str(config["vhost"]["ssl"]["dir_crt"]) - log(3, "[ssl] Retrieving cfg 'ssl' path: {}".format(path), verbose) + log(3, "[ssl-key] Retrieving cfg 'ssl' path: {}".format(path), verbose) output = os.path.join(path, name) - log(3, "[ssl] Returning: {}".format(output), verbose) + log(3, "[ssl-key] Returning: {}".format(output), verbose) return output @@ -677,7 +671,7 @@ def vhost_get_index(config, verbose): output = " ".join(elem) log(3, "[index] Returning: {}".format(output), verbose) - return " ".join(elem) + return output def vhost_get_php_fpm(config, template, docroot, proxy, verbose): From 417db553469dddf8fa0e32e1f55fc2dbb1f49af4 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:21:55 +0100 Subject: [PATCH 11/14] Add GitHub Actions --- .github/workflows/fuzzing.yml | 45 +++++++++++++++++++++++++++++++++++ .github/workflows/testing.yml | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/fuzzing.yml diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml new file mode 100644 index 0000000..f7abde7 --- /dev/null +++ b/.github/workflows/fuzzing.yml @@ -0,0 +1,45 @@ +--- + +### +### Lints all generic and json files in the whole git repository +### + +name: fuzzing +on: + pull_request: + push: + branches: + - master + tags: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + version: + - '2.7' + - '3.6' + - '3.7' + - '3.8' + + name: "[ ${{ matrix.target }} ${{ matrix.version}} ]" + steps: + - name: Checkout repository + uses: actions/checkout@master + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.version }} + architecture: x64 + + - name: Fuzzing - before install + run: | + .ci/fuzzy.sh + + - name: Fuzzing - after install + run: | + sudo make install + .ci/fuzzy.sh diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 67a18b0..95fc8ae 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -31,7 +31,7 @@ jobs: - name: Checkout repository uses: actions/checkout@master - - name: Lint + - name: Test run: | make ${target} VERSION=${version} env: From 596fffb374b2b5b68a36defc6cd6d34458c87438 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:25:17 +0100 Subject: [PATCH 12/14] Fix workflows --- .github/workflows/code-pylint.yml | 2 +- .github/workflows/fuzzing.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-pylint.yml b/.github/workflows/code-pylint.yml index 51d93ea..7ee85cd 100644 --- a/.github/workflows/code-pylint.yml +++ b/.github/workflows/code-pylint.yml @@ -4,7 +4,7 @@ ### Code style ### -name: mypy +name: pylint on: pull_request: push: diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index f7abde7..78bf46e 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -35,6 +35,10 @@ jobs: python-version: ${{ matrix.version }} architecture: x64 + - name: Install Python requirements + run: | + sudo pip install -r requirements.txt + - name: Fuzzing - before install run: | .ci/fuzzy.sh From 362cc782d53b1058142c0bde905cdeadccaf9e5d Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:30:25 +0100 Subject: [PATCH 13/14] Install pip in CI --- .github/workflows/fuzzing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 78bf46e..05e4313 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -37,7 +37,7 @@ jobs: - name: Install Python requirements run: | - sudo pip install -r requirements.txt + pip install -r requirements.txt - name: Fuzzing - before install run: | From 01a9cceb7e1d8930758544465b8e37c86bb0f47c Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 6 Dec 2020 12:38:58 +0100 Subject: [PATCH 14/14] Remove Travis CI --- .travis.yml | 108 ---------------------------------------------------- README.md | 15 ++++++-- 2 files changed, 12 insertions(+), 111 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 333fc95..0000000 --- a/.travis.yml +++ /dev/null @@ -1,108 +0,0 @@ -### -### Sudo required -### -dist: trusty -sudo: required -#sudo: false - - -### -### Set language -### -language: python - - -### -### Python Build Matrix -### -python: - - "2.7" - - "3.5" - - "3.6" - - -### -### Install required software -### -before_install: - # Install required tools - #- sudo apt-get update - #- sudo apt-get install -y pep8 pylint - - # Create directories - - sudo rm -rf /etc/nginx/conf.d || true - - sudo rm -rf /etc/httpd/conf.d || true - - sudo rm -rf /etc/apache2/conf.d || true - - - sudo mkdir -p /etc/nginx/conf.d - - sudo mkdir -p /etc/httpd/conf.d - - sudo mkdir -p /etc/apache2/conf.d - - - sudo mkdir -p /var/log/nginx - - sudo mkdir -p /var/log/httpd - - sudo mkdir -p /var/log/apache2 - - # Add open permissions - - sudo chmod 0777 /etc/nginx/conf.d - - sudo chmod 0777 /etc/httpd/conf.d - - sudo chmod 0777 /etc/apache2/conf.d - - - sudo chmod 0777 /var/log/nginx - - sudo chmod 0777 /var/log/httpd - - sudo chmod 0777 /var/log/apache2 - - -### -### -### -install: - # Install - - pip install -r requirements.txt - - pip install pylint flake8 pycodestyle - # Show versions - - pylint --version || true - - flake8 --version || true - - pycodestyle --version || true - - -### -### Show versions -### -before_script: - # Show versions - - ./bin/vhost-gen --help - - ./bin/vhost-gen --version - - -### -### Run tests -### -script: - # Coding conventions - - make lint - - # Outputs - - make test - - # Saving (nginx) - - ./bin/vhost-gen -p ./ -n name -t etc/templates/ -c etc/conf.yml -s - - cat /etc/nginx/conf.d/name.conf - - # Saving (nginx example) - - ./bin/vhost-gen -p ./ -n name -t etc/templates/ -c examples/conf.nginx.yml -s - - cat /etc/nginx/conf.d/name.conf - - # Saving (apache22 example) - - ./bin/vhost-gen -p ./ -n name -t etc/templates/ -c examples/conf.apache22.yml -s - - cat /etc//httpd/conf.d/name.conf - - # Saving (apache24 example) - - ./bin/vhost-gen -p ./ -n name -t etc/templates/ -c examples/conf.apache24.yml -s - - cat /etc/apache2/conf.d/name.conf - - # Find flaws via fuzzying (before installing config files) - - time .ci/fuzzy.sh - - # Find flaws via fuzzying (after installing config files) - - sudo make install - - time .ci/fuzzy.sh diff --git a/README.md b/README.md index 3ad4e8a..e990347 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,18 @@ [![PyPI - Implementation](https://img.shields.io/pypi/implementation/vhost-gen)](https://pypi.org/project/vhost-gen/) [![PyPI - License](https://img.shields.io/pypi/l/vhost-gen)](https://pypi.org/project/vhost-gen/) -[![Build Status](https://github.com/devilbox/vhost-gen/workflows/linting/badge.svg)](https://github.com/devilbox/vhost-gen/actions?workflow=linting) -[![Build Status](https://github.com/devilbox/vhost-gen/workflows/testing/badge.svg)](https://github.com/devilbox/vhost-gen/actions?workflow=testing) -[![Build Status](https://travis-ci.org/devilbox/vhost-gen.svg?branch=master)](https://travis-ci.org/devilbox/vhost-gen) +**Continuous Integration** + +[![testing](https://github.com/devilbox/vhost-gen/workflows/testing/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Atesting) +[![fuzzing](https://github.com/devilbox/vhost-gen/workflows/fuzzing/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Afuzzing) + +[![linting](https://github.com/devilbox/vhost-gen/workflows/linting/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Alinting) +[![pylint](https://github.com/devilbox/vhost-gen/workflows/pylint/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Apylint) +[![black](https://github.com/devilbox/vhost-gen/workflows/black/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Ablack) +[![mypy](https://github.com/devilbox/vhost-gen/workflows/mypy/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Amypy) +[![pycode](https://github.com/devilbox/vhost-gen/workflows/pycode/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Apycode) +[![pydoc](https://github.com/devilbox/vhost-gen/workflows/pydoc/badge.svg)](https://github.com/devilbox/vhost-gen/actions?query=workflow%3Apydoc) + **[vhost-gen](bin/vhost-gen)** will dynamically generate **vhost** or **reverse proxy** configuration files for Apache 2.2, Apache 2.4 and Nginx depending on what you have set in [conf.yml](etc/conf.yml). This makes it easy to switch between different web servers while keeping the exact same functionality.