From 0c34afbd3cdaaaba758d6204280df600b632245b Mon Sep 17 00:00:00 2001 From: ToMe25 <38815969+ToMe25@users.noreply.github.com> Date: Sun, 10 Mar 2024 20:26:53 +0100 Subject: [PATCH] Allow ETag based caching of static web pages Send the md5 hash of static files as the file etag, for all static files. Send a 304 Not Modified response for requests containing a "If-None-Match" header with the correct etag. The server now sends a Cache-Control header with the value "public, no-cache" for static files and "no-store" for dynamic pages. Other changes: * Fix embedded files always needing to be rebuilt * Merge Accept openmetrics and AcceptEncoding gzip check into csvHeaderContains * Make uzlib_gzip_wrapper a bit more resilient against missing input --- .gitignore | 3 + TODO.md | 8 +- .../src/uzlib_gzip_wrapper.cpp | 27 ++- platformio.ini | 6 +- shared/compress_web.py | 24 +- shared/generate_hash_header.py | 133 +++++++++++ shared/gzip_compressing_stream.py | 4 +- shared/read_ota_pass.py | 3 +- src/AsyncHeadOnlyResponse.cpp | 3 +- src/generated/README.md | 3 + src/generated/web_file_hashes.h | 45 ++++ src/prometheus.cpp | 11 +- src/prometheus.h | 8 - src/webhandler.cpp | 210 ++++++++++++++---- src/webhandler.h | 190 +++++++++++++++- 15 files changed, 587 insertions(+), 91 deletions(-) create mode 100755 shared/generate_hash_header.py mode change 100644 => 100755 shared/gzip_compressing_stream.py create mode 100644 src/generated/README.md create mode 100644 src/generated/web_file_hashes.h diff --git a/.gitignore b/.gitignore index 8575603..9aefad9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ # Python __pycache__ +# MyPy +.mypy_cache + # Credentials wifissid.txt wifipass.txt diff --git a/TODO.md b/TODO.md index 1c3e26f..d37fb3d 100644 --- a/TODO.md +++ b/TODO.md @@ -12,13 +12,13 @@ Not all of these are necessarily going to be implemented at all. * Use an asynchronous DHT library(for example https://github.com/bertmelis/esp32DHT/)? * Add (stream?) compression to uzlib_gzip_wrapper * Change callback based uzlib_ungzip_wrapper to use C++ function objects instead of C function pointers + * Consider using PIO middleware or SCons compilation callback to generate compressed web files(into build dir?) ## Web Interface * Add error message to web interface if measurement fails * Add a javescript temperature and humidity graph to the web interface? * Remove not measured values(for example humidity for the DS18B20) * Add degrees fahrenheit mode to web interface(clientside setting) - * Allow caching of static resources using a hash of their content as an ETag * Add theme switcher to web interface(clientside setting) * Web server IPv6 support * Add current commit to prometheus info and HTTP Server header @@ -31,6 +31,10 @@ Not all of these are necessarily going to be implemented at all. * Add MQTT state json * Cleanup MQTT code * Add MQTT discovery support(https://www.home-assistant.io/docs/mqtt/discovery/) - * Add WiFi info to prometheus metrics * Dynamically gzip compress metrics page? * Implement actual prometheus library as external project + * Add prometheus info metrics esptherm_build_info, esptherm_network_info, esptherm_module_info, and esptherm_sensor_info + * Add MQTT metrics to prometheus + * Add prometheus metrics for HTTP response times and sizes + * Add prometheus metrics for push statistics, if DSM is disabled + * Add measurement error metrics diff --git a/lib/uzlib_gzip_wrapper/src/uzlib_gzip_wrapper.cpp b/lib/uzlib_gzip_wrapper/src/uzlib_gzip_wrapper.cpp index 32b234d..124e7bb 100644 --- a/lib/uzlib_gzip_wrapper/src/uzlib_gzip_wrapper.cpp +++ b/lib/uzlib_gzip_wrapper/src/uzlib_gzip_wrapper.cpp @@ -31,22 +31,31 @@ uzlib_ungzip_wrapper::uzlib_ungzip_wrapper(const uint8_t *cmp_start, wsize = -15; } + void *dict = NULL; + if (cmp_end < cmp_start + 29) { + log_e("Compressed buffer too small."); + log_i("A gzip compressed 0 byte file is 29 bytes in size."); + log_i("The given file was %d bytes.", cmp_end - cmp_start); + } else { + dict = malloc(pow(2, -wsize)); + + // Read uncompressed size from compressed file. + dlen = cmp_end[-1]; + dlen = 256 * dlen + cmp_end[-2]; + dlen = 256 * dlen + cmp_end[-3]; + dlen = 256 * dlen + cmp_end[-4]; + + } + decomp = new uzlib_uncomp; - void *dict = malloc(pow(2, -wsize)); // Try anyways, since small files can be decompressed without one. - if (!dict) { + if (dict == NULL) { log_e("Failed to allocate decompression dict."); } - // Read uncompressed size from compressed file. - dlen = cmp_end[-1]; - dlen = 256 * dlen + cmp_end[-2]; - dlen = 256 * dlen + cmp_end[-3]; - dlen = 256 * dlen + cmp_end[-4]; - uzlib_uncompress_init(decomp, dict, pow(2, -wsize)); decomp->source = cmp_start; - decomp->source_limit = cmp_end - 4; + decomp->source_limit = cmp_end - 4 >= cmp_start ? cmp_end - 4 : cmp_start; decomp->source_read_cb = NULL; uzlib_gzip_parse_header(decomp); } diff --git a/platformio.ini b/platformio.ini index 9602645..91cf64d 100644 --- a/platformio.ini +++ b/platformio.ini @@ -28,7 +28,9 @@ default_envs = framework = arduino monitor_speed = 115200 upload_speed = 921600 -extra_scripts = post:shared/compress_web.py +extra_scripts = + pre:shared/compress_web.py + pre:shared/generate_hash_header.py lib_deps = ArduinoOTA ESPAsyncWebServer = https://github.com/me-no-dev/ESPAsyncWebServer.git @@ -124,7 +126,7 @@ build_flags = [env:esp_wroom_02_ota] extends = env:esp_wroom_02 upload_protocol = espota -upload_port = esp-wifi-thermometer.local +upload_port = esp-wifi-Thermometer.local extra_scripts = ${env.extra_scripts} post:shared/read_ota_pass.py diff --git a/shared/compress_web.py b/shared/compress_web.py index e1eef79..f2e69a0 100755 --- a/shared/compress_web.py +++ b/shared/compress_web.py @@ -1,16 +1,22 @@ -#!/usr/bin/python - -Import ("env") +#!/usr/bin/env python3 from enum import Enum import os -import os.path as path +from os import path +import sys from gzip_compressing_stream import GzipCompressingStream -# Unquoted spaces will be removed from these. +try: + Import ("env") # type: ignore[name-defined] +except: + print("Failed to load platformio environment!", file=sys.stderr) + exit(1) + +# Text files to potentially remove the spaces from, and compress. input_text_files = [ 'src/html/index.html', 'src/html/error.html', 'src/html/main.css', 'src/html/index.js', 'src/html/manifest.json', 'images/favicon.svg' ] +# Binary files to compress without modifying. input_binary_files = [ 'images/favicon.ico', 'images/favicon.png' ] # These files will not be gzip compressed, just copied and potentially stripped of spaces. @@ -161,11 +167,11 @@ def compress_file(input, text): Whether the input file is a text file. """ + do_gzip = not input in input_gzip_blacklist + input = path.join(env.subst('$PROJECT_DIR'), input) filename = path.basename(input) minify = text - do_gzip = not input in input_gzip_blacklist - if env.get('BUILD_TYPE') == "debug": print("Debug mode detected, not minifying text files.") minify = False @@ -215,12 +221,8 @@ def compress_file(input, text): filename = path.basename(file) else: filename = path.basename(file) + ".gz" - # Always build because the output will depend on whether we are building in debug mode. - env.AlwaysBuild(f"$BUILD_DIR/{filename}.txt.o") compress_file(file, True) for file in input_binary_files: filename = path.basename(file) + ".gz" - # Always build these because I'm too lazy to check whether they actually changed. - env.AlwaysBuild(f"$BUILD_DIR/{filename}.txt.o") compress_file(file, False) diff --git a/shared/generate_hash_header.py b/shared/generate_hash_header.py new file mode 100755 index 0000000..3d74639 --- /dev/null +++ b/shared/generate_hash_header.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +import hashlib +import os +from os import path, PathLike +import sys +from typing import Dict, Final, List + +try: + Import ("env") # type: ignore[name-defined] +except: + print("Failed to load platformio environment!", file=sys.stderr) + sys.exit(1) + +# The file endings of files to generate hashes for. +# Files also have to be in the directory specified in "static_dir" for a hash to be calculated. +static_types: Final[List[str]] = ['.gz', '.png', '.ico', '.html', '.css', '.js'] + +# The directory containing the files to calculate a hash for. +static_dir: Final[str] = 'data' + +# A list of files that would be considered static, but should not have their hashes calculated. +static_file_blacklist: Final[List[str]] = [path.join('data', 'index.html'), path.join('data', 'error.html')] + +# The path of the header file to generated. +hash_header_path: Final[str] = path.join(env.subst('$PROJECT_SRC_DIR'), 'generated', 'web_file_hashes.h') # type: ignore[name-defined] + + +def hash_file(path: PathLike | str) -> str: + """Calculates the md5 hash of a file. + + This function reads the content of a file and calculates its md5 hash. + Note that this function reads the entire file into memory at once, so it should not be used for large files. + + @param path: The path of the file to hash. + @return The hex representation of the resulting hash. + @raise IOError: If opening or reading the file fails. + """ + with open(path, 'rb') as file: + md5 = hashlib.md5(file.read()) + return md5.hexdigest() + + +def hash_files(paths: List[PathLike] | List[str]) -> Dict[str, str]: + """Calculates the hashes for a list of files. + + Creates a dictionary mapping from the filename to its md5 hash. + + @param paths: The paths of the files to hash. + @return A dictionary containing the file hashes. + @raise IOError: If reading one of the files fails. + """ + hashes: Dict[str, str] = {} + for p in paths: + id: str = path.basename(p) + hashes[id] = hash_file(p) + + return hashes + + +def generate_hash_header(hashes: Dict[str, str]) -> None: + """Generates the header file containing the hashes of the static files. + + Generates a header file, in src/generated, containing constant definitions with the hashes of the static files. + + @param hashes: The filenames and hashes to generate the header for. + """ + + print("Generating " + path.relpath(hash_header_path, env.subst("$PROJECT_ROOT"))) # type: ignore[name-defined] + + header = open(hash_header_path, 'w') + header.write( +"""/* + * web_file_hashes.h + * + * **Warning:** This file is automatically generated, and should not be edited manually. + * + * This file contains constant definitions for the hashes of the static files sent by the web server. + * + * This project is licensed under the MIT License. + * The MIT license can be found in the project root and at https://opensource.org/licenses/MIT. + */ + +#ifndef SRC_GENERATED_WEB_FILE_HASHES_H_ +#define SRC_GENERATED_WEB_FILE_HASHES_H_ + +""") + + for file, hash in hashes.items(): + header.write( +f"""/** + * The md5 hash of the file "{file}". + */ +""") + + id: str = file.upper() + for c in ['.', '-', '/', ' ']: + id = id.replace(c, '_') + + header.write(f"static constexpr const char {id}_HASH[] = \"{hash}\";") + header.write(os.linesep + os.linesep) + + header.write("#endif /* SRC_GENERATED_WEB_FILE_HASHES_H_ */" + os.linesep) + + +def main() -> int: + """The main entrypoint of this script. + + The main function executing all the functionality of this script. + Hashes the static web files, and generates a header file containing those hashes. + + @return Zero if nothing goes wrong. + @raise IOError: If opening or reading the file fails. + """ + + static_path_abs: Final[str] = path.abspath(static_dir) + static_path_blacklist_abs: Final[List[str]] = [path.abspath(path.join(env.subst('$PROJECT_DIR'), file.strip())) for file in static_file_blacklist] # type: ignore[name-defined] + + files: List[str] = env.GetProjectOption('board_build.embed_files', '').splitlines() # type: ignore[name-defined] + files.extend(env.GetProjectOption('board_build.embed_txtfiles', '').splitlines()) # type: ignore[name-defined] + files = [path.abspath(path.join(env.subst('$PROJECT_DIR'), file.strip())) for file in files if file.strip()] # type: ignore[name-defined] + files = [file for file in files if file.startswith(path.abspath(static_dir)) and path.splitext(file)[1] in static_types and file not in static_path_blacklist_abs] + + hashes = hash_files(files) + generate_hash_header(hashes) + + return 0 + + +if __name__ == '__main__' or __name__ == 'SCons.Script': + error: int = main() + if error != 0: + sys.exit(error) diff --git a/shared/gzip_compressing_stream.py b/shared/gzip_compressing_stream.py old mode 100644 new mode 100755 index 5243696..2d9b2f3 --- a/shared/gzip_compressing_stream.py +++ b/shared/gzip_compressing_stream.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + """A custom gzip compressing stream using a configurable window size. Also contains a simple command line interface.""" @@ -246,5 +248,5 @@ def main(): return 0 -if __name__ == "__main__": +if __name__ == '__main__': sys.exit(main()) diff --git a/shared/read_ota_pass.py b/shared/read_ota_pass.py index 9587f21..8fb536b 100755 --- a/shared/read_ota_pass.py +++ b/shared/read_ota_pass.py @@ -1,4 +1,5 @@ -#!/usr/bin/python +#!/usr/bin/env python3 + Import ("env") import os diff --git a/src/AsyncHeadOnlyResponse.cpp b/src/AsyncHeadOnlyResponse.cpp index 40925a5..347a885 100644 --- a/src/AsyncHeadOnlyResponse.cpp +++ b/src/AsyncHeadOnlyResponse.cpp @@ -9,12 +9,13 @@ */ #include "AsyncHeadOnlyResponse.h" +#include "fallback_log.h" #if ENABLE_WEB_SERVER == 1 web::AsyncHeadOnlyResponse::AsyncHeadOnlyResponse( AsyncWebServerResponse *wrapped, const int status_code) : AsyncBasicResponse(status_code), _wrapped(wrapped) { - + log_d("Creating head only response."); } web::AsyncHeadOnlyResponse::~AsyncHeadOnlyResponse() { diff --git a/src/generated/README.md b/src/generated/README.md new file mode 100644 index 0000000..d93c116 --- /dev/null +++ b/src/generated/README.md @@ -0,0 +1,3 @@ +# Generated Sources +This directory contains automatically generated source files and **IS NOT INTENDED TO BE MANUALLY EDITED**. +Any changes to the files in this directory are likely to be overridden during the build process. diff --git a/src/generated/web_file_hashes.h b/src/generated/web_file_hashes.h new file mode 100644 index 0000000..4d57cb9 --- /dev/null +++ b/src/generated/web_file_hashes.h @@ -0,0 +1,45 @@ +/* + * web_file_hashes.h + * + * **Warning:** This file is automatically generated, and should not be edited manually. + * + * This file contains constant definitions for the hashes of the static files sent by the web server. + * + * This project is licensed under the MIT License. + * The MIT license can be found in the project root and at https://opensource.org/licenses/MIT. + */ + +#ifndef SRC_GENERATED_WEB_FILE_HASHES_H_ +#define SRC_GENERATED_WEB_FILE_HASHES_H_ + +/** + * The md5 hash of the file "main.css.gz". + */ +static constexpr const char MAIN_CSS_GZ_HASH[] = "f144c285f1ef9fd820edeac7e3d79263"; + +/** + * The md5 hash of the file "index.js.gz". + */ +static constexpr const char INDEX_JS_GZ_HASH[] = "b7d1cc7b510da0eb5044b08fd49f6d7a"; + +/** + * The md5 hash of the file "manifest.json.gz". + */ +static constexpr const char MANIFEST_JSON_GZ_HASH[] = "0f2bc88509c2e5fa90582f041843c42f"; + +/** + * The md5 hash of the file "favicon.ico.gz". + */ +static constexpr const char FAVICON_ICO_GZ_HASH[] = "31fc8a37562605f1d0b3c919dffa4e96"; + +/** + * The md5 hash of the file "favicon.png.gz". + */ +static constexpr const char FAVICON_PNG_GZ_HASH[] = "f18756d0d8dbb95f5493d805443fa023"; + +/** + * The md5 hash of the file "favicon.svg.gz". + */ +static constexpr const char FAVICON_SVG_GZ_HASH[] = "de3746021c5d6d082e271941e00f7dbc"; + +#endif /* SRC_GENERATED_WEB_FILE_HASHES_H_ */ diff --git a/src/prometheus.cpp b/src/prometheus.cpp index 861c84d..3b3d137 100644 --- a/src/prometheus.cpp +++ b/src/prometheus.cpp @@ -263,14 +263,11 @@ size_t prom::writeMetricMetadataLine(char *buffer, #endif /* ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 */ #if ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 -bool prom::acceptsOpenMetrics(const char *accept_str) { - const char *start = strstr(accept_str, "application/openmetrics-text"); - return start != NULL; -} - web::ResponseData prom::handleMetrics(AsyncWebServerRequest *request) { const bool openmetrics = request->hasHeader("Accept") - && acceptsOpenMetrics(request->header("Accept").c_str()); + && web::csvHeaderContains(request->header("Accept").c_str(), + "application/openmetrics-text"); + if (openmetrics) { log_d("Client accepts openmetrics."); } else { @@ -284,7 +281,7 @@ web::ResponseData prom::handleMetrics(AsyncWebServerRequest *request) { "application/openmetrics-text; version=1.0.0; charset=utf-8" : "text/plain; version=0.0.4; charset=utf-8"), metrics); - response->addHeader("Cache-Control", "no-cache"); + response->addHeader("Cache-Control", web::CACHE_CONTROL_NOCACHE); response->addHeader("Vary", "Accept"); return web::ResponseData(response, metrics.length(), 200); } diff --git a/src/prometheus.h b/src/prometheus.h index 6cfc67c..6826b6a 100644 --- a/src/prometheus.h +++ b/src/prometheus.h @@ -119,14 +119,6 @@ size_t writeMetricMetadataLine(char *buffer, const char (&field_name)[fnm_l], #endif #if ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 -/** - * Checks whether the given Accept header accepts openmetrics text protocol version 1.0.0. - * - * @param accept_str The Accept header to check. - * @return True if the given header contains "application/openmetrics-text". - */ -bool acceptsOpenMetrics(const char *accept_str); - /** * The callback method to respond to a HTTP get request for the metrics page. * diff --git a/src/webhandler.cpp b/src/webhandler.cpp index bfa89fd..a5fc354 100644 --- a/src/webhandler.cpp +++ b/src/webhandler.cpp @@ -14,6 +14,7 @@ #include "prometheus.h" #endif #include "sensor_handler.h" +#include "generated/web_file_hashes.h" #include "AsyncHeadOnlyResponse.h" #ifdef ESP32 #include @@ -26,6 +27,13 @@ #if ENABLE_WEB_SERVER == 1 AsyncWebServer web::server(WEB_SERVER_PORT); std::map web::handlers; + +web::ResponseData::ResponseData(AsyncWebServerResponse *response, + size_t content_len, uint16_t status_code) : + response(response), content_length(content_len), status_code( + status_code) { + +} #endif void web::setup() { @@ -40,14 +48,14 @@ void web::setup() { registerRedirect("/", "/index.html"); registerReplacingStaticHandler("/index.html", "text/html", INDEX_HTML_START, - index_replacements); + INDEX_HTML_END - 1, index_replacements); registerCompressedStaticHandler("/main.css", "text/css", MAIN_CSS_START, - MAIN_CSS_END); + MAIN_CSS_END, MAIN_CSS_GZ_HASH); registerCompressedStaticHandler("/index.js", "text/javascript", - INDEX_JS_START, INDEX_JS_END); + INDEX_JS_START, INDEX_JS_END, INDEX_JS_GZ_HASH); registerCompressedStaticHandler("/manifest.json", "application/json", - MANIFEST_JSON_START, MANIFEST_JSON_END); + MANIFEST_JSON_START, MANIFEST_JSON_END, MANIFEST_JSON_GZ_HASH); registerRequestHandler("/temperature", HTTP_GET, [](AsyncWebServerRequest *request) -> ResponseData { @@ -55,7 +63,7 @@ void web::setup() { sensors::SENSOR_HANDLER.getTemperatureString(); AsyncWebServerResponse *response = request->beginResponse(200, "text/plain", temp.c_str()); - response->addHeader("Cache-Control", "no-cache"); + response->addHeader("Cache-Control", CACHE_CONTROL_NOCACHE); return ResponseData(response, temp.length(), 200); }); @@ -65,18 +73,18 @@ void web::setup() { sensors::SENSOR_HANDLER.getHumidityString(); AsyncWebServerResponse *response = request->beginResponse(200, "text/plain", humidity.c_str()); - response->addHeader("Cache-Control", "no-cache"); + response->addHeader("Cache-Control", CACHE_CONTROL_NOCACHE); return ResponseData(response, humidity.length(), 200); }); registerRequestHandler("/data.json", HTTP_GET, getJson); registerCompressedStaticHandler("/favicon.ico", "image/x-icon", - FAVICON_ICO_GZ_START, FAVICON_ICO_GZ_END); + FAVICON_ICO_GZ_START, FAVICON_ICO_GZ_END, FAVICON_ICO_GZ_HASH); registerCompressedStaticHandler("/favicon.png", "image/png", - FAVICON_PNG_GZ_START, FAVICON_PNG_GZ_END); + FAVICON_PNG_GZ_START, FAVICON_PNG_GZ_END, FAVICON_PNG_GZ_HASH); registerCompressedStaticHandler("/favicon.svg", "image/svg+xml", - FAVICON_SVG_GZ_START, FAVICON_SVG_GZ_END); + FAVICON_SVG_GZ_START, FAVICON_SVG_GZ_END, FAVICON_SVG_GZ_HASH); // An OPTIONS request to * is supposed to return server-wide support. registerRequestHandler("*", HTTP_OPTIONS, @@ -108,11 +116,33 @@ void web::connect() { } #if ENABLE_WEB_SERVER == 1 -web::ResponseData::ResponseData(AsyncWebServerResponse *response, - size_t content_len, uint16_t status_code) : - response(response), content_length(content_len), status_code( - status_code) { +bool web::csvHeaderContains(const char *header, const char *value) { + const char *cpos = header - 1; + const char *start = NULL; + const size_t header_len = strlen(header); + const size_t val_len = strlen(value); + while (header + header_len > ++cpos) { + if (start == NULL && isspace(*cpos) == 0 && *cpos != ',' + && *cpos != ';') { + start = cpos; + } else if (start != NULL && (*cpos == ',' || *cpos == ';')) { + if (cpos - start == val_len + && strncmp(start, value, val_len) == 0) { + return true; + } + + if (*cpos == ',') { + start = NULL; + } + } + } + if (start != NULL && cpos - start == val_len + && strncmp(start, value, val_len) == 0) { + return true; + } else { + return false; + } } web::ResponseData web::getJson(AsyncWebServerRequest *request) { @@ -149,7 +179,7 @@ web::ResponseData web::getJson(AsyncWebServerRequest *request) { AsyncWebServerResponse *response = request->beginResponse(200, "application/json", buffer); delete[] buffer; - response->addHeader("Cache-Control", "no-cache"); + response->addHeader("Cache-Control", CACHE_CONTROL_NOCACHE); return ResponseData(response, len, 200); } @@ -224,6 +254,11 @@ size_t web::replacingResponseFiller( } } +size_t web::dummyResponseFiller(const uint8_t *buffer, const size_t max_len, + const size_t index) { + return 0; +} + web::ResponseData web::defaultHeadRequestHandlerWrapper( const HTTPRequestHandler &handler, AsyncWebServerRequest *request) { ResponseData response = handler(request); @@ -321,8 +356,8 @@ web::ResponseData web::invalidMethodHandler( validStr += valid[i]; } response.response->addHeader("Allow", validStr); - log_i("A client tried to access the not existing file \"%s\".", - request->url().c_str()); + log_i("Received a request to \"%s\" with invalid method \"%s\".", + request->url().c_str(), request->methodToString()); return response; } } @@ -357,44 +392,127 @@ web::ResponseData web::optionsHandler( web::ResponseData web::staticHandler(const uint16_t status_code, const String &content_type, const uint8_t *start, const uint8_t *end, - AsyncWebServerRequest *request) { - AsyncWebServerResponse *response = request->beginResponse_P(status_code, - content_type, start, end - start); - if (!strcmp(content_type.c_str(), "text/html")) { + AsyncWebServerRequest *request, const char *etag) { + char *etag_str = NULL; + if (etag != NULL) { + const size_t etag_len = strlen(etag); + etag_str = new char[etag_len + 3]; + etag_str[0] = '"'; + memcpy(etag_str + 1, etag, etag_len); + etag_str[etag_len + 1] = '"'; + etag_str[etag_len + 2] = 0; + } + + AsyncWebServerResponse *response = NULL; + size_t content_length = end - start; + uint16_t code = status_code; + if (etag_str != NULL && request->hasHeader("If-None-Match") + && csvHeaderContains(request->header("If-None-Match").c_str(), + etag_str)) { + log_d("Client has up-to-date cached page."); + content_length = 0; + code = 304; + // TODO find a better way to avoid sending the content length. + response = request->beginResponse(content_type, content_length, + dummyResponseFiller); + } else { + response = request->beginResponse_P(code, content_type, start, + content_length); + } + + response->setCode(code); + + if (strcmp(content_type.c_str(), "text/html") == 0) { response->addHeader("Content-Security-Policy", "default-src 'self'"); } - return ResponseData(response, end - start, status_code); + + if (etag_str != NULL) { + response->addHeader("ETag", etag_str); + delete[] etag_str; + response->addHeader("Cache-Control", CACHE_CONTROL_CACHE); + } else { + response->addHeader("Cache-Control", CACHE_CONTROL_NOCACHE); + } + + return ResponseData(response, content_length, code); } web::ResponseData web::compressedStaticHandler(const uint16_t status_code, const String &content_type, const uint8_t *start, const uint8_t *end, - AsyncWebServerRequest *request) { + AsyncWebServerRequest *request, const char *etag) { + const bool accepts_gzip = request->hasHeader("Accept-Encoding") + && csvHeaderContains(request->header("Accept-Encoding").c_str(), + "gzip"); + + if (accepts_gzip) { + log_d("Client accepts gzip compressed data."); + } else { + log_d("Client doesn't accept gzip compressed data."); + } + + char *enc_etag = NULL; + if (etag != NULL) { + const size_t etag_len = strlen(etag); + enc_etag = new char[etag_len + 3 + 5 * accepts_gzip]; + enc_etag[0] = '"'; + memcpy(enc_etag + 1, etag, etag_len); + if (accepts_gzip) { + memcpy(enc_etag + etag_len + 1, "-gzip", 5); + } + enc_etag[etag_len + 1 + 5 * accepts_gzip] = '"'; + enc_etag[etag_len + 2 + 5 * accepts_gzip] = 0; + } + AsyncWebServerResponse *response = NULL; - size_t content_length; - if (request->hasHeader("Accept-Encoding") - && strstr(request->header("Accept-Encoding").c_str(), "gzip")) { + size_t content_length = 0; + uint16_t code = status_code; + if (enc_etag != NULL && request->hasHeader("If-None-Match") + && csvHeaderContains(request->header("If-None-Match").c_str(), + enc_etag)) { + log_d("Client has up-to-date cached page."); + code = 304; + // TODO find a better way to avoid sending the content length. + response = request->beginResponse(content_type, content_length, + dummyResponseFiller); + if (accepts_gzip) { + response->addHeader("Content-Encoding", "gzip"); + } + } else if (accepts_gzip) { content_length = end - start; - response = request->beginResponse_P(200, content_type, start, - end - start); + response = request->beginResponse_P(code, content_type, start, + content_length); response->addHeader("Content-Encoding", "gzip"); } else { using namespace std::placeholders; std::shared_ptr decomp = std::make_shared< gzip::uzlib_ungzip_wrapper>(start, end, GZIP_DECOMP_WINDOW_SIZE); - content_length = decomp->getDecompressedSize(); - response = request->beginResponse(content_type, content_length, - std::bind(decompressingResponseFiller, decomp, _1, _2, _3)); + content_length = max(0, decomp->getDecompressedSize()); + if (content_length == 0) { + // Make sure to send the content length regardless. + response = request->beginResponse(code, content_type, ""); + } else { + response = request->beginResponse(content_type, content_length, + std::bind(decompressingResponseFiller, decomp, _1, _2, _3)); + } } + response->setCode(code); response->addHeader("Vary", "Accept-Encoding"); - response->setCode(status_code); - if (!strcmp(content_type.c_str(), "text/html")) { + if (strcmp(content_type.c_str(), "text/html") == 0) { response->addHeader("Content-Security-Policy", "default-src 'self'"); } - return ResponseData(response, content_length, status_code); + if (enc_etag != NULL) { + response->addHeader("ETag", enc_etag); + delete[] enc_etag; + response->addHeader("Cache-Control", CACHE_CONTROL_CACHE); + } else { + response->addHeader("Cache-Control", CACHE_CONTROL_NOCACHE); + } + + return ResponseData(response, content_length, code); } web::ResponseData web::replacingRequestHandler( @@ -447,6 +565,7 @@ web::ResponseData web::replacingRequestHandler( if (!strcmp(content_type.c_str(), "text/html")) { response->addHeader("Content-Security-Policy", "default-src 'self'"); } + response->addHeader("Cache-Control", CACHE_CONTROL_NOCACHE); return ResponseData(response, content_length, status_code); } @@ -476,24 +595,37 @@ void web::registerRequestHandler(const char *uri, } void web::registerStaticHandler(const char *uri, const String &content_type, - const char *page) { + const char *page, const char *etag) { + registerStaticHandler(uri, content_type, (uint8_t*) page, + (uint8_t*) page + strlen(page), etag); +} + +void web::registerStaticHandler(const char *uri, const String &content_type, + const uint8_t *start, const uint8_t *end, const char *etag) { using namespace std::placeholders; registerRequestHandler(uri, HTTP_GET, - std::bind(staticHandler, 200, content_type, (uint8_t*) page, - (uint8_t*) page + strlen(page), _1)); + std::bind(staticHandler, 200, content_type, start, end, _1, etag)); } void web::registerCompressedStaticHandler(const char *uri, - const String &content_type, const uint8_t *start, const uint8_t *end) { + const String &content_type, const uint8_t *start, const uint8_t *end, + const char *etag) { using namespace std::placeholders; registerRequestHandler(uri, HTTP_GET, std::bind(compressedStaticHandler, 200, content_type, start, end, - _1)); + _1, etag)); } void web::registerReplacingStaticHandler(const char *uri, const String &content_type, const char *page, const std::map> replacements) { + registerReplacingStaticHandler(uri, content_type, (uint8_t*) page, + (uint8_t*) page + strlen(page), replacements); +} + +void web::registerReplacingStaticHandler(const char *uri, + const String &content_type, const uint8_t *start, const uint8_t *end, + const std::map> replacements) { using namespace std::placeholders; registerRequestHandler(uri, HTTP_GET, std::bind< @@ -502,7 +634,7 @@ void web::registerReplacingStaticHandler(const char *uri, const uint16_t, const String&, const uint8_t*, const uint8_t*, AsyncWebServerRequest*)>( replacingRequestHandler, replacements, 200, content_type, - (uint8_t*) page, (uint8_t*) page + strlen(page), _1)); + start, end, _1)); } void web::registerRedirect(const char *uri, const char *target) { diff --git a/src/webhandler.h b/src/webhandler.h index 45723d6..d9f35c3 100644 --- a/src/webhandler.h +++ b/src/webhandler.h @@ -41,21 +41,84 @@ typedef std::function< #include #include -extern const char INDEX_HTML_START[] asm("_binary_data_index_html_start"); -extern const char INDEX_HTML_END[] asm("_binary_data_index_html_end"); +/** + * A pointer to the first byte of the templated main page of the web interface. + */ +extern const uint8_t INDEX_HTML_START[] asm("_binary_data_index_html_start"); + +/** + * A pointer to the first byte after the templated main page of the web interface. + */ +extern const uint8_t INDEX_HTML_END[] asm("_binary_data_index_html_end"); + +/** + * A pointer to the first byte of the gzip compressed main css stylesheet. + */ extern const uint8_t MAIN_CSS_START[] asm("_binary_data_gzip_main_css_gz_start"); + +/** + * A pointer to the first byte after the gzip compressed main css stylesheet. + */ extern const uint8_t MAIN_CSS_END[] asm("_binary_data_gzip_main_css_gz_end"); + +/** + * A pointer to the first byte of the gzip compressed main page javascript file. + */ extern const uint8_t INDEX_JS_START[] asm("_binary_data_gzip_index_js_gz_start"); + +/** + * A pointer to the first byte after the gzip compressed main page javascript file. + */ extern const uint8_t INDEX_JS_END[] asm("_binary_data_gzip_index_js_gz_end"); + +/** + * A pointer to the first byte of the gzip compressed web app manifest. + */ extern const uint8_t MANIFEST_JSON_START[] asm("_binary_data_gzip_manifest_json_gz_start"); + +/** + * A pointer to the first byte after the gzip compressed web app manifest. + */ extern const uint8_t MANIFEST_JSON_END[] asm("_binary_data_gzip_manifest_json_gz_end"); -extern const char ERROR_HTML_START[] asm("_binary_data_error_html_start"); -extern const char ERROR_HTML_END[] asm("_binary_data_error_html_end"); + +/** + * A pointer to the first byte of the error html page. + */ +extern const uint8_t ERROR_HTML_START[] asm("_binary_data_error_html_start"); + +/** + * A pointer to the first byte after the error html page. + */ +extern const uint8_t ERROR_HTML_END[] asm("_binary_data_error_html_end"); + +/** + * A pointer to the first byte of the gzip compressed favicon ico. + */ extern const uint8_t FAVICON_ICO_GZ_START[] asm("_binary_data_gzip_favicon_ico_gz_start"); + +/** + * A pointer to the first byte after the gzip compressed favicon ico. + */ extern const uint8_t FAVICON_ICO_GZ_END[] asm("_binary_data_gzip_favicon_ico_gz_end"); + +/** + * A pointer to the first byte of the gzip compressed favicon png. + */ extern const uint8_t FAVICON_PNG_GZ_START[] asm("_binary_data_gzip_favicon_png_gz_start"); + +/** + * A pointer to the first byte after the gzip compressed favicon png. + */ extern const uint8_t FAVICON_PNG_GZ_END[] asm("_binary_data_gzip_favicon_png_gz_end"); + +/** + * A pointer to the first byte of the gzip compressed favicon svg. + */ extern const uint8_t FAVICON_SVG_GZ_START[] asm("_binary_data_gzip_favicon_svg_gz_start"); + +/** + * A pointer to the first byte after the gzip compressed favicon svg. + */ extern const uint8_t FAVICON_SVG_GZ_END[] asm("_binary_data_gzip_favicon_svg_gz_end"); /** @@ -93,6 +156,16 @@ class ResponseData { const uint16_t status_code); }; +/** + * The Cache-Control header value to send for pages that should not be cached. + */ +static constexpr char CACHE_CONTROL_NOCACHE[] = "no-store"; + +/** + * The Cache-Control header value to send for pages that may be cached. + */ +static constexpr char CACHE_CONTROL_CACHE[] = "public, no-cache"; + /** * The character to use as a template delimiter. */ @@ -127,6 +200,22 @@ void loop(); void connect(); #if ENABLE_WEB_SERVER == 1 +/** + * Checks whether the given comma separated values header accepts the has the given value as an option. + * + * This function is case sensitive. + * + * The value has be exactly match one of the given options, excluding the factors after semicolons. + * While this function can, in theory, also match these factors, this requires the order of the factors to match, not just their values. + * + * Note: While spaces after commas are allowed, spaces after values are not at this time. + * + * @param header The header value to check. + * @param value The value to look for. + * @return True if the given value is accepted by the client. + */ +bool csvHeaderContains(const char *header, const char *value); + /** * The request handler for /data.json. * Responds with a json object containing the current temperature and humidity, @@ -138,7 +227,7 @@ void connect(); ResponseData getJson(AsyncWebServerRequest *request); /** - * A AwsResponseFiller decompressing a file from memory using uzlib. + * An AwsResponseFiller decompressing a file from memory using uzlib. * * @param decomp The uzlib decompressing persistent data. * @param buffer The output buffer to write the decompressed data to. @@ -151,7 +240,7 @@ size_t decompressingResponseFiller( uint8_t *buffer, const size_t max_len, const size_t index); /** - * A AwsResponseFiller copying the given static file, and replacing the given template strings. + * An AwsResponseFiller copying the given static file, and replacing the given template strings. * If the output buffer can hold more data than available, the remainder is filled with null bytes. * * @param replacements A map containing the template strings to replace, and their replacement values. @@ -170,6 +259,17 @@ size_t replacingResponseFiller(const std::map &replacements, const uint8_t *end, uint8_t *buffer, const size_t max_len, const size_t index); +/** + * An AwsResponseFiller doing absolutely nothing, used to avoid sending the content length of 0. + * + * @param buffer The output buffer a real response filler would write to. + * @param max_len The max number of bytes to write to the output buffer. + * @param index The number of bytes already written to the client. + * @return Zero. Always. + */ +size_t dummyResponseFiller(const uint8_t *buffer, const size_t max_len, + const size_t index); + /** * A request handler wrapper for GET request handlers that automatically adapts them for HEAD requests. * @@ -216,17 +316,21 @@ ResponseData optionsHandler(const WebRequestMethodComposite validMethods, * * Automatically adds a "default-src 'self'" content security policy to "text/html" responses. * + * Allows ETag based caching, if an etag was given. + * * @param status_code The HTTP response status code to send to the client. * @param content_type The content type of the static file. * @param start A pointer to the first byte of the compressed static file. * @param end A pointer to the first byte after the end of the compressed static file. * For C strings this is the terminating NUL byte. * @param request The request to handle. + * @param etag The HTTP entity tag to use for caching. + * Use NULL to disable sending an ETag for this page. * @return The response to be sent to the client. */ ResponseData staticHandler(const uint16_t status_code, const String &content_type, const uint8_t *start, const uint8_t *end, - AsyncWebServerRequest *request); + AsyncWebServerRequest *request, const char *etag = NULL); /** * A web request handler for a compressed static file. @@ -236,17 +340,21 @@ ResponseData staticHandler(const uint16_t status_code, * * Automatically adds a "default-src 'self'" content security policy to "text/html" responses. * + * Allows ETag based caching, if an etag was given. + * * @param status_code The HTTP response status code to send to the client. * @param content_type The content type of the static file. * @param start A pointer to the first byte of the compressed static file. * @param end A pointer to the first byte after the end of the compressed static file. * For C strings this is the terminating NUL byte. * @param request The request to handle. + * @param etag The HTTP entity tag to use for caching. + * Use NULL to disable sending an ETag for this page. * @return The response to be sent to the client. */ ResponseData compressedStaticHandler(const uint16_t status_code, const String &content_type, const uint8_t *start, const uint8_t *end, - AsyncWebServerRequest *request); + AsyncWebServerRequest *request, const char *etag = NULL); /** * A web request handler for a static file with some templates to replace. @@ -257,6 +365,8 @@ ResponseData compressedStaticHandler(const uint16_t status_code, * * Automatically adds a "default-src 'self'" content security policy to "text/html" responses. * + * This request handler automatically adds a Cache-Control header forbidding caching, since the page is dynamic. + * * @param replacements A map mapping a template string to be replaced, * to a function returning its replacement value. * @param status_code The HTTP response status code to send to the client. @@ -277,6 +387,10 @@ ResponseData replacingRequestHandler( * A web request handler for a static file with some templates to replace. * Template strings have to be formatted like this: $TEMPLATE$. * + * Automatically adds a "default-src 'self'" content security policy to "text/html" responses. + * + * This request handler automatically adds a Cache-Control header forbidding caching, since the page is dynamic. + * * @param replacements A map mapping a template string to be replaced, * to its replacement string. * @param status_code The HTTP response status code to send to the client. @@ -321,30 +435,58 @@ void registerRequestHandler(const char *uri, * Registers a request handler that returns the given content type and web page each time it is called. * Registers request handlers for the request methods GET, HEAD, and OPTIONS. * Always sends response code 200. + * * Will automatically increment the prometheus request counter. + * Allows ETag based caching, if an etag was given. * * @param uri The path on which the page can be found. * @param content_type The content type for the page. * @param page The content for the page to be sent to the client. + * @param etag The HTTP entity tag to use for caching. + * Use NULL to disable sending an ETag for this page. */ void registerStaticHandler(const char *uri, const String &content_type, - const char *page); + const char *page, const char *etag = NULL); + +/** + * Registers a request handler that returns the given content type and web page each time it is called. + * Registers request handlers for the request methods GET, HEAD, and OPTIONS. + * Always sends response code 200. + * + * Will automatically increment the prometheus request counter. + * Allows ETag based caching, if an etag was given. + * + * @param uri The path on which the page can be found. + * @param content_type The content type for the page. + * @param start The pointer to the first byte of the file. + * @param end The pointer to the first byte after the end of the file. + * For C strings this is the terminating NUL byte. + * @param etag The HTTP entity tag to use for caching. + * Use NULL to disable sending an ETag for this page. + */ +void registerStaticHandler(const char *uri, const String &content_type, + const uint8_t *start, const uint8_t *end, const char *etag = NULL); /** * Registers a request handler that returns the given content each time it is called. * Registers request handlers for the request methods GET, HEAD, and OPTIONS. * Always sends response code 200. + * * Will automatically increment the prometheus request counter. * Expects the content to be a gzip compressed binary. + * Allows ETag based caching, if an etag was given. * * @param uri The path on which the file can be found. * @param content_type The content type for the file. * @param start The pointer to the first byte of the file. * @param end The pointer to the first byte after the end of the file. * For C strings this is the terminating NUL byte. + * @param etag The HTTP entity tag to use for caching. + * Use NULL to disable sending an ETag for this page. */ void registerCompressedStaticHandler(const char *uri, - const String &content_type, const uint8_t *start, const uint8_t *end); + const String &content_type, const uint8_t *start, const uint8_t *end, + const char *etag = NULL); /** * Registers a request handler that returns the given content type and web page each time it is called. @@ -358,6 +500,8 @@ void registerCompressedStaticHandler(const char *uri, * Always sends response code 200. * Will automatically increment the prometheus request counter. * + * This request handler automatically adds a Cache-Control header forbidding caching, since the page is dynamic. + * * @param uri The path on which the page can be found. * @param content_type The content type for the page. * @param page The content for the page to be sent to the client. @@ -368,6 +512,32 @@ void registerReplacingStaticHandler(const char *uri, const String &content_type, const char *page, const std::map> replacements); +/** + * Registers a request handler that returns the given content type and web page each time it is called. + * Replaces the given strings in the static file. + * Template strings have to be formatted like this: $TEMPLATE$. + * + * The templates will be replaced with the result of the function registered for them. + * Note that each function will be called once per request, no matter how often its template appears. + * + * Registers request handlers for the request methods GET, HEAD, and OPTIONS. + * Always sends response code 200. + * Will automatically increment the prometheus request counter. + * + * This request handler automatically adds a Cache-Control header forbidding caching, since the page is dynamic. + * + * @param uri The path on which the page can be found. + * @param content_type The content type for the page. + * @param start The pointer to the first byte of the file. + * @param end The pointer to the first byte after the end of the file. + * For C strings this is the terminating NUL byte. + * @param replacements A map mapping a template string to be replaced, + * to a function returning its replacement value. + */ +void registerReplacingStaticHandler(const char *uri, const String &content_type, + const uint8_t *start, const uint8_t *end, + const std::map> replacements); + /** * Registers a request handler redirecting to the given target url. * The request handler will be registered for all request methods.