diff --git a/.gitignore b/.gitignore index da346c2..bbab9e0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ ipch Debug Release *.lcl +.vs # Executables *.exe diff --git a/README.md b/README.md index a8cd9fd..6a9c0ce 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,96 @@ -# mod_mrf +# mod_mrf [AHTSE](https://github.com/lucianpls/AHTSE) -An apache module that serves tiles directly from a local MRF, 2D or 3D. -This module takes two apache configuration directives: +An apache module that serves tiles directly from a local MRF, 2D or 3D. +With the MRF data on a local SSD, this module average tile request latency is .25ms (as measured by httpd), and reaches request rates above 20000 req/sec on a single core. - **MRF On|Off** - - Defaults to on if the MRF_ConfigurationFile is provided +Apache configuration directives: + +**MRF_RegExp** + Required, only requests matching this pattern are handled. It can appear multiple times + If not provided, the module is inactive + +**MRF_Indirect On|Off** - **MRF_ConfigurationFile Filename** + If set, this module will only respond to internal subrequests - Points to a text file that contains lines, where the first word on a line is a directive, followed by parameters +**MRF_ConfigurationFile Filename** + + Points to an AHTSE Control text file, where the first word on a line is a directive, followed by parameters - Empty lines, lines that start with # are considered comments - Unknown directives are ignored - - Known directives for this module are: - **Size X Y Z C** - - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) +AHTSE Control Directives for this module are: - **DataFile string** - - Mandatory, the path to the data file of the MRF. +***DataFile path start_offset size*** + - The path to the MRF data file to serve tiles from. Start and size are optional, by default + a single DataFile is used. At least one DataFile directive is required. If the path start + with ://, the path is interpreted as an internal redirect to a path + within the same server, starting from DocRoot. Otherwise it is assumed to be a local file + name. May appear multiple times, with different start offset and size values. If the values are + present, read operations within start_offset and start_offset + size are made to the data file, + after the read offset is adjusted downward by start_offset. + If the read offset falls outside the range, the other DataFile entries are searched, + in the order in which they appear in the configuration file. Old style redirects are tested last. + The multiple entries allows an MRF data file to be split into multiple parts. Single tiles + cannot be split between sources, but overlapping ranges between source are allowed. Only one read + operation is issued, to the first DataFile entry that matches the range. If the read fails, + the server will report an error. + Start offset and size default to zero. Zero size means that any read above the offset will be + considered present in this data file. - **RegExp** - - Optional, a regular expression that must match the request URL for it to be considered a tile request. By default all URLs are considered tile requests. This directive can appear multiple times. If there are multiple RegExp lines, at least one has to match the request URL. - - **PageSize X Y 1 C** - - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size +***Size X Y Z C*** + - Mandatory, at least x and y, the size in pixels of the input MRF. + Z defaults to 1 and C defaults to 3 (these are usually not meaningful) - **IndexFile string** - - Optional, The index file name. - If not provided it uses the data file name if its extension is not three letters. - Otherwise it uses the datafile name with the extension changed to .idx - - **MimeType string** - - Optional. Defaults to autodetect. +***PageSize X Y 1 C*** + - Optional, the pagesize in pixels. X and Y default to 512. + Z has to be 1 if C is provided, which has to match the C value from size - **EmptyTile Size Offset FileName** - - Optional. By default it ignores the request if a tile is missing. - First number is assumed to be the size, second is offset. - If filename is not provided, it uses the data file name. +***RetryCount N*** + - Optional, [0 - 99). If the DataFiles are redirects, how many times to retry a redirected + read that fails to retun the requested data. The Default is 5. - **SkippedLevels N** - - Optional, how many levels to ignore, at the top of the MRF pyramid. - For example a GCS pyramid will have to skip the one tile level, so this should be 1 +***IndexFile string*** + - Optional, the index file name. Can only be provided once. + If not provided it uses the data file name if its extension is not three letters. + Otherwise it uses the first data file name with the extension changed to .idx + It can be a redirect path in the host namespace, if it starts with :// - **ETagSeed base32_string** - - Optional, 64 bits in base32 digits. Defaults to 0. - The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles - have 64 bit ETags that depend on this value. +***EmptyTile Size Offset FileName*** + - Optional, provides the tile content to be sent when the requested tile is missing. + The file has to be local, since the empty tile is read at start-up + By default the request is ignored, which results in a 404 error if a fallback mechanism does not + exist. If present, the first number is assumed to be the size, second is offset. If filename is + not given, the first data file name is used. + +***SkippedLevels N*** + - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid + will have to skip the one tile level, so this should be 1 - **Redirect path** - - Optional, if the data file is on an object store +***ETagSeed base32_string*** + - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be + this value but 65th bit is set, also the only value that has this bit set. All the other tiles + have 64 bit ETags that depend on this value. + +***Dynamic On*** + - Optional, flags the local files as dynamic, disabling any caching or file handle reuse. To be used + when the MRF files are changed at run-time, avoiding stale or even broken content. MRF in-place + modification do not require this flag because the old content is still available + + ***MMapping prefix*** + - Optional, controls the mapping of the M parameter to data source. The only value currently + implemented is _prefix_, which means that the M, as a decimal number will be added right in front of + the basename of the file, both the Index and Data. The range based data file split still applies, + each part will be prefixed by the M value + + +***CannedIndex On*** + - Optional, flags the index file as a canned format index, see mrf_apps/can.cpp. This is a dense + format index that can be much smaller. Should be used only when needed, and not recommended when + Dynamic is also on + +***Redirect path start_offset size*** + *Deprecated*, use the DataFile directive and start path with :// For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch diff --git a/macros.props b/macros.props new file mode 100644 index 0000000..d576359 --- /dev/null +++ b/macros.props @@ -0,0 +1,15 @@ + + + + + C:\httpd + + + + + + $(prefix) + true + + + \ No newline at end of file diff --git a/mod_mrf.sln b/mod_mrf.sln index 0ad3fcd..f91d834 100644 --- a/mod_mrf.sln +++ b/mod_mrf.sln @@ -1,22 +1,29 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.34928.147 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mod_mrf", "mod_mrf.vcxproj", "{C7E01836-333F-490D-A3B5-3554A8935040}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.ActiveCfg = Debug|Win32 - {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.Build.0 = Debug|Win32 - {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.ActiveCfg = Release|Win32 - {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.Build.0 = Release|Win32 + {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.ActiveCfg = Debug|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|x64.ActiveCfg = Debug|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|x64.Build.0 = Debug|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.ActiveCfg = Release|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Release|x64.ActiveCfg = Release|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {A5643073-62AA-4472-AEC6-7057F30D5683} + EndGlobalSection EndGlobal diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index f9431c3..dcb4a0e 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -1,87 +1,100 @@  - + - + Debug - Win32 + x64 - + Release - Win32 + x64 - - - + + + + + {C7E01836-333F-490D-A3B5-3554A8935040} Win32Proj mod_mrf + 10.0 - + DynamicLibrary true - v120 + v143 NotSet - + Application false - v120 + v143 true - Unicode + NotSet - + + - + + - + true .so + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + $(prefix)\include;$(prefix)\include\apr-2;$(VC_IncludePath);$(WindowsSDK_IncludePath); + $(prefix)\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) - + false + $(prefix)\include;$(prefix)\include\apr-2;$(VC_IncludePath);$(WindowsSDK_IncludePath); + $(prefix)\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) - + NotUsing Level3 Disabled WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) true - \Apache24\include + + Default Windows true - \Apache24\lib - libhttpd.lib;libapr-1.lib;libaprutil-1.lib;%(AdditionalDependencies) + + + libahtse.lib;libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 - $(OutDir)$(TargetName)$(TargetExt) - copy /y $(OutDir)$(TargetName)$(TargetExt) \Apache24\modules + copy /y $(TargetPath) $(prefix)\modules + Install to $(prefix)\modules - + Level3 - Use + NotUsing MaxSpeed true true @@ -93,7 +106,13 @@ true true true + libahtse.lib;libhttpd.lib;libapr-2.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + /EXPORT:mrf_module,@1 + + copy /y $(TargetPath) $(prefix)\modules + Install to $(prefix)\modules + diff --git a/mod_mrf.vcxproj.filters b/mod_mrf.vcxproj.filters index 2d7f900..5bbe540 100644 --- a/mod_mrf.vcxproj.filters +++ b/mod_mrf.vcxproj.filters @@ -9,21 +9,31 @@ {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hh;hpp;hxx;hm;inl;inc;xsd - - {a73e6c46-3010-4cc7-b8fa-4d7c875ede0f} + + {76786145-6ccc-4796-bd24-c8edce17e2a3} - - Header Files - - - - - DOC + + Support Files + + + Support Files + + + Support Files + + + Support Files - DOC + Support Files + + + Support Files + + + Support Files diff --git a/src/Makefile b/src/Makefile index 04e863b..8e6e13a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,24 +1,28 @@ -C_SRC = mod_mrf.cpp -FILES = $(C_SRC) +MODULE = mod_mrf +TARGET = .libs/$(MODULE).so +C_SRC = $(MODULE).cpp -DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE +DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) # Create the ../Makefile.lcl, which should define -# MOD_PATH, the folder where the apache module should reside +# DEST, the folder where the apache module should reside # INCLUDES, the gcc include commands for httpd, apr headers +# LIBTOOL, DEBUG ... -include Makefile.lcl -TARGET = .libs/mod_mrf.so +MAKEOPT ?= Makefile.lcl +include $(MAKEOPT) # Can't use apxs to build c++ modules # The options used here might depend on how apache was built -$(TARGET) : $(FILES) - libtool --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o mod_mrf.lo $(C_SRC) && touch mod_mrf.slo - libtool --silent --mode=link g++ -o mod_mrf.la -rpath $(MOD_PATH) -module -avoid-version mod_mrf.lo + +default : $(TARGET) + +$(TARGET) : $(C_SRC) + $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(EXTRA_INCLUDES) -I $(EXP_INCLUDEDIR) -pthread -c -o $(MODULE).lo $< && touch $(MODULE).slo + $(LIBTOOL) --silent --mode=link g++ -o $(MODULE).la -rpath $(LIBEXECDIR) -module -avoid-version $(MODULE).lo install : $(TARGET) - sudo cp $(TARGET) $(MOD_PATH) + $(SUDO) $(CP) $(TARGET) $(DEST) clean : - rm -rf .libs *.{o,lo,slo,la} - + $(RM) -r .libs *.{o,lo,slo,la} diff --git a/src/Makefile.lcl.example b/src/Makefile.lcl.example index 7bbb3f4..0872f8b 100644 --- a/src/Makefile.lcl.example +++ b/src/Makefile.lcl.example @@ -1,3 +1,11 @@ -INCLUDES = -I /usr/include/httpd -I /usr/include/apr-1 -MOD_PATH = /etc/httpd/modules +APXS = apxs +PREFIX ?= $(HOME) +includedir = $(shell $(APXS) -q includedir 2>/dev/null) +EXTRA_INCLUDES = $(shell $(APXS) -q EXTRA_INCLUDES 2>/dev/null) +LIBTOOL = $(shell $(APXS) -q LIBTOOL 2>/dev/null) +LIBEXECDIR = \$(shell \$(APXS) -q libexecdir 2>/dev/null) +EXP_INCLUDEDIR = $(PREFIX)/include +# SUDO = sudo +CP = cp +DEST = $(PREFIX)/modules diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 1b8c731..deb5afc 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -1,457 +1,456 @@ /* -* An OnEarth module that serves tiles from an MRF +* An AHTSE module that serves tiles from an MRF * Lucian Plesea -* (C) 2016-2017 +* (C) 2016-2019 */ -#include "mod_mrf.h" +#include #include "receive_context.h" #include #include +#include -using namespace std; +#include +#include -static void *create_dir_config(apr_pool_t *p, char *dummy) -{ - mrf_conf *c = - (mrf_conf *)apr_pcalloc(p, sizeof(mrf_conf)); - return c; -} +using namespace std; +NS_AHTSE_USE +NS_ICD_USE + +// Max count, should never happen +#define NPOS 0xffffffff +// Block size for canned format +#define BSZ 512 + +// Bit set count for 32bit values +#if defined(_WIN32) +#include +#define bsc __popcnt +#else +// This only works in gcc +#define bsc __builtin_popcount +#endif -// -// Tokenize a string into an array -// -static apr_array_header_t* tokenize(apr_pool_t *p, const char *s, char sep = '/') -{ - apr_array_header_t* arr = apr_array_make(p, 10, sizeof(char *)); - while (sep == *s) s++; - char *val; - while (*s && (val = ap_getword(p, &s, sep))) { - char **newelt = (char **)apr_array_push(arr); - *newelt = val; - } - return arr; +// The next few functions are from the mrf/mrf_apps/can program +// Check a specific bit position in a canned index header line +// The bit position is [0, 95] and the first 32bit value is skipped +static inline bool is_on(uint32_t *values, int bit) { + return 0 != (values[1 + bit / 32] & (static_cast(1) << bit % 32)); } -// Returns a table read from a file, or NULL and an error message -static apr_table_t *read_pKVP_from_file(apr_pool_t *pool, const char *fname, char **err_message) - -{ - // Should parse it here and initialize the configuration structure - ap_configfile_t *cfg_file; - apr_status_t s = ap_pcfg_openfile(&cfg_file, pool, fname); - - if (APR_SUCCESS != s) { // %pm means print status error string - *err_message = apr_psprintf(pool, "%s - %pm", fname, &s); - return NULL; - } - - char buffer[MAX_STRING_LEN]; - apr_table_t *table = apr_table_make(pool, 8); - // This can return ENOSPC if lines are too long - while (APR_SUCCESS == (s = ap_cfg_getline(buffer, MAX_STRING_LEN, cfg_file))) { - if ((strlen(buffer) == 0) || buffer[0] == '#') - continue; - const char *value = buffer; - char *key = ap_getword_white(pool, &value); - apr_table_add(table, key, value); - } - - ap_cfg_closefile(cfg_file); - if (s == APR_ENOSPC) { - *err_message = apr_psprintf(pool, "%s lines should be smaller than %d", fname, MAX_STRING_LEN); - return NULL; - } - - return table; +// Canned index file header size +// 16 byte prefix + 96 bit records in 128 bit lines +static inline uint64_t hsize(uint64_t in_size) { + return 16 + 16 * ((96 * BSZ - 1 + in_size) / (96 * BSZ)); } -// Returns NULL if it worked as expected, returns a four integer value from "x y", "x y z" or "x y z c" -static char *get_xyzc_size(apr_pool_t *p, struct sz *size, const char *value, const char*err_prefix) { - char *s; - if (!value) - return apr_psprintf(p, "%s directive missing", err_prefix); - size->x = apr_strtoi64(value, &s, 0); - size->y = apr_strtoi64(s, &s, 0); - size->c = 3; - size->z = 1; - if (errno == 0 && *s) { // Read optional third and fourth integers - size->z = apr_strtoi64(s, &s, 0); - if (*s) - size->c = apr_strtoi64(s, &s, 0); - } // Raster size is 4 params max - if (errno || *s) - return apr_psprintf(p, "%s incorrect", err_prefix); - return NULL; +// Packed block count, for a bit position in a line +static inline uint32_t block_count(uint32_t *values, int bit) { + if (!is_on(values, bit)) + return NPOS; + return (values[0] + + bsc(values[1]) * (((bit / 32) & 1) | (bit / 64)) + + bsc(values[2]) * (bit / 64) + + bsc(values[1 + (bit / 32)] & ((1ULL << (bit % 32)) - 1))); } -// Converts a 64bit value into 13 trigesimal chars -static void uint64tobase32(apr_uint64_t value, char *buffer, int flag = 0) { - static char b32digits[] = "0123456789abcdefghijklmnopqrstuv"; - // From the bottom up - buffer[13] = 0; // End of string marker - for (int i = 0; i < 12; i++, value >>= 5) - buffer[12 - i] = b32digits[value & 0x1f]; - // First char holds the empty tile flag - if (flag) flag = 0x10; // Making sure it has the right value - buffer[0] = b32digits[flag | value]; -} +// How do we map M param mapping to a file name? +enum mappings{ + MAPM_NONE = 0, + MAPM_PREFIX // The M value prefixes the file name, both index and data +}; -// Return the value from a base 32 character -// Returns a negative value if char is not a valid base32 char -static int b32(char ic) { - int c = 0xff & (static_cast(ic)); - if (c < '0') return -1; - if (c - '0' < 10) return c - '0'; - if (c < 'A') return -1; - if (c - 'A' < 22) return c - 'A' + 10; - if (c < 'a') return -1; - if (c - 'a' < 22) return c - 'a' + 10; - return -1; -} +struct mrf_conf { + // array of guard regexp, one of them has to match + apr_array_header_t *arr_rxp; -static apr_uint64_t base32decode(const char *is, int *flag) { - apr_int64_t value = 0; - const unsigned char *s = reinterpret_cast(is); - while (*s == '"') s++; // Skip initial quotes - *flag = (b32(*s) >> 4) & 1; // Pick up the flag from bit 5 - for (int v = b32(*s++) & 0xf; v >= 0; v = b32(*s++)) - value = (value << 5) + v; - return value; -} + // The raster represented by this MRF configuration + TiledRaster raster; -static void mrf_init(apr_pool_t *p, mrf_conf *c) { - struct rset level; - level.width = static_cast(1 + (c->size.x - 1) / c->pagesize.x); - level.height = static_cast(1 + (c->size.y - 1) / c->pagesize.y); - level.offset = 0; - // How many levels do we have - c->n_levels = 2 + ilogb(max(level.height, level.width) - 1); - c->rsets = (struct rset *)apr_pcalloc(p, sizeof(rset) * c->n_levels); - - // Populate rsets from the bottom, the way tile protcols count levels - // These are MRF rsets, not all of them are visible - struct rset *r = c->rsets + c->n_levels - 1; - for (int i = 0; i < c->n_levels; i++) { - *r-- = level; - // Prepare for the next level, assuming powers of two - level.offset += sizeof(TIdx) * level.width * level.height * c->size.z; - level.width = 1 + (level.width - 1) / 2; - level.height = 1 + (level.height - 1) / 2; - } - // MRF has one tile at the top - ap_assert(c->rsets->height == 1 && c->rsets->width == 1); -} + // At least one source, but there could be more + apr_array_header_t *source; -// Allow for one or more RegExp guard -// If present, at least one of them has to match the URL -static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) -{ - char *err_message = NULL; - if (c->arr_rxp == 0) - c->arr_rxp = apr_array_make(cmd->pool, 2, sizeof(ap_regex_t *)); - ap_regex_t **m = (ap_regex_t **)apr_array_push(c->arr_rxp); - *m = ap_pregcomp(cmd->pool, pattern, 0); - return (NULL != *m) ? NULL : "Bad regular expression"; -} + // The MRF index file, required + vfile_t idx; -/* - Read the configuration file, which is a key-value text file, with one key per line - comment lines that start with # - empty lines are allowed, as well as continued lines if the first one ends with \ - However, every line is limited to the Apache max string size, defaults to 8192 chars + // Used for redirect, how many times to try + // defaults to 5 + int retries; - Unknown keys, or keys that are misspelled are silently ignored - Keys are not case sensitive, but values are - Keys and values are space separated. The last value per line, if it is a string, may contain spaces + // If set, only secondary requests are allowed + int indirect; - Supported keys: + // If set, file handles are not held open + int dynamic; - Size X Y - Mandatory, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (usually not meaningful) + // How do we map M param mapping to a file name? + int mmapping; - PageSize X Y <1> - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size + // the canned index header size, or 0 for normal index + uint64_t can_hsize; +}; - DataFile string - Mandatory, the data file of the MRF. +extern module AP_MODULE_DECLARE_DATA mrf_module; - IndexFile string - Optional, The index file name. - If not provided it uses the data file name if its extension is not three letters. - Otherwise it uses the datafile name with the extension changed to .idx +#if defined(APLOG_USE_MODULE) +APLOG_USE_MODULE(mrf); +#endif - MimeType string - Optional. Defaults to autodetect +static void *create_dir_config(apr_pool_t *p, char *dummy) { + mrf_conf *c = reinterpret_cast( + apr_pcalloc(p, sizeof(mrf_conf))); + c->retries = 5; + return c; +} - EmptyTile - Optional. By default it ignores the request if a tile is missing - First number is assumed to be the size, second is offset - If filename is not provided, it uses the data file name +// Parse a comma separated list of sources, add the entries to the array +// Source may include offset and size, white space separated +static const char *parse_sources(cmd_parms *cmd, const char *src, + apr_array_header_t *arr, bool redir = false) +{ + apr_array_header_t *inputs = tokenize(cmd->temp_pool, src, ','); + for (int i = 0; i < inputs->nelts; i++) { + vfile_t *entry = &APR_ARRAY_PUSH(arr, vfile_t); + memset(entry, 0, sizeof(vfile_t)); + char *input = APR_ARRAY_IDX(inputs, i, char *); + + char *fname = ap_getword_white_nc(arr->pool, &input); + if (!fname || strlen(fname) < 1) + return "Source name missing"; + + if (redir) { // Check that it is absolute and add :/ + if (fname[0] != '/') + return apr_pstrcat(cmd->pool, "Only absolute redirects as allowed, ", + fname, " is not absolute", NULL); + fname = apr_pstrcat(arr->pool, ":/", fname, NULL); + } - SkippedLevels - Optional, how many levels to ignore, at the top of the MRF pyramid - For example a GCS pyramid will have to skip the one tile level, so this should be 1 + entry->name = fname; - Redirect - Instead of reading from the data file, make range requests to this URI + // See if there are more arguments, should be offset and size + if (*input != 0) entry->range.offset = strtoull(input, &input, 0); + if (*input != 0) entry->range.size = strtoull(input, &input, 0); + } + return nullptr; +} - ETagSeed base32_string - Optional, 64 bits in base32 digits. Defaults to 0 - The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles - have ETags that depend on this one and bit 64 is zero - */ +#define parse_redirects(cmd, src, arr) parse_sources(cmd, src, arr, true) -static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) +// +// This function sets the MRF specific parameters +// The raster size is defined using the normal libahtse parameters +// Unique directives: +// IndexFile : May be local paths or indirect, if prefixed by :// +// DataFile +// Redirect : Old style redirects, only if DataFile is not present +// RetryCount : // For indirect redirect range requests +// EmptyTile : +// Dynamic On : If the file handles are not to be hold +// +static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) { ap_assert(sizeof(apr_off_t) == 8); mrf_conf *c = (mrf_conf *)dconf; - char *err_message; - apr_table_t *kvp = read_pKVP_from_file(cmd->temp_pool, arg, &err_message); - if (NULL == kvp) return err_message; + const char *err_message, *line; - // Got the parsed kvp table, parse the configuration items - const char *line; - char *err_prefix; - - line = apr_table_get(kvp, "Size"); - if (!line) - return apr_psprintf(cmd->temp_pool, "%s Size directive is mandatory", arg); - err_prefix = apr_psprintf(cmd->temp_pool, "%s Size", arg); - err_message = get_xyzc_size(cmd->temp_pool, &(c->size), line, err_prefix); - if (err_message) return err_message; - - // PageSize is optional, use reasonable defaults - c->pagesize.x = c->pagesize.z = 512; - c->pagesize.c = c->size.c; - c->pagesize.z = 1; - line = apr_table_get(kvp, "PageSize"); - if (line) { - err_prefix = apr_psprintf(cmd->temp_pool, "%s PageSize", arg); - err_message = get_xyzc_size(cmd->temp_pool, &(c->pagesize), line, err_prefix); - if (err_message) return err_message; - } - if (c->pagesize.c != c->size.c || c->pagesize.z != 1) - return apr_psprintf(cmd->temp_pool, "%s PageSize has invalid parameters", arg); + apr_table_t *kvp = readAHTSEConfig(cmd->temp_pool, arg, &err_message); + if (NULL == kvp) + return err_message; - // Initialize the run-time structures - mrf_init(cmd->pool, c); + err_message = configRaster(cmd->pool, kvp, c->raster); + if (err_message) + return err_message; - // The DataFile is optional, if provided the index file is the same thing with the extension removed - line = apr_table_get(kvp, "DataFile"); + // Got the parsed kvp table, parse the configuration items + // Usually there is a single source, but we still need an array + c->source = apr_array_make(cmd->pool, 1, sizeof(vfile_t)); + + // Index file can also be provided, there could be a default + line = apr_table_get(kvp, "IndexFile"); + c->idx.name = apr_pstrdup(cmd->pool, line); + + // The DataFile, required, multiple times, includes redirects + line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"); + if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"))) && + (NULL != (err_message = parse_sources(cmd, line, c->source)))) + return err_message; + + // Old style redirects go at the end + if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "Redirect"))) && + (NULL != (err_message = parse_redirects(cmd, line, c->source)))) + return err_message; + + // Check that we have at least one data file + const char *firstname = nullptr; + if (0 == c->source->nelts || !(firstname = APR_ARRAY_IDX(c->source, 0, vfile_t).name)) + return "Need at least one DataFile directive"; + + line = apr_table_get(kvp, "RetryCount"); + c->retries = 1 + (line ? atoi(line) : 0); + if ((c->retries < 1) || (c->retries > 100)) + return "Invalid RetryCount value, should be 0 to 99"; + + // If an emtpy tile is not provided, it falls through, which results in a 404 error + // If provided, it has an optional size and offset followed by file name which + // defaults to datafile read the empty tile + // Default file name is the name of the first data file, if provided + line = apr_table_get(kvp, "EmptyTile"); + if (line && strlen(line) && (err_message = readFile( + cmd->pool, c->raster.missing.data, line))) + return err_message; - // Data and index in the same location by default - if (line) { // If the data file has a three letter extension, change it to idx for the index - c->datafname = apr_pstrdup(cmd->pool, line); - c->idxfname = apr_pstrdup(cmd->pool, line); + // Set the index file name based on the first data file, if there is only one + if (!c->idx.name) { + c->idx.name = apr_pstrdup(cmd->pool, firstname); char *last; - char *token = apr_strtok(c->idxfname, ".", &last); // strtok destroys the idxfile + char *token = apr_strtok(c->idx.name, ".", &last); // strtok destroys the idxfile while (*last != 0 && token != NULL) token = apr_strtok(NULL, ".", &last); - memcpy(c->idxfname, c->datafname, strlen(c->datafname)); // Get a new copy + memcpy(c->idx.name, firstname, strlen(firstname)); // Get a new copy if (token != NULL && strlen(token) == 3) memcpy(token, "idx", 3); } - // Index file can also be provided - line = apr_table_get(kvp, "IndexFile"); - if (line) - c->idxfname = apr_pstrdup(cmd->pool, line); - - // Mime type is autodetected if not provided - line = apr_table_get(kvp, "MimeType"); - if (line) - c->mime_type = apr_pstrdup(cmd->pool, line); - - // Skip levels, from the top of the MRF - line = apr_table_get(kvp, "SkippedLevels"); - if (line) - c->skip_levels = atoi(line); - - // If an emtpy tile is not provided, it falls through - // If provided, it has an optional size and offset followed by file name which defaults to datafile - // read the empty tile - const char *efname = c->datafname; // Default file name is data file - line = apr_table_get(kvp, "EmptyTile"); - if (line) { - char *last; - // Try to read a figure first - c->esize = apr_strtoi64(line, &last, 0); + if ((line = apr_table_get(kvp, "Dynamic")) && getBool(line)) + c->dynamic = true; - // If that worked, try to get an offset too - if (last != line) - apr_strtoff(&(c->eoffset), last, &last, 0); - - // If there is anything left - while (*last && isspace(*last)) last++; - if (*last != 0) - efname = last; - } + // The original index file size is the number of tiles * 16 + // Since the MRF always ends with a single tile, the total number + // of tiles in the MRF is equal to the number of tiles at level 0 + size.z + if ((line = apr_table_get(kvp, "CannedIndex")) && getBool(line)) + c->can_hsize = hsize((c->raster.size.z + c->raster.rsets[0].tiles) * 16); - line = apr_table_get(kvp, "Redirect"); - if (line) - c->redirect = apr_pstrdup(cmd->pool, line); - - // If we're provided a file name or a size, pre-read the empty tile in the - if (efname && (c->datafname == NULL || apr_strnatcmp(c->datafname, efname) || c->esize)) - { - apr_file_t *efile; - apr_off_t offset = c->eoffset; - apr_status_t stat; - - // Use the temp pool for the file open, it will close it for us - if (!c->esize) { // Don't know the size, get it from the file - apr_finfo_t finfo; - stat = apr_stat(&finfo, efname, APR_FINFO_CSIZE, cmd->temp_pool); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't stat %s %pm", efname, stat); - c->esize = (apr_uint64_t)finfo.csize; + // What parameters we need for M mapping ? + if ((line = apr_table_get(kvp, "MMapping"))) { + if (!apr_strnatcmp(line, "prefix")) { + // Direct mapping means M becomes the prefix for the file name, no folder + c->mmapping = MAPM_PREFIX; } - - stat = apr_file_open(&efile, efname, APR_FOPEN_READ | APR_FOPEN_BINARY, 0, cmd->temp_pool); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't open empty file %s, loaded from %s: %pm", - efname, arg, stat); - c->empty = (apr_uint32_t *)apr_palloc(cmd->pool, static_cast(c->esize)); - stat = apr_file_seek(efile, APR_SET, &offset); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't seek empty tile %s: %pm", efname, stat); - apr_size_t size = (apr_size_t)c->esize; - stat = apr_file_read(efile, c->empty, &size); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't read from %s, loaded from %s: %pm", - efname, arg, stat); - apr_file_close(efile); + else + return "Unknown value for MMapping"; } - line = apr_table_get(kvp, "ETagSeed"); - // Ignore the flag - int flag; - c->seed = line ? base32decode(line, &flag) : 0; - // Set the missing tile etag, with the extra bit set - uint64tobase32(c->seed, c->eETag, 1); - c->enabled = 1; return NULL; } -static int etag_matches(request_rec *r, const char *ETag) { - const char *ETagIn = apr_table_get(r->headers_in, "If-None-Match"); - return ETagIn != 0 && strstr(ETagIn, ETag); -} - -static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) -{ - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); - if (cfg->mime_type) - ap_set_content_type(r, cfg->mime_type); - else - switch (hton32(*buffer)) { - case JPEG_SIG: - ap_set_content_type(r, "image/jpeg"); - break; - case PNG_SIG: - ap_set_content_type(r, "image/png"); - break; - default: // LERC goes here too - ap_set_content_type(r, "application/octet-stream"); - } - // Is it gzipped content? - if (GZIP_SIG == hton32(*buffer)) - apr_table_setn(r->headers_out, "Content-Encoding", "gzip"); - - // TODO: Set headers, as chosen by user - ap_set_content_length(r, size); - ap_rwrite(buffer, size, r); - return OK; -} +static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FOPEN_LARGEFILE; -// Returns the empty tile if defined -static int send_empty_tile(request_rec *r) { - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); - if (etag_matches(r, cfg->eETag)) { - apr_table_setn(r->headers_out, "ETag", cfg->eETag); - return HTTP_NOT_MODIFIED; +// Return the first source which contains the index, adjusts the index offset if necessary +static const vfile_t *pick_source(const apr_array_header_t *sources, range_t *index) { + for (int i = 0; i < sources->nelts; i++) { + vfile_t *source = &APR_ARRAY_IDX(sources, i, vfile_t); + if ((source->range.offset == 0 && source->range.size == 0) + || (index->offset >= source->range.offset + && (source->range.size == 0 + || index->offset - source->range.offset + index->size <= source->range.size))) + { + index->offset -= source->range.offset; + return source; + } } - - if (!cfg->empty) return DECLINED; // Passthrough - return send_image(r, cfg->empty, static_cast(cfg->esize)); + return NULL; } // An open file handle and the matching file name, to be used as a note struct file_note { - const char *name; + const char *fname; apr_file_t *pfh; }; -static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FOPEN_LARGEFILE; - -/* - * Open or retrieve an connection cached file. - */ -static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, const char *name, - apr_int32_t flags = open_flags, const char *note_name = "MRF_INDEX_FILE") - +// +// Open or retrieve a connection cached file handle +// This is a real file +// Do no close the returned file handle, it will be removed when the connection drops +// +// Only one opened handle exists per connection/token pair +// This may lead to less caching, but avoids having too many opened files +// +static apr_status_t openConnFile(request_rec *r, apr_file_t **ppfh, const char *fname, + const char *token, apr_int32_t extra_open_flags = 0) { apr_table_t *conn_notes = r->connection->notes; - // Try to pick it up from the connection notes - file_note *fn = (file_note *) apr_table_get(conn_notes, note_name); - if ((fn != NULL) && !apr_strnatcmp(name, fn->name)) { // Match, set file and return - *ppfh = fn->pfh; + file_note *fnote = (file_note *)apr_table_get(conn_notes, token); + if (fnote && !apr_strnatcmp(fnote->fname, fname)) { // Match, return the handle + *ppfh = fnote->pfh; return APR_SUCCESS; } - // Use the connection pool for the note and file, to ensure it gets closed with the connection + // Use the connection pool, it will close the file when it gets dropped apr_pool_t *pool = r->connection->pool; - - if (fn != NULL) { // We have an file note but it is not the right file - apr_table_unset(conn_notes, note_name); // Unhook the existing note - apr_file_close(fn->pfh); // Close the existing file + if (!fnote) { // new connection file + fnote = reinterpret_cast(apr_pcalloc(pool, sizeof(file_note))); } - else { // no previous note, allocate a clean one - fn = (file_note *)apr_palloc(pool, sizeof(file_note)); + else { // Not the right file, clean it up + apr_table_unset(conn_notes, token); // Does not remove the storage + apr_file_close(fnote->pfh); } - apr_status_t stat = apr_file_open(ppfh, name, flags, 0, pool); - if (stat != APR_SUCCESS) + apr_status_t stat = apr_file_open(ppfh, fname, open_flags || extra_open_flags, 0, pool); + if (APR_SUCCESS != stat) return stat; - // Fill the note and hook it up, then return - fn->pfh = *ppfh; - fn->name = apr_pstrdup(pool, name); // The old string will persist until cleaned by the pool - apr_table_setn(conn_notes, note_name, (const char *) fn); + // Update the note and hook it up before returning + fnote->fname = apr_pstrdup(pool, fname); + fnote->pfh = *ppfh; + apr_table_setn(conn_notes, token, (const char *)fnote); return APR_SUCCESS; } -#define open_index_file open_connection_file +// Like pread, except not really thread safe +// Range reads are done if file starts with :// +// Range offset is offset, size is mgr->size +// The token is used for connection caching -// Open data file optimized for random access if possible -static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, const char *name) +static int vfile_pread(request_rec *r, storage_manager &mgr, + apr_off_t offset, const char *fname, const char *token = "MRF_DATA") { - static const char data_note_name[] = "MRF_DATA_FILE"; + auto cfg = get_conf(r, &mrf_module); + const char *name = fname; -#if defined(APR_FOPEN_RANDOM) - // apr has portable support for random access to files - return open_connection_file(r, ppfh, name, open_flags | APR_FOPEN_RANDOM, data_note_name); -#else + bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/'); + if (redirect) { + // Remote file, just use a range request + // TODO: S3 authorized requests - apr_status_t stat = open_connection_file(r, ppfh, name, open_flags, data_note_name); + // Skip the ":/" used to mark a redirect + name = fname + 2; -#if !defined(POSIX_FADV_RANDOM) - return stat; + ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); + if (!receive_filter) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't find receive filter, did you load mod_receive?"); + return 0; + } + + // Get a buffer for the received image + receive_ctx rctx; + rctx.buffer = static_cast(mgr.buffer); + rctx.maxsize = static_cast(mgr.size); + rctx.size = 0; + + // Data file is on a remote site a range request redirect with a range header + char *Range = apr_psprintf(r->pool, + "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT, + offset, offset + mgr.size); + + // S3 may return less than requested, so we retry the request a couple of times + int tries = cfg->retries; + bool failed = false; + apr_time_t now = apr_time_now(); + do { + request_rec *sr = ap_sub_req_lookup_uri(name, r, r->output_filters); + apr_table_setn(sr->headers_in, "Range", Range); + ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, + sr, sr->connection); + int status = ap_run_sub_req(sr); + ap_remove_output_filter(rf); + ap_destroy_sub_req(sr); + + if ((status != APR_SUCCESS + || sr->status != HTTP_PARTIAL_CONTENT + || static_cast(rctx.size) != mgr.size) + && (0 == tries--)) + { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't fetch data from %s, took %" APR_TIME_T_FMT "us", + name, apr_time_now() - now); + failed = true; + } + } while (!failed && static_cast(rctx.size) != mgr.size); + + return rctx.size; + } // Redirect read + + // Local file + apr_file_t *pfh; + apr_status_t stat; + + int dynamic = 0; + // Keep handles open, except if dynamic is on and fname is the default + if (cfg->dynamic) { + if (!apr_strnatcmp(token, "MRF_DATA")) { + // Only single data file, unmodified can be dynamic + if (1 == cfg->source->nelts) { + dynamic = !apr_strnatcmp(fname, APR_ARRAY_IDX(cfg->source, 0, vfile_t).name); + } + } + else { // Only unmodified index name can be dynamic + dynamic = !apr_strnatcmp(fname, cfg->idx.name); + } + } + + if (dynamic) + stat = apr_file_open(&pfh, fname, open_flags, 0, r->pool); + else + stat = openConnFile(r, &pfh, fname, token, APR_FOPEN_BUFFERED); + + if (stat != APR_SUCCESS) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't open file %s", name); + return 0; // No file + } + + apr_size_t sz = static_cast(mgr.size); + stat = apr_file_seek(pfh, APR_SET, &offset); + if (APR_SUCCESS != stat || APR_SUCCESS != apr_file_read(pfh, mgr.buffer, &sz)) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Read error in %s offset %" APR_OFF_T_FMT, name, offset); + sz = 0; + } -#else // last chance, turn random flag on if supported - apr_os_file_t fd; - if (APR_SUCCESS == apr_os_file_get(&fd, *ppfh)) - posix_fadvise(static_cast(fd), 0, 0, POSIX_FADV_RANDOM); - return stat; + if (dynamic) + apr_file_close(pfh); + // Don't close non-dynamic mode handles, they are reused + + mgr.size = sz; + return static_cast(sz); +} + +// read index, returns error message or null +// MRF index file is network order +static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset, const char *fname) { + auto cfg = get_conf(r, &mrf_module); + storage_manager dst(idx, sizeof(range_t)); + + if (cfg->can_hsize) { // No checks that the file is correct + // Original block offset + uint64_t boffset = offset / BSZ; + + // Read the line containing the target bit + uint32_t line[4]; + storage_manager lmgr(line, 16); + if (16 != vfile_pread(r, lmgr, + 16 * (1 + (boffset / 96)), fname, "MRF_INDEX")) + return "Bitmap read error"; + +#if defined(be32toh) + // Change to host endian + for (int i = 0; i < 4; i++) + line[i] = be32toh(line[i]); +#endif + + // The relocated block number for the original index record + uint64_t blockn = block_count(line, static_cast(boffset % 96)); + if (NPOS == blockn) { + idx->size = idx->offset = 0; + return nullptr; + } + + // Adjust the offset before reading the data + offset = cfg->can_hsize + blockn * BSZ + offset % BSZ; + } + + if (sizeof(range_t) != vfile_pread(r, dst, offset, fname, "MRF_INDEX")) + return "Read error"; + +#if defined(be64toh) + idx->offset = be64toh(idx->offset); + idx->size = be64toh(idx->size); #endif -#endif // APR_FOPEN_RANDOM + + return nullptr; } // Quiet error @@ -465,35 +464,43 @@ static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, const char return HTTP_INTERNAL_SERVER_ERROR; \ } -static bool our_request(request_rec *r, mrf_conf *cfg) { - if (r->method_number != M_GET || cfg->arr_rxp == NULL) - return false; - - char *url_to_match = r->args ? apr_pstrcat(r->pool, r->uri, "?", r->args, NULL) : r->uri; - for (int i = 0; i < cfg->arr_rxp->nelts; i++) { - ap_regex_t *m = APR_ARRAY_IDX(cfg->arr_rxp, i, ap_regex_t *); - if (!ap_regexec(m, url_to_match, 0, NULL, 0)) return true; // Found +// Change the file name depending on the configuration +// Returns nullptr if something went wrong +const char *apply_mmapping(request_rec *r, const sz5* tile, const char *fname) +{ + auto cfg = get_conf(r, &mrf_module); + if (cfg->mmapping == MAPM_NONE || tile->z == 0) + return fname; + + // Switch to C++ + string ret_fname(fname); + switch (cfg->mmapping) { + case MAPM_PREFIX: + size_t bnamepos = ret_fname.find_last_of("/"); + if (bnamepos == string::npos) + return NULL; + ret_fname.insert(bnamepos + 1, apr_ltoa(r->pool, static_cast(tile->z))); + break; } - return false; + return apr_pstrdup(r->pool, ret_fname.c_str()); } +static int handler(request_rec *r) { + if (r->method_number != M_GET) + return DECLINED; -static int handler(request_rec *r) -{ - // Only get and no arguments - if (r->args) return DECLINED; // Don't accept arguments - - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); - if (!cfg->enabled || !our_request(r, cfg)) return DECLINED; + auto cfg = get_conf(r, &mrf_module); + if ((cfg->indirect && !r->main) || + !requestMatches(r, cfg->arr_rxp)) + return DECLINED; apr_array_header_t *tokens = tokenize(r->pool, r->uri, '/'); - if (tokens->nelts < 3) return DECLINED; // At least Level Row Column + if (tokens->nelts < 3) + return DECLINED; // At least Level Row Column // Use a xyzc structure, with c being the level // Input order is M/Level/Row/Column, with M being optional - sz tile; + sz5 tile; memset(&tile, 0, sizeof(tile)); // Need at least three numerical arguments @@ -501,154 +508,103 @@ static int handler(request_rec *r) tile.y = apr_atoi64(*(char **)apr_array_pop(tokens)); REQ_ERR_IF(errno); tile.l = apr_atoi64(*(char **)apr_array_pop(tokens)); REQ_ERR_IF(errno); + const TiledRaster &raster(cfg->raster); + // We can ignore the error on this one, defaults to zero // The parameter before the level can't start with a digit for an extra-dimensional MRF - if (cfg->size.z != 1 && tokens->nelts) + if (tokens->nelts && (raster.size.z != 1 || cfg->mmapping != MAPM_NONE)) tile.z = apr_atoi64(*(char **)apr_array_pop(tokens)); // Don't allow access to levels less than zero, send the empty tile instead - if (tile.l < 0) - return send_empty_tile(r); + if (tile.l < 0 || tile.x < 0 || tile.y < 0) + return sendEmptyTile(r, raster.missing); - tile.l += cfg->skip_levels; + tile.l += raster.skip; // Check for bad requests, outside of the defined bounds - REQ_ERR_IF(tile.l >= cfg->n_levels); - rset *level = cfg->rsets + tile.l; - REQ_ERR_IF(tile.x >= level->width || tile.y >= level->height); + REQ_ERR_IF(tile.l >= static_cast(raster.n_levels)); + rset *level = raster.rsets + tile.l; + REQ_ERR_IF(tile.x >= static_cast(level->w) || tile.y >= static_cast(level->h)); + // Force single z if that's how the MRF is set up, maybe file name mapping applies + apr_int64_t tz = (raster.size.z != 1) ? tile.z : 0; // Offset of the index entry for this tile - apr_off_t tidx_offset = level->offset + - sizeof(TIdx) * (tile.x + level->width * (tile.z * level->height + tile.y)); - - apr_file_t *idxf, *dataf; - SERR_IF(open_index_file(r, &idxf, cfg->idxfname), - apr_psprintf(r->pool, "Can't open %s", cfg->idxfname)); - TIdx index; - apr_size_t read_size = sizeof(TIdx); - - SERR_IF(apr_file_seek(idxf, APR_SET, &tidx_offset) - || apr_file_read(idxf, &index, &read_size) - || read_size != sizeof(TIdx), - apr_psprintf(r->pool, "Tile index doesn't exist in %s", cfg->idxfname)); + apr_off_t tidx_offset = sizeof(range_t) * (level->tiles + + + level->w * (tz * level->h + tile.y) + tile.x); + + range_t index; + const char *idx_fname = apply_mmapping(r, &tile, cfg->idx.name); + const char *message = read_index(r, &index, tidx_offset, idx_fname); + if (message) { // Fatal error + if (!apr_strnatcmp(idx_fname, cfg->idx.name)) { + SERR_IF(message, message); + } + REQ_ERR_IF(message); + } // MRF index record is in network order - index.size = ntoh64(index.size); - index.offset = ntoh64(index.offset); - if (index.size < 4) // Need at least four bytes for signature check - return send_empty_tile(r); + return sendEmptyTile(r, raster.missing); - if (MAX_TILE_SIZE < index.size) { // Tile is too large, log and send error code - ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "Tile too large in %s", cfg->idxfname); - return HTTP_INTERNAL_SERVER_ERROR; - } + SERR_IF(MAX_TILE_SIZE < index.size, apr_pstrcat(r->pool, + "Tile too large found in ", idx_fname, NULL)); - // Check for conditional ETag here, no need to open the data file + // Check for conditional ETag here, no need to get the data char ETag[16]; // Try to distribute the bits a bit to generate an ETag - uint64tobase32(cfg->seed ^ (index.size << 40) ^ index.offset, ETag); - if (etag_matches(r, ETag)) { + tobase32((raster.seed ^ (index.size << 40)) ^ index.offset, ETag); + if (etagMatches(r, ETag)) { apr_table_set(r->headers_out, "ETag", ETag); return HTTP_NOT_MODIFIED; } // Now for the data part - if (!cfg->datafname && !cfg->redirect) - SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); - - apr_uint32_t *buffer = static_cast( - apr_palloc(r->pool, static_cast(index.size))); - - if (cfg->redirect) { - // TODO: S3 authorized requests - ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); - SERR_IF(!receive_filter, "Redirect needs mod_receive to be available"); - - // Get a buffer for the received image - receive_ctx rctx; - rctx.buffer = reinterpret_cast(buffer); - rctx.maxsize = static_cast(index.size); - rctx.size = 0; + const vfile_t *src = pick_source(cfg->source, &index); + const char *name = (src && src->name) ? apply_mmapping(r, &tile, src->name) : nullptr; + SERR_IF(!name, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); - ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, r, r->connection); - request_rec *sr = ap_sub_req_lookup_uri(cfg->redirect, r, r->output_filters); + apr_size_t size = static_cast(index.size); + storage_manager img(apr_palloc(r->pool, size), size); - // Data file is on a remote site a range request redirect with a range header - static const char *rfmt = "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT; - char *Range = apr_psprintf(r->pool, rfmt, index.offset, index.offset + index.size); - apr_table_setn(sr->headers_in, "Range", Range); - int status = ap_run_sub_req(sr); - ap_remove_output_filter(rf); - - if (status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT - || rctx.size != static_cast(index.size)) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s", cfg->redirect); - return HTTP_SERVICE_UNAVAILABLE; - } - apr_table_clear(r->headers_out); - } - else - { // Read from a local file - SERR_IF(open_data_file(r, &dataf, cfg->datafname), - apr_psprintf(r->pool, "Can't open %s", cfg->datafname)); - - // We got the tile index, and is not empty - SERR_IF(!buffer, - "Memory allocation error in mod_mrf"); - SERR_IF(apr_file_seek(dataf, APR_SET, (apr_off_t *)&index.offset), - apr_psprintf(r->pool, "Seek error in %s", cfg->datafname)); - read_size = static_cast(index.size); - SERR_IF(apr_file_read(dataf, buffer, &read_size) || read_size != index.size, - apr_psprintf(r->pool, "Can't read from %s", cfg->datafname)); - } + SERR_IF(!img.buffer, "Memory allocation error in mod_mrf"); + SERR_IF(img.size != static_cast(vfile_pread(r, img, index.offset, name)), + "Data read error"); // Looks fine, set the outgoing etag and then the image apr_table_set(r->headers_out, "ETag", ETag); - return send_image(r, buffer, static_cast(index.size)); + return sendImage(r, img); } -static const command_rec mrf_cmds[] = -{ - AP_INIT_FLAG( - "MRF", - CMD_FUNC ap_set_flag_slot, - (void *)APR_OFFSETOF(mrf_conf, enabled), - ACCESS_CONF, - "mod_mrf enable, defaults to on if configuration is provided" - ), - +static const command_rec cmds[] = { AP_INIT_TAKE1( - "MRF_ConfigurationFile", - CMD_FUNC mrf_file_set, // Callback + "MRF_RegExp", + (cmd_func)set_regexp, 0, // Self-pass argument ACCESS_CONF, // availability - "The configuration file for this module" + "Regular expression that the URL has to match. At least one is required." + ), + + AP_INIT_FLAG( + "MRF_Indirect", + (cmd_func) ap_set_flag_slot, + (void *)APR_OFFSETOF(mrf_conf, indirect), + ACCESS_CONF, + "If set, this configuration only responds to subrequests" ), AP_INIT_TAKE1( - "MRF_RegExp", - (cmd_func)set_regexp, + "MRF_ConfigurationFile", + (cmd_func) file_set, // Callback 0, // Self-pass argument ACCESS_CONF, // availability - "Regular expression that the URL has to match. At least one is required." + "The AHTSE configuration file for this module" ), { NULL } }; - -// Return OK or DECLINED, anything else is error -//static int check_config(apr_pool_t *pconf, apr_pool_t *plog, apr_pool_t *ptemp, server_rec *server) -//{ -// return DECLINED; - // This gets called once for the whole server, it would have to check the configuration for every folder -//} - -static void mrf_register_hooks(apr_pool_t *p) - -{ - ap_hook_handler(handler, NULL, NULL, APR_HOOK_FIRST); - // ap_hook_check_config(check_config, NULL, NULL, APR_HOOK_MIDDLE); +static void register_hooks(apr_pool_t *p) { + // Up in the stack, but leave APR_HOOK_FIRST available + ap_hook_handler(handler, NULL, NULL, APR_HOOK_FIRST + 1); } module AP_MODULE_DECLARE_DATA mrf_module = { @@ -657,6 +613,6 @@ module AP_MODULE_DECLARE_DATA mrf_module = { 0, // No dir_merge 0, // No server_config 0, // No server_merge - mrf_cmds, // configuration directives - mrf_register_hooks // processing hooks + cmds, // configuration directives + register_hooks // processing hooks }; diff --git a/src/mod_mrf.h b/src/mod_mrf.h deleted file mode 100644 index 6b271b5..0000000 --- a/src/mod_mrf.h +++ /dev/null @@ -1,117 +0,0 @@ -/* -* mod_mrf header file -* Lucian Plesea -* (C) 2016 -*/ - -#if !defined(MOD_MRF_H) -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define APR_WANT_STRFUNC -#define APR_WANT_MEMFUNC -#include - -#define CMD_FUNC (cmd_func) - -// The maximum size of a tile, to avoid MRF corruption errors -#define MAX_TILE_SIZE 4*1024*1024 - -// signatures in big endian, to autodetect tile type -#define PNG_SIG 0x89504e47 -#define JPEG_SIG 0xffd8ffe0 -#define LERC_SIG 0x436e745a - -// This one is not a type, just an encoding -#define GZIP_SIG 0x436e745a - -// Conversion to and from network order, endianess depenent - -#if (APR_IS_BIGENDIAN == 0) // Little endian -#if defined(WIN32) // Windows -#define ntoh32(v) _byteswap_ulong(v) -#define hton32(v) _byteswap_ulong(v) -#define ntoh64(v) _byteswap_uint64(v) -#define hton64(v) _byteswap_uint64(v) -#else // Assume linux -#define ntoh32(v) __builtin_bswap32(v) -#define hton32(v) __builtin_bswap32(v) -#define ntoh64(v) __builtin_bswap64(v) -#define hton64(v) __builtin_bswap64(v) -#endif -#else // Big endian, do nothing -#define ntoh32(v) (v) -#define ntoh64(v) (v) -#define hton32(v) (v) -#define hton64(v) (v) -#endif - -#if defined(APLOG_USE_MODULE) -APLOG_USE_MODULE(mrf); -#endif - -struct sz { - apr_int64_t x, y, z, c, l; -}; - -struct rset { - apr_off_t offset; - // in tiles - int width; - // in tiles - int height; -}; - -typedef struct { - apr_uint64_t offset; - apr_uint64_t size; -} TIdx; - -typedef struct { - // array of guard regexp, one of them has to match - apr_array_header_t *arr_rxp; - // The mrf data file name - char *datafname; - // The mrf index file name - char *idxfname; - // Forced mime-type, default is autodetected - char *mime_type; - // Full raster size in pixels - struct sz size; - // Page size in pixels - struct sz pagesize; - - // Levels to skip at the top - int skip_levels; - int n_levels; - struct rset *rsets; - - // Empty tile buffer, if provided - apr_uint32_t *empty; - // Size of empty tile, in bytes - apr_int64_t esize; - apr_off_t eoffset; - - // Turns the module functionality off - int enabled; - - // ETag initializer - apr_uint64_t seed; - // Buffer for the emtpy tile etag - char eETag[16]; - // The internal redirect path or null - char *redirect; - -} mrf_conf; - -extern module AP_MODULE_DECLARE_DATA mrf_module; - -#endif