UTF8 conversion stuff.

This is for dealing with Unicode file names which cannot be handled with an idealized Unicode approach as was present before. utf8proc was included to handle normalization because with externally provided filenames we have no control over enforcing our pld rules.
ZDoom · Dec 10, 2023 · 4da196a · 4da196a
1 parent df56fc6
commit 4da196a
Show file tree

Hide file tree

Showing 7 changed files with 18,773 additions and 0 deletions.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -596,6 +596,7 @@ file( GLOB HEADER_FILES
 	common/thirdparty/rapidjson/*.h
 	common/thirdparty/math/*h
 	common/thirdparty/libsmackerdec/include/*.h
+	common/thirdparty/utf8proc/*.h
 	common/rendering/*.h
 	common/rendering/gl_load/*.h
 	common/rendering/gles/*.h
@@ -1185,6 +1186,7 @@ set( GAME_SOURCES
 	${PCH_SOURCES}
 	common/utility/x86.cpp
 	common/thirdparty/strnatcmp.c
+	common/thirdparty/utf8proc/utf8proc.c
 	common/thirdparty/stb/stb_sprintf.c
 	common/utility/zstring.cpp
 	common/utility/findfile.cpp
@@ -1224,6 +1226,7 @@ set( GAME_SOURCES
 	common/filesystem/source/files_decompress.cpp
 	common/filesystem/source/fs_findfile.cpp
 	common/filesystem/source/fs_stringpool.cpp
+	common/filesystem/source/unicode.cpp
 
 )
 
@@ -1265,6 +1268,7 @@ include_directories(
 	common/thirdparty/libsmackerdec/include
 	common/thirdparty
 	common/thirdparty/stb
+	common/thirdparty/utf8proc
 	common/textures
 	common/textures/formats
 	common/textures/hires
@@ -1508,6 +1512,7 @@ source_group("Common\\Third Party\\Math" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SO
 source_group("Common\\Third Party\\RapidJSON" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/rapidjson/.+")
 source_group("Common\\Third Party\\SFMT" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/sfmt/.+")
 source_group("Common\\Third Party\\stb" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/stb/.+")
+source_group("Common\\Third Party\\utf8proc" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/utf8proc/.+")
 source_group("Utility" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/utility/.+")
 source_group("Utility\\Node Builder" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/utility/nodebuilder/.+")
 source_group("Utility\\Smackerdec" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/smackerdec/.+")

diff --git a/src/common/filesystem/source/unicode.cpp b/src/common/filesystem/source/unicode.cpp
@@ -0,0 +1,143 @@
+/*
+** unicode.cpp
+** handling for conversion / comparison of filenames
+**
+**---------------------------------------------------------------------------
+** Copyright 2019 Christoph Oelckers
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+*/
+
+#include "unicode.h"
+#include "utf8proc.h"
+
+namespace FileSys {
+//==========================================================================
+//
+//
+//
+//==========================================================================
+
+static void utf8_encode(int32_t codepoint, std::vector<char>& buffer)
+{
+	if (codepoint < 0 || codepoint > 0x10FFFF || (codepoint >= 0xD800 && codepoint < 0xDFFF))
+	{
+		codepoint = 0xFFFD;
+	}
+	uint8_t buf[4];
+	auto size = utf8proc_encode_char(codepoint, buf);
+	for(int i = 0; i < size; i++)
+		buffer.push_back(buf[i]);
+}
+
+
+//==========================================================================
+//
+// convert UTF16 to UTF8 (needed for 7z)
+//
+//==========================================================================
+
+void utf16_to_utf8(const unsigned short* in, std::vector<char>& buffer)
+{
+	buffer.clear();
+	if (!*in) return;
+
+	while (int char1 = *in++)
+	{
+		if (char1 >= 0xD800 && char1 <= 0xDBFF)
+		{
+			int char2 = *in;
+
+			if (char2 >= 0xDC00 && char2 <= 0xDFFF)
+			{
+				in++;
+				char1 -= 0xD800;
+				char2 -= 0xDC00;
+				char1 <<= 10;
+				char1 += char2;
+				char1 += 0x010000;
+			}
+			else
+			{
+				// invalid code point - replace with placeholder
+				char1 = 0xFFFD;
+			}
+		}
+		else if (char1 >= 0xDC00 && char1 <= 0xDFFF)
+		{
+			// invalid code point - replace with placeholder
+			char1 = 0xFFFD;
+		}
+		utf8_encode(char1, buffer);
+	}
+	buffer.push_back(0);
+}
+
+//==========================================================================
+//
+// convert UTF16 to UTF8 (needed for Zip)
+//
+//==========================================================================
+
+void ibm437_to_utf8(const char* in, std::vector<char>& buffer)
+{
+	static const uint16_t ibm437map[] = {
+		0x00c7,	0x00fc,	0x00e9,	0x00e2,	0x00e4,	0x00e0,	0x00e5,	0x00e7,	0x00ea,	0x00eb,	0x00e8,	0x00ef,	0x00ee,	0x00ec,	0x00c4,	0x00c5,
+		0x00c9,	0x00e6,	0x00c6,	0x00f4,	0x00f6,	0x00f2,	0x00fb,	0x00f9,	0x00ff,	0x00d6,	0x00dc,	0x00a2,	0x00a3,	0x00a5,	0x20a7,	0x0192,
+		0x00e1,	0x00ed,	0x00f3,	0x00fa,	0x00f1,	0x00d1,	0x00aa,	0x00ba,	0x00bf,	0x2310,	0x00ac,	0x00bd,	0x00bc,	0x00a1,	0x00ab,	0x00bb,
+		0x2591,	0x2592,	0x2593,	0x2502,	0x2524,	0x2561,	0x2562,	0x2556,	0x2555,	0x2563,	0x2551,	0x2557,	0x255d,	0x255c,	0x255b,	0x2510,
+		0x2514,	0x2534,	0x252c,	0x251c,	0x2500,	0x253c,	0x255e,	0x255f,	0x255a,	0x2554,	0x2569,	0x2566,	0x2560,	0x2550,	0x256c,	0x2567,
+		0x2568,	0x2564,	0x2565,	0x2559,	0x2558,	0x2552,	0x2553,	0x256b,	0x256a,	0x2518,	0x250c,	0x2588,	0x2584,	0x258c,	0x2590,	0x2580,
+		0x03b1,	0x00df,	0x0393,	0x03c0,	0x03a3,	0x03c3,	0x00b5,	0x03c4,	0x03a6,	0x0398,	0x03a9,	0x03b4,	0x221e,	0x03c6,	0x03b5,	0x2229,
+		0x2261,	0x00b1,	0x2265,	0x2264,	0x2320,	0x2321,	0x00f7,	0x2248,	0x00b0,	0x2219,	0x00b7,	0x221a,	0x207f,	0x00b2,	0x25a0,	0x00a0,
+	};
+
+	buffer.clear();
+	if (!*in) return;
+
+	while (int char1 = (uint8_t)*in++)
+	{
+		if (char1 >= 0x80) char1 = ibm437map[char1];
+		utf8_encode(char1, buffer);
+	}
+	buffer.push_back(0);
+}
+
+//==========================================================================
+//
+// create a normalized lowercase version of a string.
+//
+//==========================================================================
+
+char *tolower_normalize(const char *str) 
+{
+	utf8proc_uint8_t *retval;
+	utf8proc_map((const uint8_t*)str, 0, &retval, (utf8proc_option_t)(UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_CASEFOLD));
+	return (char*)retval;
+}
+
+}
diff --git a/src/common/filesystem/source/unicode.h b/src/common/filesystem/source/unicode.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <stdint.h>
+#include <vector>
+namespace FileSys {
+
+void utf16_to_utf8(const unsigned short* in, std::vector<char>& buffer);
+void ibm437_to_utf8(const char* in, std::vector<char>& buffer);
+int unicode_tolower(int c);
+char *tolower_normalize(const char *str);
+
+}
diff --git a/src/common/thirdparty/utf8proc/LICENSE.md b/src/common/thirdparty/utf8proc/LICENSE.md
@@ -0,0 +1,93 @@
+## utf8proc license ##
+
+**utf8proc** is a software package originally developed
+by Jan Behrens and the rest of the Public Software Group, who
+deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers.  Like the original utf8proc,
+whose copyright and license statements are reproduced below, all new
+work on the utf8proc library is licensed under the [MIT "expat"
+license](http://opensource.org/licenses/MIT):
+
+*Copyright &copy; 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Original utf8proc license ##
+
+*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Unicode data license ##
+
+This software contains data (`utf8proc_data.c`) derived from processing
+the Unicode data files. The following license applies to that data:
+
+**COPYRIGHT AND PERMISSION NOTICE**
+
+*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+under the Terms of Use in http://www.unicode.org/copyright.html.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of the Unicode data files and any associated documentation (the "Data
+Files") or Unicode software and any associated documentation (the
+"Software") to deal in the Data Files or Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, and/or sell copies of the Data Files or Software, and
+to permit persons to whom the Data Files or Software are furnished to do
+so, provided that (a) the above copyright notice(s) and this permission
+notice appear with all copies of the Data Files or Software, (b) both the
+above copyright notice(s) and this permission notice appear in associated
+documentation, and (c) there is clear notice in each modified Data File or
+in the Software as well as in the documentation associated with the Data
+File(s) or Software that the data or software has been modified.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
+
+Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
+registered in some jurisdictions. All other trademarks and registered
+trademarks mentioned herein are the property of their respective owners.