Utf-8/unicode support in legacy tools and lib. (KhronosGroup#800)

Remove all generic text macros.

Fixes KhronosGroup#764.

The *NamedFile* functions now accept full utf8 filenames on any platform. Windows applications must use WideCharToMultiByte to convert unicode filenames before calling these functions.

lib/info.c

@@ -1251,11 +1251,7 @@ ktxPrintKTX2InfoJSONForNamedFile(const char* const filename, ktx_uint32_t base_i
 {
     FILE* file = NULL;
 
-#ifdef _WIN32
-    fopen_s(&file, filename, "rb");
-#else
-    file = fopen(filename, "rb");
-#endif
+    file = ktxFOpenUTF8(filename, "rb");
 
     if (!file)
         return KTX_FILE_OPEN_FAILED;

lib/ktxint.h

@@ -293,6 +293,52 @@ KTX_error_code ktxUncompressZLIBInt(unsigned char* pDest,
 
 KTX_error_code printKTX2Info2(ktxStream* src, KTX_header2* header);
 
+/*
+ * fopen a file identified by a UTF-8 path.
+ */
+#if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <assert.h>
+#include <windows.h>
+#include <shellapi.h>
+#include <stdlib.h>
+
+// For Windows, we convert the UTF-8 path and mode to UTF-16 path and use _wfopen
+// which correctly handles unicode characters.
+static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
+    int wpLen = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
+    int wmLen = MultiByteToWideChar(CP_UTF8, 0, mode, -1, NULL, 0);
+    FILE* fp = NULL;
+    if (wpLen > 0 && wmLen > 0)
+    {
+        wchar_t* wpath = (wchar_t*)malloc(wpLen * sizeof(wchar_t));
+        wchar_t* wmode = (wchar_t*)malloc(wmLen * sizeof(wchar_t));
+        MultiByteToWideChar(CP_UTF8, 0, path, -1, wpath, wpLen);
+        MultiByteToWideChar(CP_UTF8, 0, mode, -1, wmode, wmLen);
+        // Returned errmo_t value is also set in the global errno.
+        // Apps use that for error detail as libktx only returns
+        // KTX_FILE_OPEN_FAILED.
+        (void)_wfopen_s(&fp, wpath, wmode);
+        free(wpath);
+        free(wmode);
+        return fp;
+    } else {
+        assert(KTX_FALSE && "ktxFOpenUTF8 called with zero length path or mode.");
+        return NULL;
+    }
+}
+#else
+// For other platforms there is no need for any conversion, they support UTF-8 natively
+static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
+    return fopen(path, mode);
+}
+#endif
+
 #ifdef __cplusplus
 }
 #endif

lib/texture.c

@@ -408,7 +408,7 @@ ktxTexture_CreateFromNamedFile(const char* const filename,
     if (filename == NULL || newTex == NULL)
         return KTX_INVALID_VALUE;
 
-    file = fopen(filename, "rb");
+    file = ktxFOpenUTF8(filename, "rb");
     if (!file)
        return KTX_FILE_OPEN_FAILED;
 

lib/texture1.c

@@ -458,6 +458,9 @@ ktxTexture1_constructFromStdioStream(ktxTexture1* This, FILE* stdioStream,
  * @memberof ktxTexture1 @private
  * @brief Construct a ktxTexture1 from a named KTX file.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * See ktxTextureInt_constructFromStream for details.
  *
  * @param[in] This pointer to a ktxTextureInt-sized block of memory to
@@ -484,7 +487,7 @@ ktxTexture1_constructFromNamedFile(ktxTexture1* This,
     if (This == NULL || filename == NULL)
         return KTX_INVALID_VALUE;
 
-    file = fopen(filename, "rb");
+    file = ktxFOpenUTF8(filename, "rb");
     if (!file)
        return KTX_FILE_OPEN_FAILED;
 
@@ -681,6 +684,9 @@ ktxTexture1_CreateFromStdioStream(FILE* stdioStream,
  * The address of a newly created texture reflecting the contents of the
  * file is written to the location pointed at by @p newTex.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * The create flag KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT should not be set,
  * if the ktxTexture1 is ultimately to be uploaded to OpenGL or Vulkan. This
  * will minimize memory usage by allowing, for example, loading the images

lib/texture2.c

@@ -1101,6 +1101,9 @@ ktxTexture2_constructFromStdioStream(ktxTexture2* This, FILE* stdioStream,
  * @~English
  * @brief Construct a ktxTexture from a named KTX file.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * See ktxTextureInt_constructFromStream for details.
  *
  * @param[in] This pointer to a ktxTextureInt-sized block of memory to
@@ -1127,7 +1130,7 @@ ktxTexture2_constructFromNamedFile(ktxTexture2* This,
     if (This == NULL || filename == NULL)
         return KTX_INVALID_VALUE;
 
-    file = fopen(filename, "rb");
+    file = ktxFOpenUTF8(filename, "rb");
     if (!file)
        return KTX_FILE_OPEN_FAILED;
 
@@ -1378,6 +1381,9 @@ ktxTexture2_CreateFromStdioStream(FILE* stdioStream,
  * The address of a newly created ktxTexture2 reflecting the contents of the
  * file is written to the location pointed at by @p newTex.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * The create flag KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT should not be set,
  * if the ktxTexture is ultimately to be uploaded to OpenGL or Vulkan. This
  * will minimize memory usage by allowing, for example, loading the images

lib/writer1.c

@@ -397,6 +397,9 @@ ktxTexture1_WriteToStdioStream(ktxTexture1* This, FILE* dstsstr)
  * @~English
  * @brief Write a ktxTexture object to a named file in KTX format.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * @param[in] This      pointer to the target ktxTexture object.
  * @param[in] dstname   destination file name.
  *
@@ -422,7 +425,7 @@ ktxTexture1_WriteToNamedFile(ktxTexture1* This, const char* const dstname)
     if (!This)
         return KTX_INVALID_VALUE;
 
-    dst = fopen(dstname, "wb");
+    dst = ktxFOpenUTF8(dstname, "wb");
     if (dst) {
         result = ktxTexture1_WriteToStdioStream(This, dst);
         fclose(dst);
@@ -842,6 +845,9 @@ ktxTexture1_WriteKTX2ToStdioStream(ktxTexture1* This, FILE* dstsstr)
  * @~English
  * @brief Write a ktxTexture object to a named file in KTX2 format.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
  * It can be added by code, similar to the following, prior to calling this
  * function.
@@ -878,7 +884,7 @@ ktxTexture1_WriteKTX2ToNamedFile(ktxTexture1* This, const char* const dstname)
     if (!This)
         return KTX_INVALID_VALUE;
 
-    dst = fopen(dstname, "wb");
+    dst = ktxFOpenUTF8(dstname, "wb");
     if (dst) {
         result = ktxTexture1_WriteKTX2ToStdioStream(This, dst);
         fclose(dst);

lib/writer2.c

@@ -632,6 +632,9 @@ ktxTexture2_WriteToStdioStream(ktxTexture2* This, FILE* dstsstr)
  * @~English
  * @brief Write a ktxTexture object to a named file in KTX format.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
  * It can be added by code, similar to the following, prior to calling this
  * function.
@@ -668,7 +671,7 @@ ktxTexture2_WriteToNamedFile(ktxTexture2* This, const char* const dstname)
     if (!This)
         return KTX_INVALID_VALUE;
 
-    dst = fopen(dstname, "wb");
+    dst = ktxFOpenUTF8(dstname, "wb");
     if (dst) {
         result = ktxTexture2_WriteToStdioStream(This, dst);
         fclose(dst);

utils/argparser.cpp

@@ -23,29 +23,31 @@
 #include <assert.h>
 #include "argparser.h"
 
+using namespace std;
+
 /*
  * Construct from a string of arguments.
  */
-argvector::argvector(const _tstring& sArgs)
+argvector::argvector(const string& sArgs)
 {
-    const _tstring sep(_T(" \t\n\r\v\f"));
+    const string sep(" \t\n\r\v\f");
     size_t pos;
 
     pos = sArgs.find_first_not_of(sep);
-    assert(pos != _tstring::npos);
+    assert(pos != string::npos);
 
     do {
         size_t epos = sArgs.find_first_of(sep, pos);
-        size_t len = epos == _tstring::npos ? epos : epos - pos;
+        size_t len = epos == string::npos ? epos : epos - pos;
         push_back(sArgs.substr(pos, len));
         pos = sArgs.find_first_not_of(sep, epos);
-    } while (pos != _tstring::npos);
+    } while (pos != string::npos);
 }
 
 /*
  * Construct from an array of C strings
  */
-argvector::argvector(int argc, const _TCHAR* const* argv)
+argvector::argvector(int argc, const char* const* argv)
 {
     for (int i = 0; i < argc; i++) {
         push_back(argv[i]);
@@ -56,24 +58,24 @@ argvector::argvector(int argc, const _TCHAR* const* argv)
  * Functions the same as getopt_long. See `man 3 getopt_long`.
  */
 int
-argparser::getopt(_tstring* shortopts, const struct option* longopts,
+argparser::getopt(string* shortopts, const struct option* longopts,
                   int* /*longindex*/)
 {
     if (optind == argv.size())
         return -1;
 
-    _tstring arg;
+    string arg;
     arg = argv[optind];
-    if (arg[0] != _T('-') || (arg[0] == _T('-') && arg.size() == 1))
+    if (arg[0] != '-' || (arg[0] == '-' && arg.size() == 1))
         return -1;
     optind++;
 
     int retval = '?';
-    if (arg.compare(0, 2, _T("--")) == 0) {
+    if (arg.compare(0, 2, "--") == 0) {
         if (arg.size() == 2) return -1; // " -- " separates options and files
         const struct option* opt = longopts;
         while (opt->name != nullptr) {
-            if (arg.compare(2, _tstring::npos, opt->name) == 0) {
+            if (arg.compare(2, string::npos, opt->name) == 0) {
                 retval = opt->val;
                 if (opt->has_arg != option::no_argument) {
                     if (optind >= argv.size() || (optarg = argv[optind++])[0] == '-') {
@@ -91,9 +93,9 @@ argparser::getopt(_tstring* shortopts, const struct option* longopts,
             }
             opt++;
         }
-    } else if (shortopts != nullptr && arg.compare(0, 1, _T("-")) == 0) {
+    } else if (shortopts != nullptr && arg.compare(0, 1, "-") == 0) {
         size_t pos = shortopts->find(arg.substr(1, 1));
-        if (pos != _tstring::npos) {
+        if (pos != string::npos) {
             retval = (*shortopts)[pos];
             if (pos < shortopts->length()
                 && ((*shortopts)[++pos] == ':' || (*shortopts)[pos] == ';')) {
@@ -118,7 +120,7 @@ std::istream& operator >> (std::istream& stream, const skip& x)
     stream >> std::noskipws;
 
     char c;
-    const _TCHAR* text = x.text;
+    const char* text = x.text;
     while (stream && *text++)
         stream >> c;
 

utils/argparser.h

@@ -13,26 +13,12 @@
 #include <sstream>
 #include <string>
 #include <vector>
-#if defined(_WIN32)
-  #include <tchar.h>
-#elif !defined(_TCHAR)
-  #define _TCHAR char
-  #define _T(x) x
-#endif
-#if !defined(_tstring)
-  #if defined(_UNICODE)
-    #define _tstring std::wstring
-  #else
-    #define _tstring std::string
-  #endif
-#endif
 
-
-class argvector : public std::vector<_tstring> {
+class argvector : public std::vector<std::string> {
   public:
     argvector() { };
-    argvector(const _tstring& argstring);
-    argvector(int argc, const _TCHAR* const* argv);
+    argvector(const std::string& argstring);
+    argvector(int argc, const char* const* argv);
 };
 
 class argparser {
@@ -46,17 +32,17 @@ class argparser {
         option(const char* name, has_arg_t has_arg, int* flag, int val) : name(name), has_arg(has_arg), flag(flag), val(val) {}
     };
 
-    _tstring optarg;
+    std::string optarg;
     unsigned int optind;
     argvector argv;
 
     argparser(argvector& argv, unsigned int startindex = 0)
         : optind(startindex), argv(argv) { }
 
-    argparser(int argc, const _TCHAR* const* argv1)
+    argparser(int argc, const char* const* argv1)
         : optind(1), argv(argc, argv1)  { }
 
-    int getopt(_tstring* shortopts, const struct option* longopts,
+    int getopt(std::string* shortopts, const struct option* longopts,
                int* longindex = nullptr);
 };
 
@@ -66,8 +52,8 @@ class argparser {
 // does not check whether the skipped characters are the same as it
 struct skip
 {
-    const _TCHAR* text;
-    skip(const _TCHAR* text) : text(text) {}
+    const char* text;
+    skip(const char* text) : text(text) {}
 };
 
 std::istream& operator >> (std::istream& stream, const skip& x);