diff --git a/lib/info.c b/lib/info.c index bde958453d..7fbb47c8e1 100644 --- a/lib/info.c +++ b/lib/info.c @@ -1251,11 +1251,7 @@ ktxPrintKTX2InfoJSONForNamedFile(const char* const filename, ktx_uint32_t base_i { FILE* file = NULL; -#ifdef _WIN32 - fopen_s(&file, filename, "rb"); -#else - file = fopen(filename, "rb"); -#endif + file = ktxFOpenUTF8(filename, "rb"); if (!file) return KTX_FILE_OPEN_FAILED; diff --git a/lib/ktxint.h b/lib/ktxint.h index 7656aa9e48..a5c156578e 100644 --- a/lib/ktxint.h +++ b/lib/ktxint.h @@ -293,6 +293,52 @@ KTX_error_code ktxUncompressZLIBInt(unsigned char* pDest, KTX_error_code printKTX2Info2(ktxStream* src, KTX_header2* header); +/* + * fopen a file identified by a UTF-8 path. + */ +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#include + +// For Windows, we convert the UTF-8 path and mode to UTF-16 path and use _wfopen +// which correctly handles unicode characters. +static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) { + int wpLen = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0); + int wmLen = MultiByteToWideChar(CP_UTF8, 0, mode, -1, NULL, 0); + FILE* fp = NULL; + if (wpLen > 0 && wmLen > 0) + { + wchar_t* wpath = (wchar_t*)malloc(wpLen * sizeof(wchar_t)); + wchar_t* wmode = (wchar_t*)malloc(wmLen * sizeof(wchar_t)); + MultiByteToWideChar(CP_UTF8, 0, path, -1, wpath, wpLen); + MultiByteToWideChar(CP_UTF8, 0, mode, -1, wmode, wmLen); + // Returned errmo_t value is also set in the global errno. + // Apps use that for error detail as libktx only returns + // KTX_FILE_OPEN_FAILED. + (void)_wfopen_s(&fp, wpath, wmode); + free(wpath); + free(wmode); + return fp; + } else { + assert(KTX_FALSE && "ktxFOpenUTF8 called with zero length path or mode."); + return NULL; + } +} +#else +// For other platforms there is no need for any conversion, they support UTF-8 natively +static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) { + return fopen(path, mode); +} +#endif + #ifdef __cplusplus } #endif diff --git a/lib/texture.c b/lib/texture.c index a4b046543f..76821e195b 100644 --- a/lib/texture.c +++ b/lib/texture.c @@ -408,7 +408,7 @@ ktxTexture_CreateFromNamedFile(const char* const filename, if (filename == NULL || newTex == NULL) return KTX_INVALID_VALUE; - file = fopen(filename, "rb"); + file = ktxFOpenUTF8(filename, "rb"); if (!file) return KTX_FILE_OPEN_FAILED; diff --git a/lib/texture1.c b/lib/texture1.c index 23a6b7d9c5..1591f3b6dc 100644 --- a/lib/texture1.c +++ b/lib/texture1.c @@ -458,6 +458,9 @@ ktxTexture1_constructFromStdioStream(ktxTexture1* This, FILE* stdioStream, * @memberof ktxTexture1 @private * @brief Construct a ktxTexture1 from a named KTX file. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * See ktxTextureInt_constructFromStream for details. * * @param[in] This pointer to a ktxTextureInt-sized block of memory to @@ -484,7 +487,7 @@ ktxTexture1_constructFromNamedFile(ktxTexture1* This, if (This == NULL || filename == NULL) return KTX_INVALID_VALUE; - file = fopen(filename, "rb"); + file = ktxFOpenUTF8(filename, "rb"); if (!file) return KTX_FILE_OPEN_FAILED; @@ -681,6 +684,9 @@ ktxTexture1_CreateFromStdioStream(FILE* stdioStream, * The address of a newly created texture reflecting the contents of the * file is written to the location pointed at by @p newTex. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * The create flag KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT should not be set, * if the ktxTexture1 is ultimately to be uploaded to OpenGL or Vulkan. This * will minimize memory usage by allowing, for example, loading the images diff --git a/lib/texture2.c b/lib/texture2.c index 436f9bcf39..173e5026d9 100644 --- a/lib/texture2.c +++ b/lib/texture2.c @@ -1101,6 +1101,9 @@ ktxTexture2_constructFromStdioStream(ktxTexture2* This, FILE* stdioStream, * @~English * @brief Construct a ktxTexture from a named KTX file. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * See ktxTextureInt_constructFromStream for details. * * @param[in] This pointer to a ktxTextureInt-sized block of memory to @@ -1127,7 +1130,7 @@ ktxTexture2_constructFromNamedFile(ktxTexture2* This, if (This == NULL || filename == NULL) return KTX_INVALID_VALUE; - file = fopen(filename, "rb"); + file = ktxFOpenUTF8(filename, "rb"); if (!file) return KTX_FILE_OPEN_FAILED; @@ -1378,6 +1381,9 @@ ktxTexture2_CreateFromStdioStream(FILE* stdioStream, * The address of a newly created ktxTexture2 reflecting the contents of the * file is written to the location pointed at by @p newTex. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * The create flag KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT should not be set, * if the ktxTexture is ultimately to be uploaded to OpenGL or Vulkan. This * will minimize memory usage by allowing, for example, loading the images diff --git a/lib/writer1.c b/lib/writer1.c index 7408917705..b9c8dd9e09 100644 --- a/lib/writer1.c +++ b/lib/writer1.c @@ -397,6 +397,9 @@ ktxTexture1_WriteToStdioStream(ktxTexture1* This, FILE* dstsstr) * @~English * @brief Write a ktxTexture object to a named file in KTX format. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * @param[in] This pointer to the target ktxTexture object. * @param[in] dstname destination file name. * @@ -422,7 +425,7 @@ ktxTexture1_WriteToNamedFile(ktxTexture1* This, const char* const dstname) if (!This) return KTX_INVALID_VALUE; - dst = fopen(dstname, "wb"); + dst = ktxFOpenUTF8(dstname, "wb"); if (dst) { result = ktxTexture1_WriteToStdioStream(This, dst); fclose(dst); @@ -842,6 +845,9 @@ ktxTexture1_WriteKTX2ToStdioStream(ktxTexture1* This, FILE* dstsstr) * @~English * @brief Write a ktxTexture object to a named file in KTX2 format. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * Callers are strongly urged to include a KTXwriter item in the texture's metadata. * It can be added by code, similar to the following, prior to calling this * function. @@ -878,7 +884,7 @@ ktxTexture1_WriteKTX2ToNamedFile(ktxTexture1* This, const char* const dstname) if (!This) return KTX_INVALID_VALUE; - dst = fopen(dstname, "wb"); + dst = ktxFOpenUTF8(dstname, "wb"); if (dst) { result = ktxTexture1_WriteKTX2ToStdioStream(This, dst); fclose(dst); diff --git a/lib/writer2.c b/lib/writer2.c index 0f26911401..e553e8c613 100644 --- a/lib/writer2.c +++ b/lib/writer2.c @@ -632,6 +632,9 @@ ktxTexture2_WriteToStdioStream(ktxTexture2* This, FILE* dstsstr) * @~English * @brief Write a ktxTexture object to a named file in KTX format. * + * The file name must be encoded in utf-8. On Windows convert unicode names + * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling. + * * Callers are strongly urged to include a KTXwriter item in the texture's metadata. * It can be added by code, similar to the following, prior to calling this * function. @@ -668,7 +671,7 @@ ktxTexture2_WriteToNamedFile(ktxTexture2* This, const char* const dstname) if (!This) return KTX_INVALID_VALUE; - dst = fopen(dstname, "wb"); + dst = ktxFOpenUTF8(dstname, "wb"); if (dst) { result = ktxTexture2_WriteToStdioStream(This, dst); fclose(dst); diff --git a/utils/argparser.cpp b/utils/argparser.cpp index f5dbc164e6..48620971f7 100644 --- a/utils/argparser.cpp +++ b/utils/argparser.cpp @@ -23,29 +23,31 @@ #include #include "argparser.h" +using namespace std; + /* * Construct from a string of arguments. */ -argvector::argvector(const _tstring& sArgs) +argvector::argvector(const string& sArgs) { - const _tstring sep(_T(" \t\n\r\v\f")); + const string sep(" \t\n\r\v\f"); size_t pos; pos = sArgs.find_first_not_of(sep); - assert(pos != _tstring::npos); + assert(pos != string::npos); do { size_t epos = sArgs.find_first_of(sep, pos); - size_t len = epos == _tstring::npos ? epos : epos - pos; + size_t len = epos == string::npos ? epos : epos - pos; push_back(sArgs.substr(pos, len)); pos = sArgs.find_first_not_of(sep, epos); - } while (pos != _tstring::npos); + } while (pos != string::npos); } /* * Construct from an array of C strings */ -argvector::argvector(int argc, const _TCHAR* const* argv) +argvector::argvector(int argc, const char* const* argv) { for (int i = 0; i < argc; i++) { push_back(argv[i]); @@ -56,24 +58,24 @@ argvector::argvector(int argc, const _TCHAR* const* argv) * Functions the same as getopt_long. See `man 3 getopt_long`. */ int -argparser::getopt(_tstring* shortopts, const struct option* longopts, +argparser::getopt(string* shortopts, const struct option* longopts, int* /*longindex*/) { if (optind == argv.size()) return -1; - _tstring arg; + string arg; arg = argv[optind]; - if (arg[0] != _T('-') || (arg[0] == _T('-') && arg.size() == 1)) + if (arg[0] != '-' || (arg[0] == '-' && arg.size() == 1)) return -1; optind++; int retval = '?'; - if (arg.compare(0, 2, _T("--")) == 0) { + if (arg.compare(0, 2, "--") == 0) { if (arg.size() == 2) return -1; // " -- " separates options and files const struct option* opt = longopts; while (opt->name != nullptr) { - if (arg.compare(2, _tstring::npos, opt->name) == 0) { + if (arg.compare(2, string::npos, opt->name) == 0) { retval = opt->val; if (opt->has_arg != option::no_argument) { if (optind >= argv.size() || (optarg = argv[optind++])[0] == '-') { @@ -91,9 +93,9 @@ argparser::getopt(_tstring* shortopts, const struct option* longopts, } opt++; } - } else if (shortopts != nullptr && arg.compare(0, 1, _T("-")) == 0) { + } else if (shortopts != nullptr && arg.compare(0, 1, "-") == 0) { size_t pos = shortopts->find(arg.substr(1, 1)); - if (pos != _tstring::npos) { + if (pos != string::npos) { retval = (*shortopts)[pos]; if (pos < shortopts->length() && ((*shortopts)[++pos] == ':' || (*shortopts)[pos] == ';')) { @@ -118,7 +120,7 @@ std::istream& operator >> (std::istream& stream, const skip& x) stream >> std::noskipws; char c; - const _TCHAR* text = x.text; + const char* text = x.text; while (stream && *text++) stream >> c; diff --git a/utils/argparser.h b/utils/argparser.h index 14dfa9cec2..d977bc3936 100644 --- a/utils/argparser.h +++ b/utils/argparser.h @@ -13,26 +13,12 @@ #include #include #include -#if defined(_WIN32) - #include -#elif !defined(_TCHAR) - #define _TCHAR char - #define _T(x) x -#endif -#if !defined(_tstring) - #if defined(_UNICODE) - #define _tstring std::wstring - #else - #define _tstring std::string - #endif -#endif - -class argvector : public std::vector<_tstring> { +class argvector : public std::vector { public: argvector() { }; - argvector(const _tstring& argstring); - argvector(int argc, const _TCHAR* const* argv); + argvector(const std::string& argstring); + argvector(int argc, const char* const* argv); }; class argparser { @@ -46,17 +32,17 @@ class argparser { option(const char* name, has_arg_t has_arg, int* flag, int val) : name(name), has_arg(has_arg), flag(flag), val(val) {} }; - _tstring optarg; + std::string optarg; unsigned int optind; argvector argv; argparser(argvector& argv, unsigned int startindex = 0) : optind(startindex), argv(argv) { } - argparser(int argc, const _TCHAR* const* argv1) + argparser(int argc, const char* const* argv1) : optind(1), argv(argc, argv1) { } - int getopt(_tstring* shortopts, const struct option* longopts, + int getopt(std::string* shortopts, const struct option* longopts, int* longindex = nullptr); }; @@ -66,8 +52,8 @@ class argparser { // does not check whether the skipped characters are the same as it struct skip { - const _TCHAR* text; - skip(const _TCHAR* text) : text(text) {} + const char* text; + skip(const char* text) : text(text) {} }; std::istream& operator >> (std::istream& stream, const skip& x); diff --git a/utils/ktxapp.h b/utils/ktxapp.h index d069e12d6a..09c9f45678 100644 --- a/utils/ktxapp.h +++ b/utils/ktxapp.h @@ -22,6 +22,7 @@ #include #include "argparser.h" +#include "platform_utils.h" #define QUOTE(x) #x #define STR(x) QUOTE(x) @@ -90,7 +91,7 @@ struct clamped class ktxApp { public: - virtual int main(int argc, _TCHAR* argv[]) = 0; + virtual int main(int argc, char* argv[]) = 0; virtual void usage() { cerr << " -h, --help Print this usage message and exit.\n" @@ -100,11 +101,12 @@ class ktxApp { #endif ; }; + string& getName() { return name; } protected: struct commandOptions { - std::vector<_tstring> infiles; - _tstring outfile; + std::vector infiles; + string outfile; int test; int warn; int launchDebugger; @@ -168,16 +170,16 @@ class ktxApp { * @return A stdio FILE* for the created file. If the file already exists * returns nullptr and sets errno to EEXIST. */ - static FILE* fopen_write_if_not_exists(const _tstring& path) { - FILE* file = ::_tfopen(path.c_str(), "wxb"); + static FILE* fopen_write_if_not_exists(const string& path) { + FILE* file = ::fopenUTF8(path, "wxb"); if (!file && errno == EINVAL) { - file = ::_tfopen(path.c_str(), "r"); + file = ::fopenUTF8(path, "r"); if (file) { fclose(file); file = nullptr; errno = EEXIST; } else { - file = ::_tfopen(path.c_str(), "wb"); + file = ::fopenUTF8(path, "wb"); } } return file; @@ -198,7 +200,7 @@ class ktxApp { enum StdinUse { eDisallowStdin, eAllowStdin }; enum OutfilePos { eNone, eFirst, eLast }; - void processCommandLine(int argc, _TCHAR* argv[], + void processCommandLine(int argc, char* argv[], StdinUse stdinStat = eAllowStdin, OutfilePos outfilePos = eNone) { @@ -207,14 +209,14 @@ class ktxApp { name = argv[0]; // For consistent Id, only use the stem of name; - slash = name.find_last_of(_T('\\')); - if (slash == _tstring::npos) - slash = name.find_last_of(_T('/')); - if (slash != _tstring::npos) + slash = name.find_last_of('\\'); + if (slash == string::npos) + slash = name.find_last_of('/'); + if (slash != string::npos) name.erase(0, slash+1); // Remove directory name. - dot = name.find_last_of(_T('.')); - if (dot != _tstring::npos) - name.erase(dot, _tstring::npos); // Remove extension. + dot = name.find_last_of('.'); + if (dot != string::npos) + name.erase(dot, string::npos); // Remove extension. argparser parser(argc, argv); processOptions(parser); @@ -225,9 +227,9 @@ class ktxApp { options.outfile = parser.argv[i++]; uint32_t infileCount = outfilePos == eLast ? argc - 1 : argc; for (; i < infileCount; i++) { - if (parser.argv[i][0] == _T('@')) { + if (parser.argv[i][0] == '@') { if (!loadFileList(parser.argv[i], - parser.argv[i][1] == _T('@'), + parser.argv[i][1] == '@', options.infiles)) { exit(1); } @@ -236,9 +238,9 @@ class ktxApp { } } if (options.infiles.size() > 1) { - std::vector<_tstring>::const_iterator it; + std::vector::const_iterator it; for (it = options.infiles.begin(); it < options.infiles.end(); it++) { - if (it->compare(_T("-")) == 0) { + if (it->compare("-") == 0) { error("cannot use stdin as one among many inputs."); usage(); exit(1); @@ -251,7 +253,7 @@ class ktxApp { if (options.infiles.size() == 0) { if (stdinStat == eAllowStdin) { - options.infiles.push_back(_T("-")); // Use stdin as 0 files. + options.infiles.push_back("-"); // Use stdin as 0 files. } else { error("need some input files."); usage(); @@ -263,26 +265,21 @@ class ktxApp { } } - bool loadFileList(const _tstring &f, bool relativize, - vector<_tstring>& filenames) + bool loadFileList(const string &f, bool relativize, + vector& filenames) { - _tstring listName(f); + string listName(f); listName.erase(0, relativize ? 2 : 1); FILE *lf = nullptr; -#if defined(_WIN32) - _tfopen_s(&lf, listName.c_str(), "r"); -#else - lf = _tfopen(listName.c_str(), "r"); -#endif - + lf = fopenUTF8(listName, "r"); if (!lf) { error("failed opening filename list: \"%s\": %s\n", listName.c_str(), strerror(errno)); return false; } - _tstring dirname; + string dirname; if (relativize) { size_t dirnameEnd = listName.find_last_of('/'); @@ -311,7 +308,7 @@ class ktxApp { string readFilename(p); while (readFilename.size()) { - if (readFilename[0] == _T(' ')) + if (readFilename[0] == ' ') readFilename.erase(0, 1); else break; @@ -319,7 +316,7 @@ class ktxApp { while (readFilename.size()) { const char c = readFilename.back(); - if ((c == _T(' ')) || (c == _T('\n')) || (c == _T('\r'))) + if ((c == ' ') || (c == '\n') || (c == '\r')) readFilename.erase(readFilename.size() - 1, 1); else break; @@ -422,9 +419,9 @@ class ktxApp { } #endif - _tstring name; - _tstring& version; - _tstring& defaultVersion; + string name; + string& version; + string& defaultVersion; commandOptions& options; @@ -447,7 +444,29 @@ class ktxApp { { nullptr, argparser::option::no_argument, nullptr, 0 } }; - _tstring short_opts = _T("hv"); + string short_opts = "hv"; }; +extern ktxApp& theApp; + +/** @internal + * @~English + * @brief Common main for all derived classes. + * + * Handles rewriting of argv to UTF-8 on Windows. + * Each app needs to initialize @c theApp to + * point to an instance of itself. + */ +int main(int argc, char* argv[]) +{ + InitUTF8CLI(argc, argv); +#if 0 + if (!SetConsoleOutputCP(CP_UTF8)) { + cerr << theApp.getName() << "warning: failed to set UTF-8 code page for console output." + << endl; + } +#endif + return theApp.main(argc, argv); +} + diff --git a/utils/platform_utils.h b/utils/platform_utils.h index 7152af674d..5f3efb7927 100644 --- a/utils/platform_utils.h +++ b/utils/platform_utils.h @@ -9,8 +9,6 @@ #include #include #include -#include -#include #if defined(_WIN32) #ifndef WIN32_LEAN_AND_MEAN @@ -43,15 +41,15 @@ inline std::string DecodeUTF8Path(std::string path) { } #endif -inline void InitUTF8CLI(int& argc, _TCHAR* argv[]) { -#if defined(_WIN32) && !defined(_UNICODE) +inline void InitUTF8CLI(int& argc, char* argv[]) { +#if defined(_WIN32) // Windows does not support UTF-8 argv so we have to manually acquire it - static std::vector> utf8Argv(argc); + static std::vector> utf8Argv(argc); LPWSTR commandLine = GetCommandLineW(); LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc); for (int i = 0; i < argc; ++i) { int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr); - utf8Argv[i] = std::make_unique<_TCHAR[]>(byteSize); + utf8Argv[i] = std::make_unique(byteSize); WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr); argv[i] = utf8Argv[i].get(); } @@ -61,3 +59,22 @@ inline void InitUTF8CLI(int& argc, _TCHAR* argv[]) { (void)argv; #endif } + +inline FILE* fopenUTF8(const std::string& path, const std::string& mode) { +#if defined(_WIN32) + FILE* fp; + // Returned errmo_t value is also set in the global errno. + (void)_wfopen_s(&fp, DecodeUTF8Path(path).c_str(), DecodeUTF8Path(mode).c_str()); + return fp; +#else + return fopen(path.c_str(), mode.c_str()); +#endif +} + +inline int unlinkUTF8(const std::string& path) { +#if defined(_WIN32) + return _wunlink(DecodeUTF8Path(path).c_str()); +#else + return unlink(path.c_str()); +#endif +} diff --git a/utils/scapp.h b/utils/scapp.h index ea2bb757bf..24ca263790 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -558,7 +558,7 @@ class scApp : public ktxApp { } int encode(ktxTexture2* texture, const string& swizzle, - const _tstring& filename); + const string& filename); void usage() { @@ -1075,7 +1075,7 @@ scApp::processOption(argparser& parser, int opt) */ int scApp::encode(ktxTexture2* texture, const string& swizzle, - const _tstring& filename) + const string& filename) { ktx_error_code_e result; diff --git a/utils/stdafx.h b/utils/stdafx.h index 7513ba4949..e6a68716a1 100644 --- a/utils/stdafx.h +++ b/utils/stdafx.h @@ -15,47 +15,14 @@ #include #ifdef _WIN32 #include - #include #if _MSC_VER < 1900 #define snprintf _snprintf #endif #else #include - #define _setmode(x, y) 0 - #define _tmain main - #define _tcsncmp strncmp - #define _tcscmp strcmp - #define _tgetenv getenv - #define _tcscpy strcpy - #define _tcsncpy strncpy - #define _tcsnpcpy strnpcpy - #define _stscanf sscanf - #define _tcslen strlen - #define _tcscat strcat - #define _tcsrchr strrchr - #define _tcschr strchr - #define _tfopen fopen - #define _trename rename - #define _tunlink unlink - #if !defined(_TCHAR) - #define _TCHAR char - #define _T(x) x - #endif -#endif -#if !defined(_tstring) - #if defined(_UNICODE) - #define _tstring std::wstring - #else - #define _tstring std::string - #endif #endif + #include #include #include - -#if defined(_UNICODE) - #define _tstring std::wstring -#else - #define _tstring std::string -#endif