diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index 4cee297..0bea9be 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -210,7 +210,10 @@ runs: if ("$so" -notmatch 'MBCS->UTF-8 ok on 2 checks') { throw "selftest did not exercise the MBCS->UTF-8 conversion" } - if ("$so" -notmatch 'catalog decoding ok on 3 checks') { + # The suffix proves the best-fit case can fail: cp1252 offers a lookalike for U+0100. + # Where a codepage offers none, that case cannot catch anything. The last two checks + # run only there, so the count and the suffix move together. + if ("$so" -notmatch 'catalog decoding ok on 6 checks \(best-fit acp\)') { throw 'selftest did not report catalog decoding: a legacy catalog may be silently mangled' } # Swapping decode and unescape is invisible on this cp1252 runner, so this drives CP932. diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 5ed0ac7..599eec1 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -724,7 +724,12 @@ jobs: # --selftest throws one exception on purpose. $p = Start-Process (Join-Path $bin 'WinHTTrack.exe') -ArgumentList '--selftest' -Wait -PassThru ` -RedirectStandardOutput "$PWD\crash-st.txt" -RedirectStandardError "$PWD\crash-st.err" - if ($p.ExitCode -ne 0) { throw "--selftest exited $($p.ExitCode) from the staged tree" } + if ($p.ExitCode -ne 0) { + # The selftest names the failing check on stderr, and the exit code alone does not. + Write-Host "--- selftest stdout ---`n$(Get-Content "$PWD\crash-st.txt" -Raw -ErrorAction SilentlyContinue)" + Write-Host "--- selftest stderr ---`n$(Get-Content "$PWD\crash-st.err" -Raw -ErrorAction SilentlyContinue)" + throw "--selftest exited $($p.ExitCode) from the staged tree" + } if (-not (Test-Path $report)) { throw 'no crash report written: the first-chance hook never ran' } $trace = Get-Content $report -Raw Write-Host "--- crash report ---`n$trace" diff --git a/WinHTTrack/Shell.cpp b/WinHTTrack/Shell.cpp index 6e7f671..043a717 100755 --- a/WinHTTrack/Shell.cpp +++ b/WinHTTrack/Shell.cpp @@ -1715,6 +1715,23 @@ void CopyTextUTF8ToCP(LPSTR dest, int destSize, LPCSTR lpString) { freet(cp); } +// Contract in Shell.h. +BOOL CopyTextWideToCPExact(LPSTR dest, int destSize, LPCWSTR wide) { + // A best-fit substitute leaves lpUsedDefaultChar clear, so without WC_NO_BEST_FIT_CHARS an + // approximation comes back and reads as held. CP_UTF8 rejects both that flag and the pointer. + const BOOL utf8 = GetACP() == CP_UTF8; + BOOL lost = FALSE; + int n; + + if (destSize <= 0) + return FALSE; + n = WideCharToMultiByte(CP_ACP, utf8 ? 0 : WC_NO_BEST_FIT_CHARS, wide, -1, dest, destSize, + NULL, utf8 ? NULL : &lost); + if (n <= 0) + dest[0] = '\0'; + return n > 0 && !lost; +} + bool ShellOpen(LPCSTR file, int nShowCmd) { return (INT_PTR) ShellExecute(NULL, "open", file, NULL, NULL, nShowCmd) > 32; } diff --git a/WinHTTrack/Shell.h b/WinHTTrack/Shell.h index 54be02c..77c28a5 100755 --- a/WinHTTrack/Shell.h +++ b/WinHTTrack/Shell.h @@ -159,6 +159,11 @@ char *strdupt_utf8(const char *const s); // TTN_NEEDTEXTA tooltip text. Falls back to the raw bytes if conversion fails. void CopyTextUTF8ToCP(LPSTR dest, int destSize, LPCSTR lpString); +// The ANSI-codepage form of a wide string, always NUL-terminated, with best-fit substitutes +// blocked as the engine's converter blocks them. --selftest needs its own oracle for that. +// FALSE: the codepage substituted; dest is empty instead if it could not convert at all. +BOOL CopyTextWideToCPExact(LPSTR dest, int destSize, LPCWSTR wide); + // Drop a trailing '/' or '\\', reporting whether there was one. Empty-string safe. inline bool StripTrailingSlash(char* s) { const size_t len = (s != NULL) ? strlen(s) : 0; diff --git a/WinHTTrack/WinHTTrack.cpp b/WinHTTrack/WinHTTrack.cpp index 43f62d2..acca14f 100755 --- a/WinHTTrack/WinHTTrack.cpp +++ b/WinHTTrack/WinHTTrack.cpp @@ -345,13 +345,9 @@ BOOL CWinHTTrackApp::InitInstance() // Only reachable by typing into a dialog, so test the MBCS->UTF-8 conversion here instead. { static const WCHAR wide[] = { 'c', 'a', 'f', 0x00E9, 0 }; /* cafe-acute */ - BOOL lost = FALSE; - // WideCharToMultiByte rejects a non-NULL lpUsedDefaultChar when the code page is CP_UTF8. - BOOL *const plost = (GetACP() == CP_UTF8) ? NULL : &lost; char ansi[16]; - const int n = WideCharToMultiByte(CP_ACP, 0, wide, -1, ansi, sizeof(ansi), - NULL, plost); - if (n > 0 && !lost) { /* skip where the ANSI codepage cannot hold it at all */ + /* skip where the codepage cannot hold it exactly */ + if (CopyTextWideToCPExact(ansi, sizeof(ansi), wide)) { int nchecks = 0; char *got = strdupt_utf8(ansi); /* freet() nulls it, so not const */ if (got == NULL || strcmp(got, "caf\xc3\xa9") != 0) { diff --git a/WinHTTrack/newlang.cpp b/WinHTTrack/newlang.cpp index 64767c6..43a96b1 100755 --- a/WinHTTrack/newlang.cpp +++ b/WinHTTrack/newlang.cpp @@ -363,41 +363,81 @@ void LANG_SELFTEST_ESCAPE_ORDER(void) { /* --selftest: the catalogs are the only non-ASCII data the GUI loads, and the English-only smoke walk cannot see a decoding regression. */ void LANG_SELFTEST_DECODE(void) { - static const WCHAR wide[] = { 'c', 'a', 'f', 0x00E9, 0 }; /* cafe-acute */ - BOOL lost = FALSE; - // WideCharToMultiByte rejects a non-NULL lpUsedDefaultChar when the code page is CP_UTF8. - BOOL *const plost = (GetACP() == CP_UTF8) ? NULL : &lost; - char want[16]; - const int n = WideCharToMultiByte(CP_ACP, 0, wide, -1, want, sizeof(want), NULL, plost); - - if (n > 0 && !lost) { /* skip where the ANSI codepage cannot hold it at all */ - static const struct { const char* in; const char* want; const char* what; } cases[] = { - /* A converted catalog reaches the ANSI codepage. */ - { "caf\xc3\xa9", NULL, "utf-8" }, - /* A catalog still in its legacy charset must survive byte for byte: drop the - UTF-8 gate and the engine's converter substitutes U+FFFD, giving "caf?". */ - { "caf\xe9", "caf\xe9", "legacy" }, - { "Options", "Options", "ascii" }, - { NULL, NULL, NULL } - }; - int nchecks = 0; - for(int k=0 ; cases[k].in != NULL ; k++) { - const char* const expect = (cases[k].want != NULL) ? cases[k].want : want; - const int len = (int) strlen(cases[k].in); - char got[64]; - if (IsValidUTF8(cases[k].in, len)) - CopyTextUTF8ToCP(got, sizeof(got), cases[k].in); - else - lstrcpynA(got, cases[k].in, sizeof(got)); - if (strcmp(got, expect) != 0) { - fprintf(stderr, "FATAL: %s catalog text gave '%s', expected '%s'\n", - cases[k].what, got, expect); + static const WCHAR cafe[] = { 'c', 'a', 'f', 0x00E9, 0 }; /* cafe-acute */ + static const WCHAR macron[] = { 0x0100, 0 }; /* A-macron, 'A' by best-fit on cp1252 */ + static const struct { const char* in; const WCHAR* wide; const char* want; const char* what; } cases[] = { + /* A converted catalog reaches the ANSI codepage. */ + { "caf\xc3\xa9", cafe, NULL, "utf-8" }, + /* The engine blocks best-fit, so a character the codepage lacks must arrive as its + default character, not as a lookalike. */ + { "\xc4\x80", macron, NULL, "best-fit" }, + /* A catalog still in its legacy charset must survive byte for byte: drop the + UTF-8 gate and the engine's converter substitutes U+FFFD, giving "caf?". */ + { "caf\xe9", NULL, "caf\xe9", "legacy" }, + { "Options", NULL, "Options", "ascii" }, + { NULL, NULL, NULL, NULL } + }; + int nchecks = 0; + + for(int k=0 ; cases[k].in != NULL ; k++) { + const int len = (int) strlen(cases[k].in); + char expect[64]; + char got[64]; + /* Expecting what this codepage holds, so none has to be skipped. */ + if (cases[k].wide != NULL) + CopyTextWideToCPExact(expect, sizeof(expect), cases[k].wide); + else + lstrcpynA(expect, cases[k].want, sizeof(expect)); + if (IsValidUTF8(cases[k].in, len)) + CopyTextUTF8ToCP(got, sizeof(got), cases[k].in); + else + lstrcpynA(got, cases[k].in, sizeof(got)); + if (strcmp(got, expect) != 0) { + fprintf(stderr, "FATAL: %s catalog text gave '%s', expected '%s'\n", + cases[k].what, got, expect); + fflush(stderr); + ExitProcess(3); + } else + nchecks++; + } + + { + /* The case above only discriminates where the codepage has a lookalike to offer. Where it + does, pin the substitute to the codepage's own default character. A second + WideCharToMultiByte call would only prove the two agree. */ + CPINFO cpinfo; + char def[MAX_DEFAULTCHAR + 1]; + char expect[64], lookalike[64]; + const int n = WideCharToMultiByte(CP_ACP, 0, macron, -1, lookalike, sizeof(lookalike), + NULL, NULL); + const BOOL held = CopyTextWideToCPExact(expect, sizeof(expect), macron); + const BOOL bestfit = n > 0 && strcmp(lookalike, expect) != 0; + + if (bestfit) { + if (!GetCPInfo(CP_ACP, &cpinfo)) { + fprintf(stderr, "FATAL: no CPINFO for the ANSI codepage\n"); + fflush(stderr); + ExitProcess(3); + } + memcpy(def, cpinfo.DefaultChar, MAX_DEFAULTCHAR); + def[MAX_DEFAULTCHAR] = '\0'; + /* Counted apart: a helper that forgot its lpUsedDefaultChar check still substitutes, + so sharing one increment would leave that half unpinned. */ + if (held) { + fprintf(stderr, "FATAL: U+0100 reported as held by a codepage that substitutes it\n"); + fflush(stderr); + ExitProcess(3); + } else + nchecks++; + if (strcmp(expect, def) != 0) { + fprintf(stderr, "FATAL: U+0100 became '%s', expected the default character '%s'\n", + expect, def); fflush(stderr); ExitProcess(3); } else nchecks++; } - printf("catalog decoding ok on %d checks\n", nchecks); + printf("catalog decoding ok on %d checks%s\n", nchecks, bestfit ? " (best-fit acp)" : ""); } }