ICU: fix the use of locale names in Windows for ICU functions 2

The previous attempt did not work because the function _create_locale returns
different locales for the same locale names. Therefore, use the Language Code
Identifiers (LCIDS) to get the canonical locale names both for libc and icu.

Important: now you cannot use language and country strings in locale names e.g.
"Russian_Russia[.encoding]" or "English_United States[.encoding]". Instead, use
"ru-RU[.encoding]" or "en-US[.encoding]" correspondingly.

List of canonical names of Windows locales and their LCIDS:
https://www.microsoft.com/resources/msdn/goglobal/default.mspx

Author: Marina Polyakova

src/backend/commands/dbcommands.c

@@ -149,8 +149,13 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 	char	   *dbcanonname = NULL;
 	char		dbcollprovider;
 	char	   *dbcollate_full_name;
+	char	   *icu_wincollate = NULL;
 	char	   *langtag = NULL;
 	const char *collate;
+#ifdef WIN32
+	char	   *dbcollate_original;
+	char	   *dbctype_original;
+#endif
 
 	/* Extract options from the statement node tree */
 	foreach(option, stmt->options)
@@ -393,6 +398,11 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 
 	/* Check that the chosen locales are valid, and get canonical spellings */
 
+#ifdef WIN32
+	dbcollate_original = dbcollate;
+	dbctype_original = dbctype;
+#endif
+
 	if (!check_locale(LC_CTYPE, dbctype, &canonname, '\0'))
 		ereport(ERROR,
 				(errcode(ERRCODE_WRONG_OBJECT_TYPE),
@@ -429,6 +439,15 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 	/* check lc_collate and lc_ctype for icu if we need it */
 	if (dbcollprovider == COLLPROVIDER_ICU)
 	{
+#ifdef WIN32
+		/* set the libc canonnames for dbcollate and dbctype */
+
+		check_winlocale(dbcollate_original, &canonname, NULL);
+		dbcollate = canonname;
+
+		check_winlocale(dbctype_original, &canonname, NULL);
+		dbctype = canonname;
+#endif
 		if (!check_locale(LC_COLLATE, dbcollate, NULL, dbcollprovider))
 			ereport(ERROR,
 					(errcode(ERRCODE_WRONG_OBJECT_TYPE),
@@ -449,17 +468,19 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 	if (dbcollprovider == COLLPROVIDER_ICU)
 	{
 #ifdef WIN32
-		collate = (const char *) IsoLocaleName(dbcollate, LC_COLLATE);
-		if (collate == NULL)
-#endif							/* WIN32 */
+		if (!locale_is_c(dbcollate))
 		{
-			collate = (const char *) dbcollate;
+			check_winlocale(dbcollate, NULL, &icu_wincollate);
+			collate = (const char *) icu_wincollate;
 		}
+#else							/* not WIN32 */
+		collate = (const char *) dbcollate;
+#endif							/* not WIN32 */
 		langtag = get_icu_language_tag(collate);
 		collate = get_icu_collate(collate, langtag);
 	}
 	else
-#endif							/* USE_ICU */
+#endif
 	{
 		/* COLLPROVIDER_LIBC */
 		collate = (const char *) dbcollate;
@@ -797,6 +818,8 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 		pfree(dbcanonname);
 	if (langtag)
 		pfree(langtag);
+	if (icu_wincollate)
+		pfree(icu_wincollate);
 
 	PG_END_ENSURE_ERROR_CLEANUP(createdb_failure_callback,
 								PointerGetDatum(&fparms));

src/backend/utils/adt/pg_locale.c

@@ -131,6 +131,10 @@ typedef struct
 static HTAB *collation_cache = NULL;
 
 
+#if defined(WIN32) && defined(LC_MESSAGES)
+static char *IsoLocaleName(const char *);	/* MSVC specific */
+#endif
+
 #ifdef USE_ICU
 static char *check_icu_locale(const char *locale);
 #endif
@@ -251,7 +255,7 @@ pg_perm_setlocale(int category, const char *locale, char collprovider)
 			envvar = "LC_MESSAGES";
 			envbuf = lc_messages_envbuf;
 #ifdef WIN32
-			result = IsoLocaleName(locale, LC_CTYPE);
+			result = IsoLocaleName(locale);
 			if (result == NULL)
 				result = locale;
 #endif							/* WIN32 */
@@ -972,6 +976,114 @@ cache_locale_time(void)
 }
 
 
+#if defined(WIN32) && defined(LC_MESSAGES)
+/*
+ * Convert a Windows setlocale() argument to a Unix-style one.
+ *
+ * Regardless of platform, we install message catalogs under a Unix-style
+ * LL[_CC][.ENCODING][@VARIANT] naming convention.  Only LC_MESSAGES settings
+ * following that style will elicit localized interface strings.
+ *
+ * Before Visual Studio 2012 (msvcr110.dll), Windows setlocale() accepted "C"
+ * (but not "c") and strings of the form <Language>[_<Country>][.<CodePage>],
+ * case-insensitive.  setlocale() returns the fully-qualified form; for
+ * example, setlocale("thaI") returns "Thai_Thailand.874".  Internally,
+ * setlocale() and _create_locale() select a "locale identifier"[1] and store
+ * it in an undocumented _locale_t field.  From that LCID, we can retrieve the
+ * ISO 639 language and the ISO 3166 country.  Character encoding does not
+ * matter, because the server and client encodings govern that.
+ *
+ * Windows Vista introduced the "locale name" concept[2], closely following
+ * RFC 4646.  Locale identifiers are now deprecated.  Starting with Visual
+ * Studio 2012, setlocale() accepts locale names in addition to the strings it
+ * accepted historically.  It does not standardize them; setlocale("Th-tH")
+ * returns "Th-tH".  setlocale(category, "") still returns a traditional
+ * string.  Furthermore, msvcr110.dll changed the undocumented _locale_t
+ * content to carry locale names instead of locale identifiers.
+ *
+ * MinGW headers declare _create_locale(), but msvcrt.dll lacks that symbol.
+ * IsoLocaleName() always fails in a MinGW-built postgres.exe, so only
+ * Unix-style values of the lc_messages GUC can elicit localized messages.  In
+ * particular, every lc_messages setting that initdb can select automatically
+ * will yield only C-locale messages.  XXX This could be fixed by running the
+ * fully-qualified locale name through a lookup table.
+ *
+ * This function returns a pointer to a static buffer bearing the converted
+ * name or NULL if conversion fails.
+ *
+ * [1] http://msdn.microsoft.com/en-us/library/windows/desktop/dd373763.aspx
+ * [2] http://msdn.microsoft.com/en-us/library/windows/desktop/dd373814.aspx
+ */
+static char *
+IsoLocaleName(const char *winlocname)
+{
+#if (_MSC_VER >= 1400)			/* VC8.0 or later */
+	static char iso_lc_messages[32];
+	_locale_t	loct = NULL;
+
+	if (pg_strcasecmp("c", winlocname) == 0 ||
+		pg_strcasecmp("posix", winlocname) == 0)
+	{
+		strcpy(iso_lc_messages, "C");
+		return iso_lc_messages;
+	}
+
+	loct = _create_locale(LC_CTYPE, winlocname);
+	if (loct != NULL)
+	{
+#if (_MSC_VER >= 1700)			/* Visual Studio 2012 or later */
+		size_t		rc;
+		char	   *hyphen;
+
+		/* Locale names use only ASCII, any conversion locale suffices. */
+		rc = wchar2char(iso_lc_messages, loct->locinfo->locale_name[LC_CTYPE],
+						sizeof(iso_lc_messages), NULL);
+		_free_locale(loct);
+		if (rc == -1 || rc == sizeof(iso_lc_messages))
+			return NULL;
+
+		/*
+		 * Since the message catalogs sit on a case-insensitive filesystem, we
+		 * need not standardize letter case here.  So long as we do not ship
+		 * message catalogs for which it would matter, we also need not
+		 * translate the script/variant portion, e.g. uz-Cyrl-UZ to
+		 * uz_UZ@cyrillic.  Simply replace the hyphen with an underscore.
+		 *
+		 * Note that the locale name can be less-specific than the value we
+		 * would derive under earlier Visual Studio releases.  For example,
+		 * French_France.1252 yields just "fr".  This does not affect any of
+		 * the country-specific message catalogs available as of this writing
+		 * (pt_BR, zh_CN, zh_TW).
+		 */
+		hyphen = strchr(iso_lc_messages, '-');
+		if (hyphen)
+			*hyphen = '_';
+#else
+		char		isolang[32],
+					isocrty[32];
+		LCID		lcid;
+
+		lcid = loct->locinfo->lc_handle[LC_CTYPE];
+		if (lcid == 0)
+			lcid = MAKELCID(MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), SORT_DEFAULT);
+		_free_locale(loct);
+
+		if (!GetLocaleInfoA(lcid, LOCALE_SISO639LANGNAME, isolang, sizeof(isolang)))
+			return NULL;
+		if (!GetLocaleInfoA(lcid, LOCALE_SISO3166CTRYNAME, isocrty, sizeof(isocrty)))
+			return NULL;
+		snprintf(iso_lc_messages, sizeof(iso_lc_messages) - 1, "%s_%s", isolang, isocrty);
+#endif
+		return iso_lc_messages;
+	}
+	return NULL;
+#else
+	return NULL;				/* Not supported on this version of msvc/mingw */
+#endif							/* _MSC_VER >= 1400 */
+}
+#endif							/* WIN32 && LC_MESSAGES */
+
+
 /*
  * Detect aging strxfrm() implementations that, in a subset of locales, write
  * past the specified buffer length.  Affected users must update OS packages
@@ -1725,32 +1837,36 @@ char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen,
  *
  * Otherwise return a malloc'd copy of locale if it is not NULL.
  *
- * In both cases, return the name of the locale in the Unix style if possible.
+ * In Windows, check that the name is not long (for example,
+ * "English_United States[.encoding]").
  */
 static char *
 check_icu_locale(const char *locale)
 {
 	char	   *canonname = NULL;
+	char	   *winlocale = NULL;
 	char	   *result;
 
+#ifdef WIN32
+	if (!locale_is_c(locale))
+	{
+		check_winlocale(locale, NULL, &winlocale);
+		locale = (const char *) winlocale;
+	}
+#else							/* not WIN32 */
 	if (locale && strlen(locale) == 0)
 	{
 		check_locale(LC_COLLATE, locale, &canonname, COLLPROVIDER_LIBC);
 		locale = (const char *) canonname;
 	}
-
-#ifdef WIN32
-	{
-		char	   *iso_locale = IsoLocaleName(locale, LC_COLLATE);
-		if (iso_locale)
-			locale = iso_locale;
-	}
-#endif
+#endif							/* not WIN32 */
 
 	result = locale ? pstrdup(locale) : NULL;
 
 	if (canonname)
 		pfree(canonname);
+	if (winlocale)
+		pfree(winlocale);
 
 	return result;
 }

src/backend/utils/init/postinit.c

@@ -295,6 +295,7 @@ CheckMyDatabase(const char *name, bool am_superuser)
 	char	   *datcollate;
 	char		collprovider;
 	char	   *collversion;
+	char	   *wincollate = NULL;
 	char	   *langtag = NULL;
 	const char *collcollate;
 	char	   *actual_versionstr;
@@ -437,18 +438,19 @@ CheckMyDatabase(const char *name, bool am_superuser)
 #ifdef USE_ICU
 	if (collprovider == COLLPROVIDER_ICU)
 	{
+		collcollate = (const char *) collate;
 #ifdef WIN32
-		collcollate = (const char *) IsoLocaleName(collate, LC_COLLATE);
-		if (collcollate == NULL)
-#endif							/* WIN32 */
+		if (!locale_is_c(collcollate))
 		{
-			collcollate = (const char *) collate;
+			check_winlocale(collcollate, NULL, &wincollate);
+			collcollate = (const char *) wincollate;
 		}
+#endif							/* WIN32 */
 		langtag = get_icu_language_tag(collcollate);
 		collcollate = get_icu_collate(collcollate, langtag);
 	}
 	else
-#endif							/* USE_ICU */
+#endif
 	{
 		/* COLLPROVIDER_LIBC */
 		collcollate = (const char *) collate;
@@ -493,6 +495,8 @@ CheckMyDatabase(const char *name, bool am_superuser)
 		pfree(langtag);
 	if (actual_versionstr)
 		pfree(actual_versionstr);
+	if (wincollate)
+		pfree(wincollate);
 
 	check_strxfrm_bug();