bpo-35697, decimal: Fix locale formatting

Include/internal/pycore_fileutils.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#ifndef Py_BUILD_CORE
-#  error "Py_BUILD_CORE must be defined to include this header"
+#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN defined"
 #endif
 
 #include <locale.h>   /* struct lconv */

Misc/NEWS.d/next/Library/2019-01-09-12-35-57.bpo-35697._TAUNc.rst

@@ -0,0 +1,7 @@
+The :mod:`decimal` module now supports formatting number to the "n" type
+when the ``LC_NUMERIC`` locale uses a different encoding than the
+``LC_CTYPE`` locale. It now sets temporarily the ``LC_CTYPE`` locale to the
+``LC_NUMERIC`` locale to decode ``decimal_point`` and ``thousands_sep`` byte
+strings if they are non-ASCII or longer than 1 byte, and the ``LC_NUMERIC``
+locale is different than the ``LC_CTYPE`` locale.  This temporary change
+affects other threads.

Modules/_decimal/_decimal.c

@@ -32,7 +32,9 @@
 #include "structmember.h"
 #include "complexobject.h"
 #include "mpdecimal.h"
+#include "pycore_fileutils.h"
 
+#include <locale.h>
 #include <stdlib.h>
 
 #include "docstrings.h"
@@ -3013,32 +3015,6 @@ dec_replace_fillchar(char *dest)
      }
 }
 
-/* Convert decimal_point or thousands_sep, which may be multibyte or in
-   the range [128, 255], to a UTF8 string. */
-static PyObject *
-dotsep_as_utf8(const char *s)
-{
-    PyObject *utf8;
-    PyObject *tmp;
-    wchar_t buf[2];
-    size_t n;
-
-    n = mbstowcs(buf, s, 2);
-    if (n != 1) { /* Issue #7442 */
-        PyErr_SetString(PyExc_ValueError,
-            "invalid decimal point or unsupported "
-            "combination of LC_CTYPE and LC_NUMERIC");
-        return NULL;
-    }
-    tmp = PyUnicode_FromWideChar(buf, n);
-    if (tmp == NULL) {
-        return NULL;
-    }
-    utf8 = PyUnicode_AsUTF8String(tmp);
-    Py_DECREF(tmp);
-    return utf8;
-}
-
 /* Formatted representation of a PyDecObject. */
 static PyObject *
 dec_format(PyObject *dec, PyObject *args)
@@ -3048,6 +3024,7 @@ dec_format(PyObject *dec, PyObject *args)
     PyObject *dot = NULL;
     PyObject *sep = NULL;
     PyObject *grouping = NULL;
+    char *grouping_buffer = NULL;
     PyObject *fmtarg;
     PyObject *context;
     mpd_spec_t spec;
@@ -3133,24 +3110,33 @@ dec_format(PyObject *dec, PyObject *args)
             goto finish;
         }
     }
-    else {
-        size_t n = strlen(spec.dot);
-        if (n > 1 || (n == 1 && !isascii((uchar)spec.dot[0]))) {
-            /* fix locale dependent non-ascii characters */
-            dot = dotsep_as_utf8(spec.dot);
-            if (dot == NULL) {
-                goto finish;
-            }
-            spec.dot = PyBytes_AS_STRING(dot);
+    else if (spec.locale) {
+        struct lconv *lc = localeconv();
+        if (_Py_GetLocaleconvNumeric(lc, &dot, &sep) < 0) {
+            goto finish;
         }
-        n = strlen(spec.sep);
-        if (n > 1 || (n == 1 && !isascii((uchar)spec.sep[0]))) {
-            /* fix locale dependent non-ascii characters */
-            sep = dotsep_as_utf8(spec.sep);
-            if (sep == NULL) {
-                goto finish;
-            }
-            spec.sep = PyBytes_AS_STRING(sep);
+
+        grouping_buffer = _PyMem_Strdup(lc->grouping);
+        if (grouping_buffer == NULL) {
+            PyErr_NoMemory();
+            goto finish;
+        }
+        spec.grouping = grouping_buffer;
+
+        spec.dot = PyUnicode_AsUTF8(dot);
+        if (spec.dot == NULL) {
+            goto finish;
+        }
+
+        spec.sep = PyUnicode_AsUTF8(sep);
+        if (spec.sep == NULL) {
+            goto finish;
+        }
+
+        if (mpd_validate_lconv(&spec) < 0) {
+            PyErr_SetString(PyExc_ValueError,
+                "invalid localeconv()");
+            goto finish;
         }
     }
 
@@ -3176,6 +3162,7 @@ dec_format(PyObject *dec, PyObject *args)
 
 finish:
     Py_XDECREF(grouping);
+    PyMem_Free(grouping_buffer);
     Py_XDECREF(sep);
     Py_XDECREF(dot);
     if (replace_fillchar) PyMem_Free(fmt);

Modules/_decimal/libmpdec/io.c

@@ -784,6 +784,7 @@ mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps)
     spec->dot = "";
     spec->sep = "";
     spec->grouping = "";
+    spec->locale = 0;
 
 
     /* presume that the first character is a UTF-8 fill character */
@@ -871,6 +872,7 @@ mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps)
         if (*spec->sep) {
             return 0;
         }
+        spec->locale = 1;
         spec->type = *cp++;
         spec->type = (spec->type == 'N') ? 'G' : 'g';
         lc = localeconv();

Modules/_decimal/libmpdec/mpdecimal.h

@@ -397,6 +397,7 @@ typedef struct mpd_spec_t {
     const char *dot;       /* decimal point */
     const char *sep;       /* thousands separator */
     const char *grouping;  /* grouping of digits */
+    int locale;            /* use localeconv() */
 } mpd_spec_t;
 
 /* output to a string */

PCbuild/_decimal.vcxproj

@@ -61,7 +61,7 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;MASM;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;MASM;Py_BUILD_CORE_BUILTIN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)' == 'Win32'">CONFIG_32;PPRO;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'">CONFIG_64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

setup.py

@@ -2002,7 +2002,7 @@ def detect_ctypes(self, inc_dirs, lib_dirs):
             ext.libraries.append('dl')
 
     def _decimal_ext(self):
-        extra_compile_args = []
+        extra_compile_args = ['-DPy_BUILD_CORE_BUILTIN']
         undef_macros = []
         if '--with-system-libmpdec' in sysconfig.get_config_var("CONFIG_ARGS"):
             include_dirs = []