Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 1bc9235

Browse files
committed
Fix NUMERIC modulus to properly truncate division in computation.
Division rounding was causing incorrect results. Test case: test=> SELECT 12345678901234567890 % 123; ?column? ---------- 78 (1 row) Was returning -45.
1 parent a766064 commit 1bc9235

File tree

1 file changed

+22
-19
lines changed

1 file changed

+22
-19
lines changed

src/backend/utils/adt/numeric.c

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* Copyright (c) 1998-2005, PostgreSQL Global Development Group
1515
*
1616
* IDENTIFICATION
17-
* $PostgreSQL: pgsql/src/backend/utils/adt/numeric.c,v 1.83 2005/04/06 23:56:07 neilc Exp $
17+
* $PostgreSQL: pgsql/src/backend/utils/adt/numeric.c,v 1.84 2005/06/04 14:12:50 momjian Exp $
1818
*
1919
*-------------------------------------------------------------------------
2020
*/
@@ -265,7 +265,7 @@ static void sub_var(NumericVar *var1, NumericVar *var2, NumericVar *result);
265265
static void mul_var(NumericVar *var1, NumericVar *var2, NumericVar *result,
266266
int rscale);
267267
static void div_var(NumericVar *var1, NumericVar *var2, NumericVar *result,
268-
int rscale);
268+
int rscale, bool round);
269269
static int select_div_scale(NumericVar *var1, NumericVar *var2);
270270
static void mod_var(NumericVar *var1, NumericVar *var2, NumericVar *result);
271271
static void ceil_var(NumericVar *var, NumericVar *result);
@@ -906,14 +906,14 @@ compute_bucket(Numeric operand, Numeric bound1, Numeric bound2,
906906
sub_var(&operand_var, &bound1_var, &operand_var);
907907
sub_var(&bound2_var, &bound1_var, &bound2_var);
908908
div_var(&operand_var, &bound2_var, result_var,
909-
select_div_scale(&operand_var, &bound2_var));
909+
select_div_scale(&operand_var, &bound2_var), true);
910910
}
911911
else
912912
{
913913
sub_var(&bound1_var, &operand_var, &operand_var);
914914
sub_var(&bound1_var, &bound2_var, &bound1_var);
915915
div_var(&operand_var, &bound1_var, result_var,
916-
select_div_scale(&operand_var, &bound1_var));
916+
select_div_scale(&operand_var, &bound1_var), true);
917917
}
918918

919919
mul_var(result_var, count_var, result_var,
@@ -1266,7 +1266,7 @@ numeric_div(PG_FUNCTION_ARGS)
12661266
/*
12671267
* Do the divide and return the result
12681268
*/
1269-
div_var(&arg1, &arg2, &result, rscale);
1269+
div_var(&arg1, &arg2, &result, rscale, true);
12701270

12711271
res = make_result(&result);
12721272

@@ -2246,7 +2246,7 @@ numeric_variance(PG_FUNCTION_ARGS)
22462246
{
22472247
mul_var(&vN, &vNminus1, &vNminus1, 0); /* N * (N - 1) */
22482248
rscale = select_div_scale(&vsumX2, &vNminus1);
2249-
div_var(&vsumX2, &vNminus1, &vsumX, rscale); /* variance */
2249+
div_var(&vsumX2, &vNminus1, &vsumX, rscale, true); /* variance */
22502250

22512251
res = make_result(&vsumX);
22522252
}
@@ -2322,7 +2322,7 @@ numeric_stddev(PG_FUNCTION_ARGS)
23222322
{
23232323
mul_var(&vN, &vNminus1, &vNminus1, 0); /* N * (N - 1) */
23242324
rscale = select_div_scale(&vsumX2, &vNminus1);
2325-
div_var(&vsumX2, &vNminus1, &vsumX, rscale); /* variance */
2325+
div_var(&vsumX2, &vNminus1, &vsumX, rscale, true); /* variance */
23262326
sqrt_var(&vsumX, &vsumX, rscale); /* stddev */
23272327

23282328
res = make_result(&vsumX);
@@ -3840,7 +3840,7 @@ mul_var(NumericVar *var1, NumericVar *var2, NumericVar *result,
38403840
*/
38413841
static void
38423842
div_var(NumericVar *var1, NumericVar *var2, NumericVar *result,
3843-
int rscale)
3843+
int rscale, bool round)
38443844
{
38453845
int div_ndigits;
38463846
int res_sign;
@@ -4079,8 +4079,11 @@ div_var(NumericVar *var1, NumericVar *var2, NumericVar *result,
40794079
result->sign = res_sign;
40804080

40814081
/* Round to target rscale (and set result->dscale) */
4082-
round_var(result, rscale);
4083-
4082+
if (round)
4083+
round_var(result, rscale);
4084+
else
4085+
trunc_var(result, rscale);
4086+
40844087
/* Strip leading and trailing zeroes */
40854088
strip_var(result);
40864089
}
@@ -4178,7 +4181,7 @@ mod_var(NumericVar *var1, NumericVar *var2, NumericVar *result)
41784181
*/
41794182
rscale = select_div_scale(var1, var2);
41804183

4181-
div_var(var1, var2, &tmp, rscale);
4184+
div_var(var1, var2, &tmp, rscale, false);
41824185

41834186
trunc_var(&tmp, 0);
41844187

@@ -4294,7 +4297,7 @@ sqrt_var(NumericVar *arg, NumericVar *result, int rscale)
42944297

42954298
for (;;)
42964299
{
4297-
div_var(&tmp_arg, result, &tmp_val, local_rscale);
4300+
div_var(&tmp_arg, result, &tmp_val, local_rscale, true);
42984301

42994302
add_var(result, &tmp_val, result);
43004303
mul_var(result, &const_zero_point_five, result, local_rscale);
@@ -4384,7 +4387,7 @@ exp_var(NumericVar *arg, NumericVar *result, int rscale)
43844387

43854388
/* Compensate for input sign, and round to requested rscale */
43864389
if (xneg)
4387-
div_var(&const_one, result, result, rscale);
4390+
div_var(&const_one, result, result, rscale, true);
43884391
else
43894392
round_var(result, rscale);
43904393

@@ -4450,7 +4453,7 @@ exp_var_internal(NumericVar *arg, NumericVar *result, int rscale)
44504453
add_var(&ni, &const_one, &ni);
44514454
mul_var(&xpow, &x, &xpow, local_rscale);
44524455
mul_var(&ifac, &ni, &ifac, 0);
4453-
div_var(&xpow, &ifac, &elem, local_rscale);
4456+
div_var(&xpow, &ifac, &elem, local_rscale, true);
44544457

44554458
if (elem.ndigits == 0)
44564459
break;
@@ -4534,7 +4537,7 @@ ln_var(NumericVar *arg, NumericVar *result, int rscale)
45344537
*/
45354538
sub_var(&x, &const_one, result);
45364539
add_var(&x, &const_one, &elem);
4537-
div_var(result, &elem, result, local_rscale);
4540+
div_var(result, &elem, result, local_rscale, true);
45384541
set_var_from_var(result, &xx);
45394542
mul_var(result, result, &x, local_rscale);
45404543

@@ -4544,7 +4547,7 @@ ln_var(NumericVar *arg, NumericVar *result, int rscale)
45444547
{
45454548
add_var(&ni, &const_two, &ni);
45464549
mul_var(&xx, &x, &xx, local_rscale);
4547-
div_var(&xx, &ni, &elem, local_rscale);
4550+
div_var(&xx, &ni, &elem, local_rscale, true);
45484551

45494552
if (elem.ndigits == 0)
45504553
break;
@@ -4614,7 +4617,7 @@ log_var(NumericVar *base, NumericVar *num, NumericVar *result)
46144617
/* Select scale for division result */
46154618
rscale = select_div_scale(&ln_num, &ln_base);
46164619

4617-
div_var(&ln_num, &ln_base, result, rscale);
4620+
div_var(&ln_num, &ln_base, result, rscale, true);
46184621

46194622
free_var(&ln_num);
46204623
free_var(&ln_base);
@@ -4752,7 +4755,7 @@ power_var_int(NumericVar *base, int exp, NumericVar *result, int rscale)
47524755
round_var(result, rscale);
47534756
return;
47544757
case -1:
4755-
div_var(&const_one, base, result, rscale);
4758+
div_var(&const_one, base, result, rscale, true);
47564759
return;
47574760
case 2:
47584761
mul_var(base, base, result, rscale);
@@ -4790,7 +4793,7 @@ power_var_int(NumericVar *base, int exp, NumericVar *result, int rscale)
47904793

47914794
/* Compensate for input sign, and round to requested rscale */
47924795
if (neg)
4793-
div_var(&const_one, result, result, rscale);
4796+
div_var(&const_one, result, result, rscale, true);
47944797
else
47954798
round_var(result, rscale);
47964799
}

0 commit comments

Comments
 (0)