1515 *
1616 *
1717 * IDENTIFICATION
18- * $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.102 2001/11/05 17:46:29 momjian Exp $
18+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.103 2002/01/03 04:02:34 tgl Exp $
1919 *
2020 *-------------------------------------------------------------------------
2121 */
124124#define DEFAULT_NOT_UNK_SEL (1.0 - DEFAULT_UNK_SEL)
125125#define DEFAULT_BOOL_SEL 0.5
126126
127+ /*
128+ * Clamp a computed probability estimate (which may suffer from roundoff or
129+ * estimation errors) to valid range. Argument must be a float variable.
130+ */
131+ #define CLAMP_PROBABILITY (p ) \
132+ do { \
133+ if (p < 0.0) \
134+ p = 0.0; \
135+ else if (p > 1.0) \
136+ p = 1.0; \
137+ } while (0)
138+
139+
127140static bool convert_to_scalar (Datum value ,Oid valuetypid ,double * scaledvalue ,
128141Datum lobound ,Datum hibound ,Oid boundstypid ,
129142double * scaledlobound ,double * scaledhibound );
@@ -285,6 +298,7 @@ eqsel(PG_FUNCTION_ARGS)
285298for (i = 0 ;i < nnumbers ;i ++ )
286299sumcommon += numbers [i ];
287300selec = 1.0 - sumcommon - stats -> stanullfrac ;
301+ CLAMP_PROBABILITY (selec );
288302
289303/*
290304 * and in fact it's probably a good deal less. We
@@ -356,10 +370,7 @@ eqsel(PG_FUNCTION_ARGS)
356370}
357371
358372/* result should be in range, but make sure... */
359- if (selec < 0.0 )
360- selec = 0.0 ;
361- else if (selec > 1.0 )
362- selec = 1.0 ;
373+ CLAMP_PROBABILITY (selec );
363374
364375PG_RETURN_FLOAT8 ((float8 )selec );
365376}
@@ -669,10 +680,7 @@ scalarineqsel(Query *root, Oid operator, bool isgt,
669680ReleaseSysCache (statsTuple );
670681
671682/* result should be in range, but make sure... */
672- if (selec < 0.0 )
673- selec = 0.0 ;
674- else if (selec > 1.0 )
675- selec = 1.0 ;
683+ CLAMP_PROBABILITY (selec );
676684
677685return selec ;
678686}
@@ -867,10 +875,7 @@ patternsel(PG_FUNCTION_ARGS, Pattern_Type ptype)
867875restsel = pattern_selectivity (rest ,ptype );
868876selec = prefixsel * restsel ;
869877/* result should be in range, but make sure... */
870- if (selec < 0.0 )
871- selec = 0.0 ;
872- else if (selec > 1.0 )
873- selec = 1.0 ;
878+ CLAMP_PROBABILITY (selec );
874879result = selec ;
875880}
876881
@@ -1179,10 +1184,7 @@ booltestsel(Query *root, BooleanTest *clause, int varRelid)
11791184}
11801185
11811186/* result should be in range, but make sure... */
1182- if (selec < 0.0 )
1183- selec = 0.0 ;
1184- else if (selec > 1.0 )
1185- selec = 1.0 ;
1187+ CLAMP_PROBABILITY (selec );
11861188
11871189return (Selectivity )selec ;
11881190}
@@ -1285,10 +1287,7 @@ nulltestsel(Query *root, NullTest *clause, int varRelid)
12851287}
12861288
12871289/* result should be in range, but make sure... */
1288- if (selec < 0.0 )
1289- selec = 0.0 ;
1290- else if (selec > 1.0 )
1291- selec = 1.0 ;
1290+ CLAMP_PROBABILITY (selec );
12921291
12931292return (Selectivity )selec ;
12941293}
@@ -1448,6 +1447,7 @@ eqjoinsel(PG_FUNCTION_ARGS)
14481447}
14491448}
14501449}
1450+ CLAMP_PROBABILITY (matchprodfreq );
14511451/* Sum up frequencies of matched and unmatched MCVs */
14521452matchfreq1 = unmatchfreq1 = 0.0 ;
14531453for (i = 0 ;i < nvalues1 ;i ++ )
@@ -1457,6 +1457,8 @@ eqjoinsel(PG_FUNCTION_ARGS)
14571457else
14581458unmatchfreq1 += numbers1 [i ];
14591459}
1460+ CLAMP_PROBABILITY (matchfreq1 );
1461+ CLAMP_PROBABILITY (unmatchfreq1 );
14601462matchfreq2 = unmatchfreq2 = 0.0 ;
14611463for (i = 0 ;i < nvalues2 ;i ++ )
14621464{
@@ -1465,6 +1467,8 @@ eqjoinsel(PG_FUNCTION_ARGS)
14651467else
14661468unmatchfreq2 += numbers2 [i ];
14671469}
1470+ CLAMP_PROBABILITY (matchfreq2 );
1471+ CLAMP_PROBABILITY (unmatchfreq2 );
14681472pfree (hasmatch1 );
14691473pfree (hasmatch2 );
14701474
@@ -1474,6 +1478,8 @@ eqjoinsel(PG_FUNCTION_ARGS)
14741478 */
14751479otherfreq1 = 1.0 - stats1 -> stanullfrac - matchfreq1 - unmatchfreq1 ;
14761480otherfreq2 = 1.0 - stats2 -> stanullfrac - matchfreq2 - unmatchfreq2 ;
1481+ CLAMP_PROBABILITY (otherfreq1 );
1482+ CLAMP_PROBABILITY (otherfreq2 );
14771483
14781484/*
14791485 * We can estimate the total selectivity from the point of
@@ -1544,6 +1550,9 @@ eqjoinsel(PG_FUNCTION_ARGS)
15441550if (HeapTupleIsValid (statsTuple2 ))
15451551ReleaseSysCache (statsTuple2 );
15461552}
1553+
1554+ CLAMP_PROBABILITY (selec );
1555+
15471556PG_RETURN_FLOAT8 ((float8 )selec );
15481557}
15491558
@@ -2213,6 +2222,9 @@ convert_timevalue_to_scalar(Datum value, Oid typid)
22132222 *
22142223 * var: identifies the attribute to examine.
22152224 * stats: pg_statistic tuple for attribute, or NULL if not available.
2225+ *
2226+ * NB: be careful to produce an integral result, since callers may compare
2227+ * the result to exact integer counts.
22162228 */
22172229static double
22182230get_att_numdistinct (Query * root ,Var * var ,Form_pg_statistic stats )
@@ -2254,7 +2266,7 @@ get_att_numdistinct(Query *root, Var *var, Form_pg_statistic stats)
22542266if (stats -> stadistinct > 0.0 )
22552267return stats -> stadistinct ;
22562268if (stats -> stadistinct < 0.0 )
2257- return - stats -> stadistinct * ntuples ;
2269+ return floor (( - stats -> stadistinct * ntuples ) + 0.5 ) ;
22582270}
22592271
22602272/*