[Libreoffice-commits] .: sc/source

[Libreoffice Gerrit user]

 sc/source/core/tool/interpr3.cxx |   24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

New commits:
commit a9a734a680dda8c177f92cf2f14061c9b785ec8d
Author: Werner Koerner <wk661lo at gmail.com>
Date:   Tue Dec 11 22:23:39 2012 +0100

    calc: fix mathematical error in CritBinom
    
    CRITBINOM(462,0.8,0.9), expected result 381, calculated value is 462.
    Similar errors in BinomDist, B, CritBinom and NegBinomDist were
    fixed by commit 5cf55f5b7800e443c4f087e72ae05abc8b7fef45.
    
    Change-Id: I9b12a1c4410ec72258ae1fb68409ad00c922b94c
    Reviewed-on: https://gerrit.libreoffice.org/1301
    Reviewed-by: Eike Rathke <erack at redhat.com>
    Tested-by: Eike Rathke <erack at redhat.com>

diff --git a/sc/source/core/tool/interpr3.cxx b/sc/source/core/tool/interpr3.cxx
index ada1cff..638806d 100644
--- a/sc/source/core/tool/interpr3.cxx
+++ b/sc/source/core/tool/interpr3.cxx
@@ -1348,26 +1348,25 @@ void ScInterpreter::ScCritBinom()
     RTL_LOGFILE_CONTEXT_AUTHOR( aLogger, "sc", "er", "ScInterpreter::ScCritBinom" );
     if ( MustHaveParamCount( GetByte(), 3 ) )
     {
-        double alpha  = GetDouble();                    // alpha
-        double p      = GetDouble();                    // p
+        double alpha  = GetDouble();
+        double p      = GetDouble();
         double n      = ::rtl::math::approxFloor(GetDouble());
         if (n < 0.0 || alpha <= 0.0 || alpha >= 1.0 || p < 0.0 || p > 1.0)
             PushIllegalArgument();
         else
         {
-            double q = 1.0 - p;
+            double q = (0.5 - p) + 0.5;           // get one bit more for p near 1.0
             double fFactor = pow(q,n);
-            if (fFactor == 0.0)
+            if (fFactor <= ::std::numeric_limits<double>::min())
             {
                 fFactor = pow(p, n);
-                if (fFactor == 0.0)
+                if (fFactor <= ::std::numeric_limits<double>::min())
                     PushNoValue();
                 else
                 {
-                    double fSum = 1.0 - fFactor; sal_uLong max = (sal_uLong) n;
-                    sal_uLong i;
-
-                    for ( i = 0; i < max && fSum >= alpha; i++)
+                    double fSum = 1.0 - fFactor;
+                    sal_uInt32 max = static_cast<sal_uInt32> (n), i;
+                    for (i = 0; i < max && fSum >= alpha; i++)
                     {
                         fFactor *= (n-i)/(i+1)*q/p;
                         fSum -= fFactor;
@@ -1377,10 +1376,9 @@ void ScInterpreter::ScCritBinom()
             }
             else
             {
-                double fSum = fFactor; sal_uLong max = (sal_uLong) n;
-                sal_uLong i;
-
-                for ( i = 0; i < max && fSum < alpha; i++)
+                double fSum = fFactor;
+                sal_uInt32 max = static_cast<sal_uInt32> (n), i;
+                for (i = 0; i < max && fSum < alpha; i++)
                 {
                     fFactor *= (n-i)/(i+1)*p/q;
                     fSum += fFactor;