The following procedure demonstrates how much the choice of numerical algorithm can effect the accuracy of results. This procedure computes the standard deviation by two methods. All standard deviation results should be the same value 1.4907122E-01 Note how the method 1 blows up when the data is offset by 1000.0 Note significant errors in method 1 with even an offset of 100.0 This is using 32 bit floating point that gives at least 6 digits accuracy with float_text_io; use float_text_io; with float_math_lib; use float_math_lib; with text_io; use text_io; procedure stddev is X : array(1..3,1..10) of float :=((0.8,0.8,0.9,0.9,1.0,1.0,1.1,1.1,1.2,1.2), (100.8,100.8,100.9,100.9,101.0,101.0,101.1,101.1,101.2,101.2), (1000.8,1000.8,1000.9,1000.9,1001.0,1001.0,1001.1,1001.1,1001.2,1001.2)); sum_x : float; sum_x_x : float; mean : float; sum : float; standard_deviation : float; N : float := float(X'length(2)); begin float_text_io.default_aft := 7; -- -- method 1, one loop -- for j in X'range(1) loop put_line("STDDEV"); sum_x := 0.0; sum_x_x := 0.0; for i in X'range(2) loop -- accumulate sum of X and sum of X**2 sum_x := sum_x + X(j,i); sum_x_x := sum_x_x + X(j,i)*X(j,i); end loop; put(sum_x); put_line(" = sum_x, sum of X's"); put(sum_x_x); put_line(" = sum_x_x, sum of Xi **2"); put(sum_x*sum_x/N); put_line(" = sum_x * sum_x / N"); if sum_x*sum_x/N > sum_x_x then put_line("computation error from method 1"); else standard_deviation := SQRT( (sum_x_x - sum_x*sum_x/N) / (N-1.0) ); end if; put(standard_deviation); put_line(" = standard deviation, method 1, one loop "); new_line; -- -- method 2, two loops -- mean := 0.0; for i in X'range(2) loop -- accumulate sum to get mean mean := mean + X(j,i); end loop; mean := mean / N; -- have mean put(mean); put_line(" = mean"); sum := 0.0; for i in X'range(2) loop -- accumulate deviations sum := sum + (X(j,i)-mean)*(X(j,i)-mean); end loop; put(sum); -- have sum of deviations put_line(" = sum of square of deviations from mean"); standard_deviation := SQRT( sum / (N-1.0) ); put(standard_deviation); put_line(" = standard deviation, method 2, two loops "); new_line(2); end loop; end stddev; Results of running the above procedure STDDEV 1.0000000E+01 = sum_x, sum of X's 1.0200001E+01 = sum_x_x, sum of Xi **2 1.0000000E+01 = sum_x * sum_x / N 1.4907148E-01 = standard deviation, method 1, one loop 1.0000000E+00 = mean 2.0000005E-01 = sum of square of deviations from mean 1.4907122E-01 = standard deviation, method 2, two loops STDDEV 1.0100000E+03 = sum_x, sum of X's 1.0201020E+05 = sum_x_x, sum of Xi **2 1.0201000E+05 = sum_x * sum_x / N 1.4731391E-01 = standard deviation, method 1, one loop 1.0100000E+02 = mean 1.9999389E-01 = sum of square of deviations from mean 1.4906892E-01 = standard deviation, method 2, two loops STDDEV 1.0010001E+04 = sum_x, sum of X's 1.0020010E+07 = sum_x_x, sum of Xi **2 1.0020012E+07 = sum_x * sum_x / N computation error from method 1 1.4906892E-01 = standard deviation, method 1, one loop 1.0010001E+03 = mean 2.0000015E-01 = sum of square of deviations from mean 1.4907126E-01 = standard deviation, method 2, two loops