Branch data     Line data    Source code

       1                 :            : // *****************************************************************************
       2                 :            : /*!
       3                 :            :   \file      src/RNGTest/TestU01Suite.cpp
       4                 :            :   \copyright 2012-2015 J. Bakosi,
       5                 :            :              2016-2018 Los Alamos National Security, LLC.,
       6                 :            :              2019-2021 Triad National Security, LLC.
       7                 :            :              All rights reserved. See the LICENSE file for details.
       8                 :            :   \brief     TestU01 random number generator test suite
       9                 :            :   \details   This file declares the TestU01 random number generator test suite,
      10                 :            :     which facilitates subjecting any supported random number generator to a
      11                 :            :     battery of statistical tests interfaced to the TestU01 library.
      12                 :            : */
      13                 :            : // *****************************************************************************
      14                 :            : 
      15                 :            : #include <string>
      16                 :            : #include <iostream>
      17                 :            : #include <cstddef>
      18                 :            : 
      19                 :            : #include "NoWarning/format.hpp"
      20                 :            : 
      21                 :            : #include "TestU01Suite.hpp"
      22                 :            : #include "TestStack.hpp"
      23                 :            : #include "SmallCrush.hpp"
      24                 :            : #include "Crush.hpp"
      25                 :            : #include "BigCrush.hpp"
      26                 :            : #include "Options/RNG.hpp"
      27                 :            : #include "RNGTest/Options/Battery.hpp"
      28                 :            : #include "NoWarning/rngtest.decl.h"
      29                 :            : #include "WalkerBuildConfig.hpp"
      30                 :            : 
      31                 :            : extern CProxy_Main mainProxy;
      32                 :            : 
      33                 :            : namespace rngtest {
      34                 :            : 
      35                 :            : extern TestStack g_testStack;
      36                 :            : 
      37                 :            : } // rngtest::
      38                 :            : 
      39                 :            : using rngtest::TestU01Suite;
      40                 :            : 
      41                 :          4 : TestU01Suite::TestU01Suite( ctr::BatteryType suite ) :
      42                 :            :   m_ctrs(),
      43                 :            :   m_tests(),
      44                 :            :   m_name(),
      45                 :            :   m_npval(0),
      46                 :            :   m_ncomplete(0),
      47                 :            :   m_ntest(0),
      48                 :            :   m_nfail(),
      49                 :            :   m_time(),
      50         [ +  - ]:          4 :   m_failed()
      51                 :            : // *****************************************************************************
      52                 :            : // Constructor
      53                 :            : //! \param[in] suite Enum id selecting TestU01 battery type
      54                 :            : // *****************************************************************************
      55                 :            : {
      56                 :            :   // Add statistical tests to suite
      57         [ +  - ]:          4 :   if ( suite == ctr::BatteryType::SMALLCRUSH )
      58         [ +  - ]:          8 :     m_name = addTests< SmallCrush >();
      59         [ -  - ]:          0 :   else if ( suite == ctr::BatteryType::CRUSH )
      60         [ -  - ]:          0 :     m_name = addTests< Crush >();
      61         [ -  - ]:          0 :   else if ( suite == ctr::BatteryType::BIGCRUSH )
      62         [ -  - ]:          0 :     m_name = addTests< BigCrush >();
      63 [ -  - ][ -  - ]:          0 :   else Throw( "Non-TestU01 RNG test suite passed to TestU01Suite" );
         [ -  - ][ -  - ]
         [ -  - ][ -  - ]
      64                 :            : 
      65                 :            :   // Construct all tests and store handles
      66 [ +  + ][ +  - ]:        244 :   for (const auto& t : m_ctrs) m_tests.emplace_back( t() );
      67                 :            : 
      68                 :            :   // Collect number of results from all tests (one per RNG)
      69 [ +  - ][ +  + ]:         44 :   for (std::size_t i=0; i<ntest(); ++i) m_tests[i].npval();
      70                 :          4 : }
      71                 :            : 
      72                 :            : void
      73                 :         40 : TestU01Suite::npval( std::size_t n )
      74                 :            : // *****************************************************************************
      75                 :            : // Collect number of p-values from a statistical test
      76                 :            : //! \param[in] n Number of p-values the test contributes to the total
      77                 :            : // *****************************************************************************
      78                 :            : {
      79                 :         40 :   m_npval += n;
      80                 :            : 
      81         [ +  + ]:         40 :   if ( ++m_ntest == ntest() ) {
      82 [ +  - ][ +  - ]:          4 :     printer().battery( ntest(), m_npval );
      83                 :            :     // Collect test names from all tests (one per RNG)
      84                 :          4 :     m_ntest = 0;
      85         [ +  + ]:         44 :     for (std::size_t i=0; i<ntest(); ++i) m_tests[i].names();
      86                 :            :   }
      87                 :         40 : }
      88                 :            : 
      89                 :            : void
      90                 :         40 : TestU01Suite::names( std::vector< std::string > n )
      91                 :            : // *****************************************************************************
      92                 :            : // Collect test names from a statistical test
      93                 :            : //! \param[in] n Vector of test names (there can be more than one from one test)
      94                 :            : // *****************************************************************************
      95                 :            : {
      96                 :         40 :   auto print = printer();
      97                 :            : 
      98         [ +  - ]:         40 :   print.names( n );
      99                 :            : 
     100 [ +  - ][ +  + ]:         40 :   if ( ++m_ntest == ntest() ) {
     101                 :            :     const auto& rngs = g_inputdeck.get< tag::selected, tag::rng >();
     102         [ +  - ]:          8 :     std::stringstream ss;
     103         [ +  - ]:          4 :     ss << "RNGs tested (" << rngs.size() << ")";
     104         [ +  - ]:          4 :     print.section( ss.str() );
     105                 :            :     #ifdef HAS_MKL
     106         [ +  - ]:          4 :     print.MKLParams( rngs, g_inputdeck.get< tag::param, tag::rngmkl >() );
     107                 :            :     #endif
     108         [ +  - ]:          4 :     print.RNGSSEParams( rngs, g_inputdeck.get< tag::param, tag::rngsse >() );
     109         [ +  - ]:          4 :     print.Random123Params( rngs,
     110                 :            :                            g_inputdeck.get< tag::param, tag::rng123 >() );
     111                 :            :     print.endpart();
     112         [ +  - ]:          4 :     print.part( m_name );
     113         [ +  - ]:          4 :     print.statshead( "Statistics computed",
     114                 :          4 :                      m_npval*rngs.size(),
     115         [ +  - ]:          8 :                      m_ctrs.size() );
     116                 :            : 
     117                 :            :     // Run battery of RNG tests
     118         [ +  + ]:        124 :     for (const auto& t : m_tests) t.run();
     119                 :            : 
     120                 :            :     // Initialize space for counting the number failed tests per RNG. Note
     121                 :            :     // that we could use tk::ctr::RNGType as the map-key here instead of the
     122                 :            :     // RNG name (RNGType would be an enum vs the name which is a string),
     123                 :            :     // but then the RNGType would have to be part of the status (instead of
     124                 :            :     // the RNG name) returning from a completed test by TestU01Props::run().
     125                 :            :     // That would require a return type that is less generic than the
     126                 :            :     // current vector of vector of strings. Since the RNG name is already
     127                 :            :     // part of the status, we just match the RNG name for counting failed
     128                 :            :     // tests per RNG and keep the status more generic. See also the
     129                 :            :     // discussion on the return type in TestU01Props::run().
     130         [ +  - ]:          8 :     tk::ctr::RNG rng;
     131         [ +  + ]:         16 :     for (const auto& r : rngs )
     132         [ +  - ]:         24 :       m_nfail[ rng.name(r) ] = 0;
     133                 :            :   }
     134                 :         40 : }
     135                 :            : 
     136                 :            : void
     137                 :        120 : TestU01Suite::evaluate( std::vector< std::vector< std::string > > status )
     138                 :            : // *****************************************************************************
     139                 :            : // Evaluate statistical test
     140                 :            : //! \param[in] status Status vectors of strings for a test
     141                 :            : // *****************************************************************************
     142                 :            : {
     143         [ +  - ]:        240 :   printer().test( ++m_ncomplete, m_ctrs.size(), m_nfail, status );
     144                 :            : 
     145                 :            :   // Store information on failed test for final assessment
     146         [ +  + ]:        300 :   for (std::size_t p=0; p<status[1].size(); ++p)
     147         [ +  + ]:        180 :     if (status[1][p].size() > 4)
     148                 :          7 :       m_failed.emplace_back( status[0][p], status[2][0], status[1][p] );
     149                 :            : 
     150         [ +  + ]:        120 :   if ( m_ncomplete == m_ctrs.size() ) {
     151                 :            :     // Collect measured test run times
     152                 :          4 :     m_ncomplete = 0;
     153         [ +  + ]:        124 :     for (const auto& t : m_tests) t.time();
     154                 :            :   }
     155                 :        120 : }
     156                 :            : 
     157                 :            : void
     158                 :        120 : TestU01Suite::time( std::pair< std::string, tk::real > t )
     159                 :            : // *****************************************************************************
     160                 :            : // Collect test times measured in seconds from a statistical test
     161                 :            : //! \param[in] t Measured time to do the test for an RNG
     162                 :            : // *****************************************************************************
     163                 :            : {
     164                 :        120 :   m_time[ t.first ] += t.second;
     165                 :            : 
     166         [ +  + ]:        120 :   if ( ++m_ncomplete == m_ctrs.size() ) assess();
     167                 :        120 : }
     168                 :            : 
     169                 :            : void
     170                 :          4 : TestU01Suite::assess()
     171                 :            : // *****************************************************************************
     172                 :            : // Output final assessment
     173                 :            : // *****************************************************************************
     174                 :            : {
     175                 :          4 :   auto print = printer();
     176                 :            : 
     177                 :            :   // Output summary of failed tests for all RNGs tested
     178         [ +  + ]:          4 :   if ( !m_failed.empty() ) {
     179                 :            :     const auto& rngs = g_inputdeck.get< tag::selected, tag::rng >();
     180 [ +  - ][ +  - ]:          6 :     print.failed( "Failed statistics", m_npval*rngs.size(), m_failed );
     181 [ +  - ][ +  - ]:          2 :   } else print.note< tk::QUIET >( "All tests passed" );
     182                 :            : 
     183                 :            :   // Cost and quality assessment only for more than one RNG
     184         [ +  - ]:          4 :   if (m_time.size() > 1) {
     185                 :            :     // Output measured times per RNG in order of computational cost
     186         [ +  - ]:          4 :     print.cost( "Generator cost",
     187                 :            :                 "Measured times in seconds in increasing order (low is good)",
     188 [ +  - ][ +  - ]:         12 :                 m_time );
         [ +  - ][ +  - ]
                 [ -  - ]
     189                 :            :     // Output number of failed tests per RNG in order of decreasing quality
     190         [ +  - ]:          4 :     print.rank( "Generator quality",
     191                 :            :                 "Number of failed tests in increasing order (low is good)",
     192 [ +  - ][ +  - ]:         12 :                 m_nfail );
         [ +  - ][ -  - ]
     193                 :            :   }
     194                 :            : 
     195                 :            :   // Quit
     196         [ +  - ]:          4 :   mainProxy.finalize();
     197                 :          4 : }
     198                 :            : 
     199                 :            : std::size_t
     200                 :        172 : TestU01Suite::ntest() const
     201                 :            : // *****************************************************************************
     202                 :            : // Return the number of statistical tests per each RNG tested
     203                 :            : //! \return The number of statistical tests for each RNG tested
     204                 :            : // *****************************************************************************
     205                 :            : {
     206                 :            :   const auto& rngs = g_inputdeck.get< tag::selected, tag::rng >();
     207                 :        172 :   return m_ctrs.size() / rngs.size();
     208                 :            : }
     209                 :            : 
     210                 :            : #include "NoWarning/testu01suite.def.h"