[Beignet] [PATCH v2] Statistics of case running
Zhigang Gong
zhigang.gong at linux.intel.com
Tue Apr 1 22:36:21 PDT 2014
LGTM, pushed, thanks.
On Tue, Apr 01, 2014 at 04:31:26PM +0800, Yi Sun wrote:
> summary:
> -----------------
> 1. Add struct RStatistics to count passed number(passCount), failed number(failCount), finished run number(finishrun).
>
> 2. Print statistics line , if the termial is too narrow, doesn't print it:
> ......
> test_load_program_from_bin() [SUCCESS]
> profiling_exec() [SUCCESS]
> enqueue_copy_buf() [SUCCESS]
> [run/total: 656/656] pass: 629; fail: 25; pass rate: 0.961890
>
> 3. If case crashes, count it as failed, add the function to show statistic summary.
>
> 4. When all cases finished, list a summary like follows:
> summary:
> ----------
> total: 656
> run: 656
> pass: 629
> fail: 25
> pass rate: 0.961890
>
> 5. If ./utest_run &> log, the log will be a little messy, tring the following command to analyse the log:
>
> sed 's/\r/\n/g' log | egrep "\w*\(\)" | sed -e 's/\s//g'
>
> After analysed:
> -----------------
> ......
> builtin_minmag_float2()[SUCCESS]
> builtin_minmag_float4()[SUCCESS]
> builtin_minmag_float8()[SUCCESS]
> builtin_minmag_float16()[SUCCESS]
> builtin_nextafter_float()[FAILED]
> builtin_nextafter_float2()[FAILED]
> builtin_nextafter_float4()[FAILED]
> ......
>
> 6. Fix one issue, print out the crashed case name.
>
> 7. Delete the debug line in utests/compiler_basic_arithmetic.cpp, which
> output the kernel name.
>
> 8. Define function statistics() in struct UTest, which called by "utest_run -a/-c/-n".
> We just call this function to run each case, and print the statistics line.
> ---
> utests/compiler_basic_arithmetic.cpp | 1 -
> utests/utest.cpp | 88 ++++++++++++++++++++++++++++++------
> utests/utest.hpp | 26 +++++++++--
> 3 files changed, 94 insertions(+), 21 deletions(-)
>
> Signed-off-by: Yi Sun <yi.sun at intel.com>
> Signed-off-by: Yangwei Shui <yangweix.shui at intel.com>
>
> diff --git a/utests/compiler_basic_arithmetic.cpp b/utests/compiler_basic_arithmetic.cpp
> index 0e5ec41..ba05de0 100644
> --- a/utests/compiler_basic_arithmetic.cpp
> +++ b/utests/compiler_basic_arithmetic.cpp
> @@ -15,7 +15,6 @@ static void test_exec(const char* kernel_name)
>
> // Setup kernel and buffers
> OCL_CREATE_KERNEL_FROM_FILE("compiler_basic_arithmetic", kernel_name);
> -std::cout <<"kernel name: " << kernel_name << std::endl;
> buf_data[0] = (T*) malloc(sizeof(T) * n);
> buf_data[1] = (T*) malloc(sizeof(T) * n);
> for (uint32_t i = 0; i < n; ++i) ((T*)buf_data[0])[i] = (T) rand();
> diff --git a/utests/utest.cpp b/utests/utest.cpp
> index 718916f..e747309 100644
> --- a/utests/utest.cpp
> +++ b/utests/utest.cpp
> @@ -26,14 +26,35 @@
> #include <vector>
> #include <string>
> #include <iostream>
> +#include <sys/ioctl.h>
> +#include <unistd.h>
> #include <cstring>
>
> +#define MAX_SUM_LINE 256
> +
> using namespace std;
> vector<UTest> *UTest::utestList = NULL;
> +// Initialize and declare statistics struct
> +RStatistics UTest::retStatistics;
> +
> void releaseUTestList(void) { delete UTest::utestList; }
> +void runAllNoIssueAtExit(void) {
> + // If case crashes, count it as fail, and accumulate finishrun
> + if(UTest::retStatistics.finishrun != UTest::utestList->size()) {
> + UTest::retStatistics.finishrun++;
> + UTest::retStatistics.failCount++;
> + }
> + printf("\nsummary:\n----------\n");
> + printf(" total: %zu\n",UTest::utestList->size());
> + printf(" run: %zu\n",UTest::retStatistics.finishrun);
> + printf(" pass: %zu\n",UTest::retStatistics.passCount);
> + printf(" fail: %zu\n",UTest::retStatistics.failCount);
> + printf(" pass rate: %f\n",1-(float)UTest::retStatistics.failCount/(float)UTest::utestList->size());
> + }
>
> UTest::UTest(Function fn, const char *name, bool haveIssue, bool needDestroyProgram)
> : fn(fn), name(name), haveIssue(haveIssue), needDestroyProgram(needDestroyProgram) {
> +
> if (utestList == NULL) {
> utestList = new vector<UTest>;
> atexit(releaseUTestList);
> @@ -46,16 +67,53 @@ static bool strequal(const char *s1, const char *s2) {
> return false;
> }
>
> +void UTest::do_run(struct UTest utest){
> + // winsize is a struct in ioctl.h, contains terminal column number
> + struct winsize size;
> + char spaceList[MAX_SUM_LINE] = {0};
> +
> + //Obtain terminal column size
> + ioctl(STDOUT_FILENO, TIOCGWINSZ, &size);
> +
> + //A string contain MAX_SUM_LINE spaces, to hide the statistic line in stdout
> + for (size_t j = 0; j < size.ws_col; j++){
> + if ( j >= MAX_SUM_LINE - 1 ) break;
> + spaceList[j] = ' ';
> + spaceList[j+1] = '\0';
> + }
> + printf("\r%s\r%s()", spaceList, utest.name);
> +
> + // Run one case in utestList
> + (utest.fn)();
> +
> + // Print dynamic statistics line
> + sprintf(spaceList, "\n [run/total: %zu/%zu]\
> + pass: %zu; fail: %zu; pass rate: %f\r",
> + retStatistics.finishrun+1, utestList->size(),
> + retStatistics.passCount,
> + retStatistics.failCount,
> + 1-(float)retStatistics.failCount/(float)utestList->size());
> +
> + // If terminal column size lower than length of statistic line, print nothing, If not, print the statistics line
> + if (size.ws_col > strlen(spaceList))
> + printf("%s", spaceList);
> + else
> + printf("\n");
> +
> + // Refresh console
> + fflush(stdout);
> +}
> +
> void UTest::run(const char *name) {
> if (name == NULL) return;
> if (utestList == NULL) return;
> - for (size_t i = 0; i < utestList->size(); ++i) {
> - const UTest &utest = (*utestList)[i];
> + atexit(runAllNoIssueAtExit);
> +
> + for (; retStatistics.finishrun < utestList->size(); ++retStatistics.finishrun) {
> + const UTest &utest = (*utestList)[retStatistics.finishrun];
> if (utest.name == NULL || utest.fn == NULL ) continue;
> if (strequal(utest.name, name)) {
> - std::cout << utest.name << ":" << std::endl;
> - (utest.fn)();
> - std::cout << std::endl;
> + do_run(utest);
> cl_kernel_destroy(true);
> cl_buffer_destroy();
> }
> @@ -64,12 +122,12 @@ void UTest::run(const char *name) {
>
> void UTest::runAll(void) {
> if (utestList == NULL) return;
> - for (size_t i = 0; i < utestList->size(); ++i) {
> - const UTest &utest = (*utestList)[i];
> + atexit(runAllNoIssueAtExit);
> +
> + for (; retStatistics.finishrun < utestList->size(); ++retStatistics.finishrun) {
> + const UTest &utest = (*utestList)[retStatistics.finishrun];
> if (utest.fn == NULL) continue;
> - std::cout << utest.name << ":" << std::endl;
> - (utest.fn)();
> - std::cout << std::endl;
> + do_run(utest);
> cl_kernel_destroy(utest.needDestroyProgram);
> cl_buffer_destroy();
> }
> @@ -77,12 +135,12 @@ void UTest::runAll(void) {
>
> void UTest::runAllNoIssue(void) {
> if (utestList == NULL) return;
> - for (size_t i = 0; i < utestList->size(); ++i) {
> - const UTest &utest = (*utestList)[i];
> + atexit(runAllNoIssueAtExit);
> +
> + for (; retStatistics.finishrun < utestList->size(); ++retStatistics.finishrun) {
> + const UTest &utest = (*utestList)[retStatistics.finishrun];
> if (utest.fn == NULL || utest.haveIssue) continue;
> - std::cout << utest.name << ":" << std::endl;
> - (utest.fn)();
> - std::cout << std::endl;
> + do_run(utest);
> cl_kernel_destroy(utest.needDestroyProgram);
> cl_buffer_destroy();
> }
> diff --git a/utests/utest.hpp b/utests/utest.hpp
> index 01d4a8c..e6a7749 100644
> --- a/utests/utest.hpp
> +++ b/utests/utest.hpp
> @@ -31,6 +31,14 @@
> #include <vector>
> #include <iostream>
>
> +/*! struct for statistics */
> +struct RStatistics
> +{
> + size_t passCount;
> + size_t failCount;
> + size_t finishrun;
> +};
> +
> /*! Quick and dirty unit test system with registration */
> struct UTest
> {
> @@ -58,6 +66,10 @@ struct UTest
> static void runAll(void);
> /*! List all test cases */
> static void listAllCases(void);
> + /*! Statistics struct */
> + static RStatistics retStatistics;
> + /*! Do run a test case actually */
> + static void do_run(struct UTest utest);
> };
>
> /*! Register a new unit test */
> @@ -84,11 +96,13 @@ struct UTest
> do { \
> try { \
> EXPR; \
> - std::cout << " " << #EXPR << " [SUCCESS]" << std::endl; \
> + std::cout << " [SUCCESS]"; \
> + UTest::retStatistics.passCount += 1; \
> } \
> catch (Exception e) { \
> - std::cout << " " << #EXPR << " [FAILED]" << std::endl; \
> - std::cout << " " << e.what() << std::endl; \
> + std::cout << " [FAILED]"; \
> + std::cout << "\n " << e.what(); \
> + UTest::retStatistics.failCount++; \
> } \
> } while (0)
>
> @@ -96,10 +110,12 @@ struct UTest
> do { \
> try { \
> EXPR; \
> - std::cout << " " << #EXPR << " [FAILED]" << std::endl; \
> + std::cout << " [FAILED]"; \
> + retStatistics.failCount++; \
> } \
> catch (gbe::Exception e) { \
> - std::cout << " " << #EXPR << " [SUCCESS]" << std::endl; \
> + std::cout << " [SUCCESS]"; \
> + retStatistics.passCount++; \
> } \
> } while (0)
>
> --
> 1.8.4.2
>
> _______________________________________________
> Beignet mailing list
> Beignet at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/beignet
More information about the Beignet
mailing list