[Libreoffice-commits] dev-tools.git: test-bugzilla-files/execute.sh test-bugzilla-files/new-control.py test-bugzilla-files/test-bugzilla-files.py

[Markus Mohrhard]

 test-bugzilla-files/execute.sh             |    2 
 test-bugzilla-files/new-control.py         |   77 +++++++++++++++++++++--------
 test-bugzilla-files/test-bugzilla-files.py |   22 +-------
 3 files changed, 63 insertions(+), 38 deletions(-)

New commits:
commit 77eb5fd822f0827e288e26940eb867858d6a6f7b
Author: Markus Mohrhard <markus.mohrhard at googlemail.com>
Date:   Sat Nov 1 22:22:57 2014 +0100

    move from dir based testing to task list based
    
    Instead of going through a whole directory in one task split the whole
    into 100 file tasks. This should make sure that all cores are always
    doing some work.

diff --git a/test-bugzilla-files/execute.sh b/test-bugzilla-files/execute.sh
index 9203a7e..e2c6076 100755
--- a/test-bugzilla-files/execute.sh
+++ b/test-bugzilla-files/execute.sh
@@ -1,4 +1,4 @@
 mkdir /srv/crashtestdata/control/$1
 cd /srv/crashtestdata/control/$1
-TMPDIR=/srv/crashtestdata/tmpdir /home/buildslave/build/instdir/program/python /home/buildslave/source/dev-tools/test-bugzilla-files/test-bugzilla-files.py --soffice=path:/home/buildslave/build/instdir/program/soffice --userdir=file:///home/buildslave/.config/libreoffice_$1/4 /home/buildslave/source/bugdocs3/$1 2>&1 | tee /srv/crashtestdata/console_$i.log
+TMPDIR=/srv/crashtestdata/tmpdir /home/buildslave/build/instdir/program/python /home/buildslave/source/dev-tools/test-bugzilla-files/test-bugzilla-files.py --soffice=path:/home/buildslave/build/instdir/program/soffice --userdir=file:///home/buildslave/.config/libreoffice_$1/4 $1 2>&1 | tee /srv/crashtestdata/console_$i.log
 rm core*
diff --git a/test-bugzilla-files/new-control.py b/test-bugzilla-files/new-control.py
index 25617a7..c27cb5f 100644
--- a/test-bugzilla-files/new-control.py
+++ b/test-bugzilla-files/new-control.py
@@ -21,26 +21,63 @@ import os.path
 import concurrent.futures
 import time
 import subprocess
+import getopt
+import sys
 
-def get_directories():
-    d='.'
-    directories = [o for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]
-    return directories
+def partition(l, n):
+    for i in range(0, len(l), n):
+        yield l[i:i+n]
+
+def get_tasks(directory, files_per_task):
+    flist = [os.path.join(dirpath, f) for dirpath, dirnames, fnames in os.walk(directory) for f in fnames]
+
+    partitioned_list = list(partition(flist, files_per_task))
+    task_files = []
+    i = 0
+    for list_item in partitioned_list:
+        filename = "task" + str(i)
+        task_file = open(filename, "w")
+        for item in list_item:
+            task_file.write("%s\n" % item)
+        task_files.append(filename)
+        i += 1
+    return task_files
+
+def execute_task(task_file):
+    # subprocess.call("./execute.sh " + task_file, shell=True)
+    time.sleep(1)
+
+def usage():
+    message = """usage: {program} [option] dir"
+ - h | --help: print usage information
+ 
+ 'dir' is the path to the directory with the test files"""
+    print(message.format(program = os.path.basename(sys.argv[0])))
+
+if __name__ == "__main__":
+    opts, args = getopt.getopt(sys.argv[1:], "hd:", ["help", "directory="])
+    print(args)
+    print(opts)
+    if "-h" in opts or "--help" in opts:
+        usage()
+        sys.exit()
+
+    for opt, arg in opts:
+        if opt in ("-d", "--directory"):
+            directory = arg
 
-def execute_task(directory):
-    print("Yeah")
     print(directory)
-    subprocess.call("./execute.sh " + directory, shell=True)
-    time.sleep(10)
-    return 
-
-with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
-    future_to_task = {executor.submit(execute_task, dirs): dirs for dirs in get_directories()}
-    for future in concurrent.futures.as_completed(future_to_task):
-        task = future_to_task[future]
-        try:
-            future.result()
-        except Exception as exc:
-            print('%r generated an exception: %s' % (task, exc))
-        else:
-            print('%r successfully passed' % (task))
+    if not os.path.isdir(directory):
+        sys.exit(1)
+
+    task_size = 100
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        future_to_task = {executor.submit(execute_task, task_file): task_file for task_file in get_tasks(directory, task_size)}
+        for future in concurrent.futures.as_completed(future_to_task):
+            task = future_to_task[future]
+            try:
+                future.result()
+            except Exception as exc:
+                print('%r generated an exception: %s' % (task, exc))
+            else:
+                print('%r successfully passed' % (task))
diff --git a/test-bugzilla-files/test-bugzilla-files.py b/test-bugzilla-files/test-bugzilla-files.py
index 2bd907e..11cf41c 100644
--- a/test-bugzilla-files/test-bugzilla-files.py
+++ b/test-bugzilla-files/test-bugzilla-files.py
@@ -69,21 +69,9 @@ def partition(list, pred):
             right.append(e)
     return (left, right)
 
-def filelist(dir, suffix):
-    if len(dir) == 0:
-        raise Exception("filelist: empty directory")
-    if not(dir[-1] == "/"):
-        dir += "/"
-    files = [dir + f for f in os.listdir(dir)]
-#    print(files)
-    return [f for f in files
-                    if os.path.isfile(f) and os.path.splitext(f)[1] == suffix]
-
-def getFiles(dirs, suffix):
-#    print( dirs )
-    files = []
-    for dir in dirs:
-        files += filelist(dir, suffix)
+def getFiles(file_list_name):
+    file_list = open(file_list_name, "r")
+    files = file_list.readlines()
     return files
 
 ### UNO utilities ###
@@ -593,7 +581,7 @@ def parseArgs(argv):
     return (dict(optlist), args)
 
 def usage():
-    message = """usage: {program} [option]... [directory]..."
+    message = """usage: {program} [option]... [task_file]..."
  -h | --help:      print usage information
  --soffice=method:location
                    specify soffice instance to connect to
@@ -601,7 +589,7 @@ def usage():
  --userdir=URL     specify user installation directory for 'path' method
  --valgrind        pass --valgrind to soffice for 'path' method
 
- 'location' is a pathname, not a URL. 'userdir' is a URL. the 'directory' parameters should be
+ 'location' is a pathname, not a URL. 'userdir' is a URL. the 'task_file' parameters should be
   full absolute pathnames, not URLs."""
     print(message.format(program = os.path.basename(sys.argv[0])))