[Libreoffice-commits] dev-tools.git: qa/createBlogReport.py

Libreoffice Gerrit user logerrit at kemper.freedesktop.org
Fri Sep 7 15:51:54 UTC 2018


 qa/createBlogReport.py |  350 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 350 insertions(+)

New commits:
commit 07438263d7b86ab68f4affff1ccab060b3733960
Author:     Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Fri Sep 7 17:50:37 2018 +0200
Commit:     Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Fri Sep 7 17:51:27 2018 +0200

    QA: new script for blog reports

diff --git a/qa/createBlogReport.py b/qa/createBlogReport.py
new file mode 100755
index 0000000..99a8246
--- /dev/null
+++ b/qa/createBlogReport.py
@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+import common
+from datetime import datetime, timedelta
+import argparse
+
+import matplotlib
+import matplotlib.pyplot as plt
+
+lKeywords = ['haveBacktrace', 'regression', 'bisected']
+
+
+def util_create_basic_schema():
+    return {
+        'id': [],
+        'author': {},
+        'day': {},
+        'difftime': []
+        }
+
+def util_create_statList():
+    return {
+        'created': util_create_basic_schema(),
+        'confirmed': util_create_basic_schema(),
+        'verified': util_create_basic_schema(),
+        'fixed': util_create_basic_schema(),
+        'metabug': util_create_basic_schema(),
+        'keywords': { k : util_create_basic_schema() for k in lKeywords},
+        'people' : {},
+        'unconfirmedCount' : {},
+        'stat': {'oldest': datetime.now(), 'newest': datetime(2001, 1, 1)}
+    }
+
+def util_increase_action(value, rowId, creatorMail, day, difftime=-1):
+    value['id'].append(rowId)
+    if creatorMail not in value['author']:
+        value['author'][creatorMail] = 0
+    value['author'][creatorMail] += 1
+
+    if day not in value['day']:
+        value['day'][day] = 0
+    value['day'][day] += 1
+
+    if difftime >= 0:
+        value['difftime'].append(difftime)
+
+def util_decrease_action(value, creatorMail, day):
+    value['id'].pop()
+    value['author'][creatorMail] -= 1
+    value['day'][day] -= 1
+
+    if value['difftime']:
+        value['difftime'].pop()
+
+def check_date(xDate, cfg):
+    if xDate >= cfg.Date[0] and xDate < cfg.Date[1]:
+        return True
+    else:
+        return False
+
+def daterange(cfg):
+    for n in range(int ((cfg.Date[1] - cfg.Date[0]).days)):
+        yield cfg.Date[0] + timedelta(n)
+
+def analyze_bugzilla_data(statList, bugzillaData, cfg):
+    print("Analyzing bugzilla\n", end="", flush=True)
+
+
+    unconfirmedCountPerDay = {}
+    fixedBugs = []
+    for key, row in bugzillaData['bugs'].items():
+        rowId = row['id']
+
+        #Ignore META bugs and deletionrequest bugs.
+        if not row['summary'].lower().startswith('[meta]') and row['component'] != 'deletionrequest':
+            creationDate = datetime.strptime(row['creation_time'], "%Y-%m-%dT%H:%M:%SZ")
+
+
+            #Some old bugs were directly created as NEW, skipping the UNCONFIRMED status
+            #Use the oldest bug ID in the unconfirmed list
+            if rowId >= 89589:
+                strDay = creationDate.strftime("%Y-%m-%d")
+                if strDay not in unconfirmedCountPerDay:
+                    unconfirmedCountPerDay[strDay] = 0
+                unconfirmedCountPerDay[strDay] += 1
+
+            rowStatus = row['status']
+            rowResolution = row['resolution']
+
+            rowKeywords = row['keywords']
+
+            creatorMail = row['creator']
+
+            #get information about created bugs in the period of time
+            if check_date(creationDate, cfg):
+                creationDay = str(creationDate.strftime("%Y-%m-%d"))
+                util_increase_action(statList['created'], rowId, creatorMail, creationDay)
+
+            common.util_check_bugzilla_mail(
+                    statList, creatorMail, row['creator_detail']['real_name'], creationDate, rowId)
+
+            isFixed = False
+            isConfirmed = False
+            isVerified = False
+            dayConfirmed = None
+            dayVerified = None
+            authorConfirmed = None
+            authorVerified = None
+            for action in row['history']:
+                actionMail = action['who']
+                actionDate = datetime.strptime(action['when'], "%Y-%m-%dT%H:%M:%SZ")
+
+                common.util_check_bugzilla_mail(
+                        statList, actionMail, '', actionDate, rowId)
+
+                actionDay = str(actionDate.strftime("%Y-%m-%d"))
+                diffTime = (actionDate - creationDate).days
+
+                for change in action['changes']:
+
+                        if change['field_name'] == 'status':
+                            addedStatus = change['added']
+                            removedStatus = change['removed']
+
+                            #See above
+                            if rowId >= 89589:
+                                if removedStatus == "UNCONFIRMED":
+                                    strDay = actionDate.strftime("%Y-%m-%d")
+                                    if strDay not in unconfirmedCountPerDay:
+                                        unconfirmedCountPerDay[strDay] = 0
+                                    unconfirmedCountPerDay[strDay] -= 1
+
+                                elif addedStatus == 'UNCONFIRMED':
+                                    strDay = actionDate.strftime("%Y-%m-%d")
+                                    if strDay not in unconfirmedCountPerDay:
+                                        unconfirmedCountPerDay[strDay] = 0
+                                    unconfirmedCountPerDay[strDay] += 1
+
+                            if check_date(actionDate, cfg):
+                                if removedStatus == "UNCONFIRMED":
+                                    util_increase_action(statList['confirmed'], rowId, actionMail, actionDay, diffTime)
+                                    dayConfirmed = actionDay
+                                    authorConfirmed = actionMail
+                                    isConfirmed = True
+
+                                elif addedStatus == 'UNCONFIRMED' and isConfirmed:
+                                    util_decrease_action(statList['confirmed'], authorConfirmed, dayConfirmed)
+                                    isConfirmed = False
+
+                                if addedStatus == 'VERIFIED':
+                                    util_increase_action(statList['verified'], rowId, creatorMail, actionDay, diffTime)
+                                    dayVerified = actionDay
+                                    authorVerified = actionMail
+                                    isVerified = True
+
+                                elif removedStatus == 'VERIFIED' and isVerified:
+                                    util_decrease_action(statList['verified'], authorVerified, dayVerified)
+                                    isVerified = False
+
+                        elif change['field_name'] == 'resolution':
+                            if check_date(actionDate, cfg):
+                                addedResolution = change['added']
+                                removedResolution = change['removed']
+                                if addedResolution == 'FIXED' and not removedResolution:
+                                    fixedBugs.append(rowId)
+                                    isFixed = True
+
+                                elif removedResolution == 'FIXED' and isFixed and not addedResolution:
+                                    fixedBugs.pop()
+                                    isFixed = False
+
+                        elif change['field_name'] == 'keywords':
+                            if check_date(actionDate, cfg):
+                                keywordsAdded = change['added'].split(", ")
+                                for keyword in keywordsAdded:
+                                    if keyword in lKeywords:
+                                        if keyword in rowKeywords:
+                                            util_increase_action(statList['keywords'][keyword], rowId, actionMail, actionDay, diffTime)
+
+                        elif change['field_name'] == 'blocks':
+                            if check_date(actionDate, cfg):
+                                if change['added']:
+                                    for metabug in change['added'].split(', '):
+                                        if int(metabug) in row['blocks']:
+                                            util_increase_action(statList['metabug'], rowId, actionMail, actionDay, diffTime)
+
+            commentMail = None
+            comments = row['comments'][1:]
+            bugFixers = set()
+            for idx, comment in enumerate(comments):
+                commentMail = comment['creator']
+                commentDate = datetime.strptime(comment['time'], "%Y-%m-%dT%H:%M:%SZ")
+
+                common.util_check_bugzilla_mail(
+                        statList, commentMail, '', commentDate, rowId)
+
+                if check_date(commentDate, cfg) and rowId in fixedBugs:
+                    if commentMail == "libreoffice-commits at lists.freedesktop.org":
+                        commentText = comment['text']
+                        author =  commentText.split(' committed a patch related')[0]
+                        if author not in bugFixers:
+                            bugFixers.add(author)
+                            diffTime = (commentDate - creationDate).days
+                            commentDay = commentDate.strftime("%Y-%m-%d")
+                            util_increase_action(statList['fixed'], rowId, author, commentDay, diffTime)
+
+
+            for person in row['cc_detail']:
+                email = person['email']
+                if commentMail == email or actionMail == email:
+                    common.util_check_bugzilla_mail(statList, email, person['real_name'])
+
+    for k, v in statList['people'].items():
+        if not statList['people'][k]['name']:
+            statList['people'][k]['name'] = statList['people'][k]['email'].split('@')[0]
+
+        statList['people'][k]['oldest'] = statList['people'][k]['oldest'].strftime("%Y-%m-%d")
+        statList['people'][k]['newest'] = statList['people'][k]['newest'].strftime("%Y-%m-%d")
+
+    for single_date in daterange(cfg):
+        single_day = single_date.strftime("%Y-%m-%d")
+
+        #Fill empty days to be displayed on the charts
+        for k0, v0 in statList.items():
+            if k0 == 'keywords':
+                for k1, v1 in statList['keywords'].items():
+                    if single_day not in statList['keywords'][k1]['day']:
+                        statList['keywords'][k1]['day'][single_day] = 0
+            else:
+                if 'day' in statList[k0]:
+                    if single_day not in statList[k0]['day']:
+                        statList[k0]['day'][single_day] = 0
+
+        totalCount = 0
+        for k, v in unconfirmedCountPerDay.items():
+            xDay = datetime.strptime( k, "%Y-%m-%d")
+            if xDay < single_date:
+                totalCount += v
+
+        statList['unconfirmedCount'][single_day] = totalCount
+
+def makeStrong(text):
+    return "<strong>" + str(text) + "</strong>"
+
+def makeLI(text):
+    return "<li>" + str(text) + "</li>"
+
+def makeH2(text):
+    return "<h2>" + str(text) + "</h2>"
+
+def createPlot(valueDict, plotType, plotTitle, plotLabel, plotColor):
+
+    x, y = zip(*sorted(valueDict.items(), key = lambda x:datetime.strptime(x[0], '%Y-%m-%d')))
+    if plotType == "line":
+        plt.plot(y, label=plotLabel, linewidth=2, color=plotColor)
+    elif plotType == "bar":
+        plt.bar(range(len(y)), y, label=plotLabel, width=0.8, color=plotColor)
+
+    plt.xticks(range(len(x)), x, rotation=90)
+    plt.title(plotTitle)
+    plt.xlabel("Date")
+    plt.ylabel("Number");
+    plt.legend();
+    ax = plt.gca()
+    ax.grid(axis="y", linestyle='--')
+    #Remove even labels
+    for count, i in enumerate(ax.get_xticklabels()):
+        if count % 2 == 1:
+            i.set_visible(False)
+    #plt.show()
+    filePath = "/tmp/" + plotLabel.replace(" ", "_") + ".png"
+    print("Saving plot " + plotLabel + " to " + filePath)
+    plt.savefig(filePath)
+    plt.gcf().clear()
+
+def createSection(fp, value, sectionName, action, actionPerson, plotColor):
+    print(makeH2(sectionName), file=fp)
+    print("{} bugs have been {} by {} people.".format(
+        makeStrong(len(value["id"])), action,
+        makeStrong(len(value["author"]))), file=fp)
+
+    print(file=fp)
+    print(makeStrong("Top 10 " + actionPerson), file=fp)
+    print("<ol>", file=fp)
+    sortedList = sorted(value["author"].items(), key=lambda x: x[1], reverse=True)
+    itCount = 1
+    for item in sortedList:
+        if itCount > 10:
+            break
+        if action == 'fixed':
+            print(makeLI("{} ( {} )".format(item[0], item[1])), file=fp)
+        else:
+            print(makeLI("{} ( {} )".format(statList['people'][item[0]]['name'], item[1])), file=fp)
+        itCount += 1
+    print("</ol>", file=fp)
+    print(file=fp)
+    print('<img src="PATH_HERE/' + sectionName.replace(' ', '_') + \
+            '.png" alt="" width="640" height="480" class="alignnone size-full" />', file=fp)
+    print(file=fp)
+
+    createPlot(value['day'], "bar", sectionName + " Per Day", sectionName, plotColor)
+
+def createReport(statList):
+    fileName = '/tmp/blogReport.txt'
+    fp = open(fileName, 'w', encoding='utf-8')
+    print("creating Blog Report in " + fileName)
+    createSection(fp, statList['created'], "Reported Bugs", "reported", "Reporters", "red")
+    createSection(fp, statList['confirmed'], "Triaged Bugs", "triaged", "Triagers", "gold")
+    createSection(fp, statList['fixed'], "Fixed Bugs", "fixed", "Fixers", "darksalmon")
+    createSection(fp, statList['verified'], "Verified Bugs", "verified", "Verifiers", "palegreen")
+    createSection(fp, statList['metabug'], "Categorized Bugs", "categorized with a metabug", "Categorizers", "lightpink")
+    createSection(fp, statList['keywords']['bisected'], "Bisected Bugs", "bisected", "Bisecters", "orange")
+
+    print(makeH2("Evolution of Unconfirmed Bugs"), file=fp)
+    print(file=fp)
+    print('<img src="PATH_HERE/Unconfirmed_Bugs.png" alt="" width="640" height="480" class="alignnone size-full" />', file=fp)
+    print(file=fp)
+    createPlot(statList['unconfirmedCount'], "line", "Unconfirmed Bugs Per Day", "Unconfirmed Bugs", "blue")
+    fp.close()
+
+def mkdate(datestr):
+      try:
+        return datetime.strptime(datestr, '%Y-%m-%d')
+      except ValueError:
+        raise argparse.ArgumentTypeError(datestr + ' is not a proper date string')
+
+if __name__ == '__main__':
+    parser=argparse.ArgumentParser()
+    parser.add_argument('Date',type=mkdate, nargs=2, help="Introduce the starting date as first" + \
+            " argument and the ending date as second argument")
+    args=parser.parse_args()
+
+    print("Reading and writing data from " + common.dataDir)
+
+    bugzillaData = common.get_bugzilla()
+
+    statList = util_create_statList()
+
+    analyze_bugzilla_data(statList, bugzillaData, args)
+
+    createReport(statList)
+
+    print('End of report')


More information about the Libreoffice-commits mailing list