Xen Code Review Kibana-dashboard builder

This notebook aims at building a Kibana dashboard focused on the several aspects of interest for the Xen community.

This builds the following panels:

  • Time-focused panel at the level of patch series
  • People/Organizations panel at the level of patches/comments/reviews
  • Backlog/commits panel at the level of patch series

In [1]:
# Elastic search library
from elasticsearch import Elasticsearch
# python-mysql database access
import MySQLdb
import logging
import pandas
from ConfigParser import ConfigParser

In [2]:
config = ConfigParser()
config.read("settings")

args = {}
# There are two sections: mysql and elasticsearch
if config.has_section("mysql"):
    if config.has_option("mysql", "user") and \
        config.has_option("mysql", "password") and \
        config.has_option("mysql", "mlstats_db") and \
        config.has_option("mysql", "cvsanaly_db") and \
        config.has_option("mysql", "code_review_db"):
        args["mysql"] = dict(config.items("mysql"))

if config.has_section("elasticsearch"):
    if config.has_option("elasticsearch", "user") and \
       config.has_option("elasticsearch", "password") and \
       config.has_option("elasticsearch", "host") and \
       config.has_option("elasticsearch", "port") and \
       config.has_option("elasticsearch", "path"):
         args["elasticsearch"] = dict(config.items("elasticsearch"))

if not(args.has_key("mysql") and args.has_key("elasticsearch")):
    raise Exception("Section 'mysql' or section 'elasticsearch' not found in the 'settings' file")

In [4]:
def connect(args):
   user = args["mysql"]["user"]
   password = args["mysql"]["user"]
   host = "localhost"
   db = args["mysql"]["code_review_db"]

   try:
      db = MySQLdb.connect(user = user, passwd = password, db = db, charset='utf8')
      return db, db.cursor()
   except:
      logging.error("Database connection error")
      raise


def execute_query(connector, query):
   results = int (connector.execute(query))
   cont = 0
   if results > 0:
      result1 = connector.fetchall()
      return result1
   else:
      return []
    

db, cursor = connect(args)

In [5]:
# Insert data in ElasticSearch
def to_json(row, columns):
    # Function that translates from tuple to JSON doc
    doc = {}
    
    for column in columns:
       
        value = row[columns.index(column) + 1] 
       
        try:
            doc[column] = value
        except:
            doc[column] = ""
   
    return doc

Time focused panel

This panel plays with the following data:

  • Filters:
    • Patches per patch serie
    • Loops per patch serie
  • Evolutionary charts:
    • Time to merge
    • Time to commit
    • Time to re-work a patch serie
    • Cycle time
    • TIme to first review

In [6]:
# Calculating the list of patch series found in the database
# Basic info about a patch serie:
#   * Patch Serie id
#   * Patch Serie subject
#   * Patch Serie sender
#   * Patch Serie initial sending date
#   * Patch Serie number of patches
#   * Patch Serie number of versions
#   * Patch Serie number of comments
#   * Patch Serie participants (people commenting)
#   * Patch Serie time 2 merge
#   * Patch Serie merge time
query_patchserie = """ SELECT ps.id as patchserie_id,
                   ps.message_id as message_id,
                   ps.subject as patchserie_subject,
                   pe.email as patchserie_sender,
                   SUBSTRING_INDEX(pe.email, '@', -1) as sender_domain,
                   MIN(psv.date_utc) as patchserie_sent_date,
                   max(t1.patches) as patchserie_numpatches,
                   COUNT(DISTINCT(version)) as patchserie_versions,
                   count(distinct(c.id)) as patchserie_comments,
                   COUNT(DISTINCT(c.submitter_id)) as patchserie_commenters
            FROM patch_series ps,
                 patch_series_version psv,
                 patches p,
                 people pe,
                 comments c,
                 (SELECT psv.ps_id, 
                             p.ps_version_id, 
                             count(distinct(p.id)) as patches 
                  FROM patch_series_version psv, 
                           patches p 
                  WHERE psv.id = p.ps_version_id  
                  GROUP BY psv.ps_id, 
                           p.ps_version_id) t1
            WHERE ps.id = psv.ps_id AND
                  psv.id = p.ps_version_id AND
                  p.ps_version_id = t1.ps_version_id AND
                  p.id = c.patch_id AND
                  p.submitter_id = pe.id
            GROUP BY ps.id
            """


# Calculating the time to merge. Later merging result with previous query
query_time2merge = """ SELECT psv.ps_id as patch_serie,
                   TIMESTAMPDIFF(SECOND, MIN(psv.date_utc), MAX(c.committer_date_utc)) as time2merge,
                   MIN(psv.date_utc) as first_patch_date,
                   MAX(c.committer_date_utc) as merge_time
            FROM patch_series_version psv,
                 patches p,
                 commits c
            WHERE psv.id=p.ps_version_id AND 
                  p.commit_id = c.id 
            GROUP BY psv.ps_id; """

# Calculating the time to commit
query_time2commit = """ SELECT psv.ps_id as patch_serie,
                   TIMESTAMPDIFF(SECOND, MAX(c.date_utc), MAX(commits.committer_date_utc)) as time2commit,
                   MAX(c.date_utc) as last_comment_date, 
                   MAX(commits.committer_date_utc) as commit_time 
            FROM patch_series_version psv, 
                 patches p, 
                 comments c, 
                 commits 
            WHERE psv.id = p.ps_version_id AND 
                  p.commit_id = commits.id AND 
                  p.id=c.patch_id 
            GROUP BY psv.ps_id
            HAVING time2commit >= 0 and time2commit < 2000*3600*24 """

In [7]:
patchserie_data = execute_query(cursor, query_patchserie)
time2merge_data = execute_query(cursor, query_time2merge)
time2commit_data = execute_query(cursor, query_time2commit)

In [8]:
patchseries_list = list(patchserie_data)
time2merge_list = list(time2merge_data)
time2commit_list = list(time2commit_data)

In [9]:
patchseries_df = pandas.DataFrame(patchseries_list, columns=["patchserie_id", "message_id", "subject", "sender", "sender_domain", "sent_date", "num_patches", "num_versions", "num_comments", "num_commenters"])
time2merge_df = pandas.DataFrame(time2merge_list, columns=["patchserie_id", "time2merge", "sent_date", "mergetime"])
time2commit_df = pandas.DataFrame(time2commit_list, columns=["patchserie_id", "time2commit", "lastcommentdate", "committime"])

In [10]:
patchseries_df = pandas.merge(patchseries_df, time2merge_df, on="patchserie_id", how="left")
patchseries_df = pandas.merge(patchseries_df, time2commit_df, on="patchserie_id", how="left")

In [11]:
patchseries_df = patchseries_df.fillna(-1)
#patchseries_df

In [12]:
patchseries_df.columns


Out[12]:
Index([u'patchserie_id', u'message_id', u'subject', u'sender', u'sender_domain', u'sent_date_x', u'num_patches', u'num_versions', u'num_comments', u'num_commenters', u'time2merge', u'sent_date_y', u'mergetime', u'time2commit', u'lastcommentdate', u'committime'], dtype='object')

In [11]:
patchseries_df


Out[11]:
patchserie_id message_id subject sender sender_domain sent_date_x num_patches num_versions num_comments num_commenters time2merge sent_date_y mergetime time2commit lastcommentdate committime
0 6 <4B4344040200007800028412@vpn.id2.novell.com> linux/privcmd: fix for proper operation in com... JBeulich@novell.com novell.com 2010-01-05 12:52:04 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
1 8 <ED3036A092A28F4C91B0B4360DD128EA3059B708@shzs... xend: passthrough: also do_FLR when a device i... dexuan.cui@intel.com intel.com 2010-01-05 15:44:47 1 1 5 3 59553 2010-01-05 15:44:47 2010-01-06 08:17:20 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
2 9 <de76405a1001060419x714c215fo8b2cc1c134627ae4@... Handle PoD case in hvm_hap_nested_page_fault() dunlapg@umich.edu umich.edu 2010-01-06 12:19:37 1 1 1 1 1546 2010-01-06 12:19:37 2010-01-06 12:45:23 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
3 10 <4B44BBF30200007800028783@vpn.id2.novell.com> x86: fix NUMA handling (c/s 20599) JBeulich@novell.com novell.com 2010-01-06 15:36:03 1 1 3 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
4 12 <de76405a1001070713p543ed648p6e0469bade818ed7@... Per-cpu xentrace buffers dunlapg@umich.edu umich.edu 2010-01-07 15:13:48 1 1 4 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
5 22 <20100108082904.GA31397@ub-qhe2> xend: fix options for assigned pci qing.he@intel.com intel.com 2010-01-08 08:29:04 1 1 3 2 11402 2010-01-08 08:29:04 2010-01-08 11:39:06 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
6 26 <201001111809.38345.Christoph.Egger@amd.com> [TOOLS] blktap: make memshr optional Christoph.Egger@amd.com amd.com 2010-01-11 17:09:38 1 1 2 2 49903 2010-01-11 17:09:38 2010-01-12 07:01:21 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7 34 <4B4CA391.2040005@amd.com> numa: fix problems with memory-less nodes andre.przywara@amd.com amd.com 2010-01-12 16:30:09 1 1 4 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
8 42 <alpine.DEB.2.00.1001141718550.27920@kaball-de... implement xc_map_foreign_bulk for minios stefano.stabellini@eu.citrix.com eu.citrix.com 2010-01-14 17:33:03 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
9 44 <4B506D7F.1050500@amd.com> xend: Fix non-contiguous NUMA node assignment andre.przywara@amd.com amd.com 2010-01-15 13:28:31 1 1 5 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
10 45 <4B5096EB.4090901@oracle.com> Change default cpufreq governor to ondemand john.haxby@oracle.com oracle.com 2010-01-15 16:25:15 1 1 1 1 178915 2010-01-15 16:25:15 2010-01-17 18:07:10 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
11 47 <4B5443BC020000780002A81E@vpn.id2.novell.com> linux/blkfront: fixes for 'xm block-detach ...... JBeulich@novell.com novell.com 2010-01-18 10:19:24 1 1 2 2 17388571 2010-01-18 10:19:24 2010-08-07 16:28:55 13760510 2010-03-01 10:07:05 2010-08-07 16:28:55
12 52 <20100119095250.GB20691@joejin-pc.cn.oracle.com> Fix blkback/blktap sysfs read bug. joe.jin@oracle.com oracle.com 2010-01-19 09:52:50 1 1 16 4 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
13 53 <6CADD16F56BC954D8E28F3836FA7ED7112A9AB19BD@sh... Change vcpu_avail type to hex string when pass... dongxiao.xu@intel.com intel.com 2010-01-20 05:11:28 1 1 4 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
14 54 <20100121004658.GA28792@joejin-pc.cn.oracle.com> Fix blk{back,tap} sysfs race joe.jin@oracle.com oracle.com 2010-01-21 00:46:58 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
15 57 <60E426D47DE8EA47AA104E65008A100D14458756F3@sh... VT-d: improve RMRR validity checking weidong.han@intel.com intel.com 2010-01-21 02:46:12 1 1 75 10 193032 2010-01-21 02:46:12 2010-01-23 08:23:24 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
16 64 <6CADD16F56BC954D8E28F3836FA7ED711313B899A0@sh... [VT-d] Dis-allow PCI device assignment if PoD ... dongxiao.xu@intel.com intel.com 2010-01-21 12:28:13 1 1 3 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
17 66 <4B59CCB1.8050504@tycho.ncsc.mil> tools/ Exposes Flask XSM AVC functions to user... mbgrego@tycho.ncsc.mil tycho.ncsc.mil 2010-01-22 16:05:05 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
18 68 <ED3036A092A28F4C91B0B4360DD128EA35C4D5A1@shzs... x86: check if desc->action is NULL when unbind... dexuan.cui@intel.com intel.com 2010-01-22 08:01:44 1 1 2 2 10774 2010-01-22 08:01:44 2010-01-22 11:01:18 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
19 70 <7kockhxvc2.fsf@pingu.sky.yk.fujitsu.co.jp> xend: destroy restored domain when its device ... kouya@jp.fujitsu.com jp.fujitsu.com 2010-01-26 05:56:39 1 1 2 1 97388 2010-01-26 05:56:39 2010-01-27 08:59:47 25478 2010-01-27 01:55:09 2010-01-27 08:59:47
20 72 <EB8593BCECAB3D40A8248BE0B6400A3835A434DD@shzs... enhancement for unmaskable MSI and two cosmeti... xiantao.zhang@intel.com intel.com 2010-01-26 14:10:28 1 1 4 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
21 73 <EB8593BCECAB3D40A8248BE0B6400A3835A4384E@shzs... HVM vcpu hotplug: Fix acpi method NTFY bug jinsong.liu@intel.com intel.com 2010-01-27 05:33:00 1 1 11 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
22 74 <201001271417.51234.Christoph.Egger@amd.com> svm: Fix migration in the middle of a syscall Christoph.Egger@amd.com amd.com 2010-01-27 13:17:50 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
23 76 <C8EDE645B81E5141A8C6B2F73FD9265117C58A3367@sh... MCE: Handle the vMCA bank correctly yunhong.jiang@intel.com intel.com 2010-01-28 05:55:50 1 1 4 2 89494 2010-01-28 05:55:50 2010-01-29 06:47:24 13976 2010-01-29 02:54:28 2010-01-29 06:47:24
24 77 <C8EDE645B81E5141A8C6B2F73FD9265117C58A3365@sh... MCE: Not GP fault when guest write non 0s or ... yunhong.jiang@intel.com intel.com 2010-01-28 05:55:53 1 1 4 2 89527 2010-01-28 05:55:53 2010-01-29 06:48:00 14597 2010-01-29 02:44:43 2010-01-29 06:48:00
25 79 <C8EDE645B81E5141A8C6B2F73FD9265117C58A3368@sh... MCE: Fix the vMCE address translation for HVM ... yunhong.jiang@intel.com intel.com 2010-01-28 05:55:58 1 1 5 2 89595 2010-01-28 05:55:58 2010-01-29 06:49:13 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
26 83 <4B621D41.5030805@scsiguy.com> iommu-remote: Fix lost serial TX interrupts. ... gibbs@scsiguy.com scsiguy.com 2010-01-28 23:26:57 1 1 5 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
27 84 <C8EDE645B81E5141A8C6B2F73FD9265117C58A35F4@sh... DOM0: Add Machine check support to dom0 yunhong.jiang@intel.com intel.com 2010-01-28 10:13:57 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
28 85 <C8EDE645B81E5141A8C6B2F73FD9265117C58A35FD@sh... Dom0: Don't switch back to user space stack i... yunhong.jiang@intel.com intel.com 2010-01-28 10:26:17 1 1 6 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
29 86 <C7731B80-57B6-4E0D-86F9-824709736047@rice.edu> evtchn_do_upcall: Fix logic to start correctly... kaushik@rice.edu rice.edu 2010-01-29 05:34:42 1 1 4 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
7287 9566 <1455705264-17744-1-git-send-email-ian.campbel... xl: uptime: skip dom0 when calling print_domU_... ian.campbell@citrix.com citrix.com 2016-02-17 10:34:23 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7288 9569 <1455726686-23091-1-git-send-email-konrad.wilk... xen/blback: Fit the important information of t... konrad.wilk@oracle.com oracle.com 2016-02-17 16:31:26 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7289 9570 <1455717855-2344-1-git-send-email-ian.campbell... xl: free config_data on error in domain_create ian.campbell@citrix.com citrix.com 2016-02-17 14:04:13 1 1 7 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7290 9571 <1455719438-11095-1-git-send-email-ian.campbel... tools: gtracestat: Drop unused functions do_cs... ian.campbell@citrix.com citrix.com 2016-02-17 14:30:37 1 1 2 2 180772 2016-02-17 14:30:37 2016-02-19 16:43:29 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7291 9572 <1455720074-11321-1-git-send-email-ian.campbel... tools: remove xenpmd ian.campbell@citrix.com citrix.com 2016-02-17 14:41:14 1 1 3 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7292 9573 <1455721113-11540-1-git-send-email-ian.campbel... xenpaging: do not leak if --pagefile given twice ian.campbell@citrix.com citrix.com 2016-02-17 14:58:33 1 1 2 1 15831 2016-02-17 14:58:33 2016-02-17 19:22:24 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7293 9574 <1455723595-30904-1-git-send-email-ian.campbel... xenpaging: don't try to log via xch if xc_inte... ian.campbell@citrix.com citrix.com 2016-02-17 15:39:55 1 1 1 1 176715 2016-02-17 15:39:55 2016-02-19 16:45:10 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7294 9576 <1455738711-32328-1-git-send-email-andrew.coop... xen: Avoid left shifting into a sign bit andrew.cooper3@citrix.com citrix.com 2016-02-17 19:51:51 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7295 9577 <1455741740-11469-1-git-send-email-andrew.coop... [VERY RFC] Clang: Issues with .data.rel.ro rel... andrew.cooper3@citrix.com citrix.com 2016-02-17 20:42:20 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7296 9578 <1455764256-8700-1-git-send-email-cardoe@cardo... xen: add randconfig target to Makefile cardoe@cardoe.com cardoe.com 2016-02-18 02:57:35 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7297 9579 <1455793357-5712-1-git-send-email-ian.campbell... docs: document handling of metacharacter escap... ian.campbell@citrix.com citrix.com 2016-02-18 11:02:37 1 1 4 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7298 9580 <1455798403-21953-1-git-send-email-andrew.coop... xen/x86: Annotate deliberate fallthrough cases... andrew.cooper3@citrix.com citrix.com 2016-02-18 12:26:43 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7299 9581 <1455792328-28855-1-git-send-email-czuzu@bitde... x86/hvm_event: fix uninitialized struct field ... czuzu@bitdefender.com bitdefender.com 2016-02-18 10:45:28 1 1 6 3 12216 2016-02-18 10:45:28 2016-02-18 14:09:04 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7300 9582 <1455792823-29003-1-git-send-email-czuzu@bitde... x86/monitor: minor left-shift undefined behavi... czuzu@bitdefender.com bitdefender.com 2016-02-18 10:53:43 1 1 1 1 21233 2016-02-18 10:53:43 2016-02-18 16:47:36 9832 2016-02-18 14:03:44 2016-02-18 16:47:36
7301 9583 <1455806185-29539-1-git-send-email-andrew.coop... travis: Drop bridge-utils and iproute2 andrew.cooper3@citrix.com citrix.com 2016-02-18 14:36:25 1 1 1 1 7904 2016-02-18 14:36:25 2016-02-18 16:48:09 7083 2016-02-18 14:50:06 2016-02-18 16:48:09
7302 9584 <1455830545-848-1-git-send-email-cardoe@cardoe... tools/configure: only require bcc/ld86/as86 wh... cardoe@cardoe.com cardoe.com 2016-02-18 21:22:22 1 2 12 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7303 9586 <1455818622-30625-1-git-send-email-andrew.coop... xen/x86: Map Xen code/data/bss with superpages andrew.cooper3@citrix.com citrix.com 2016-02-18 18:03:38 1 1 10 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7304 9587 <1455824116-13783-1-git-send-email-czuzu@bitde... arm/monitor vm-events: Implement guest-request... czuzu@bitdefender.com bitdefender.com 2016-02-18 19:35:16 1 1 22 6 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7305 9588 <1455833409-19139-1-git-send-email-andrew.coop... hvmloader: Use xen/errno.h rather than the hos... andrew.cooper3@citrix.com citrix.com 2016-02-18 22:10:09 1 1 9 5 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7306 9589 <1455887316-9223-1-git-send-email-mcgrof@kerne... x86/init: replace paravirt_enabled() were poss... mcgrof@kernel.org kernel.org 2016-02-19 13:08:27 10 1 11 5 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7307 9591 <1455877704-44419-1-git-send-email-roger.pau@c... x86/PVHv2: add XEN_ prefix to HVM_START_MAGIC_... roger.pau@citrix.com citrix.com 2016-02-19 10:28:24 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7308 9592 <1455882291-1867-1-git-send-email-ian.campbell... MAINTAINERS: Drop myself as xen netback mainta... ian.campbell@citrix.com citrix.com 2016-02-19 11:44:51 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7309 9593 <1455911749-89579-1-git-send-email-cardoe@card... MAINTAINERS: add myself for Travis CI config cardoe@cardoe.com cardoe.com 2016-02-19 19:55:49 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7310 9599 <1455931646-5672-1-git-send-email-arei.gonglei... xen-netfront: set real_num_tx_queues to zreo a... arei.gonglei@huawei.com huawei.com 2016-02-20 01:27:26 1 1 3 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7311 9600 <1455982445-19054-1-git-send-email-cardoe@card... x86: CONFIG_X86 defined by Kconfig cardoe@cardoe.com cardoe.com 2016-02-20 15:34:04 1 1 2 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7312 9601 <1461418307-5845-1-git-send-email-wei.w.wang@i... Porting the intel_pstate driver to Xen JBeulich@suse.com suse.com 2016-04-23 13:31:47 1 1 26 4 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7313 9604 <1461419854-5968-1-git-send-email-wei.w.wang@i... x86/cpu_hotplug: add the unregister_cpu_notifi... wei.w.wang@intel.com intel.com 2016-05-13 07:50:00 1 1 3 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7314 9606 <1461419875-6050-1-git-send-email-wei.w.wang@i... x86/intel_pstate: relocate the driver register... wei.w.wang@intel.com intel.com 2016-05-13 07:50:36 1 1 5 2 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7315 9607 <1461419884-6091-1-git-send-email-wei.w.wang@i... x86/intel_pstate: the main boby of the intel_p... wei.w.wang@intel.com intel.com 2016-04-23 13:58:04 1 1 5 3 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999
7316 9610 <1461419914-6214-1-git-send-email-wei.w.wang@i... x86/intel_pstate: enable xenpm to control the ... wei.w.wang@intel.com intel.com 2016-05-13 07:51:45 1 1 1 1 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999 -1 1969-12-31 23:59:59.999999999 1969-12-31 23:59:59.999999999

7317 rows × 16 columns


In [13]:
user = args["elasticsearch"]["user"]
password = args["elasticsearch"]["password"]
host = args["elasticsearch"]["host"]
port = args["elasticsearch"]["port"]
path = args["elasticsearch"]["path"]
connection = "http://" + user + ":" + password + "@" + host + ":" + port + "/" + path

INDEX = 'xen-patchseries-timefocused'

# Building the ES connection
es = Elasticsearch([connection])
#es.indices.create(index=INDEX)

In [14]:
patchseries_df["time2commit"] = (patchseries_df["time2commit"] / 3600.0) / 24.0
patchseries_df["time2merge"] = (patchseries_df["time2merge"] / 3600.0) / 24.0

columns = patchseries_df.columns.values.tolist()
print columns
for row in patchseries_df.itertuples():
    # Let's insert into ES each tuple found in the dataset

    uniq_id = row[0]
    doc = to_json(row, columns)
    result = es.index(index=INDEX, doc_type='patchserie', id = uniq_id, body=doc)


['patchserie_id', 'message_id', 'subject', 'sender', 'sender_domain', 'sent_date_x', 'num_patches', 'num_versions', 'num_comments', 'num_commenters', 'time2merge', 'sent_date_y', 'mergetime', 'time2commit', 'lastcommentdate', 'committime']

Developer/Domain Focused-panel [Deprecated]

This panel is focused on the analysis of the developers, their reviewers, their domains, and the time to merge, time to first review and time to commit.


In [57]:
exit()
people_query = """ SELECT p.subject as patch_subject,
                          p.message_id as message_id,
                          p.date_utc as patch_sent_date,
                          pe.email as patch_sender,
                          SUBSTRING_INDEX(pe.email, '@', -1) as patch_sender_domain,
                          ps.id as patchserie_id,
                          ps.subject as patchserie_subject,
                          MIN(psv.date_utc) as patchserie_sent_date,
                          t1.email as patch_comment_sender,
                          t1.domain as patch_comment_domain,
                          t1.comment_date as patch_comment_date,
                          TIMESTAMPDIFF(SECOND, p.date_utc, t1.comment_date) as patch_time2comment
                    FROM patch_series ps,
                         patch_series_version psv,
                         patches p,
                         people pe,
                         (select p.id as patch_id,
                                 pe.email as email,
                                 SUBSTRING_INDEX(pe.email, '@', -1) as domain,
                                 c.date_utc as comment_date
                          from patches p,
                               comments c,
                               people pe
                          where c.patch_id = p.id and
                                c.submitter_id = pe.id) t1
                    WHERE ps.id = psv.ps_id AND
                          psv.id = p.ps_version_id AND
                          p.submitter_id = pe.id AND
                          p.id = t1.patch_id
                    GROUP BY p.id; """


#The issue is in the database. All of the TZ must be substracted. So the negative ones will be added (double substraction),
# while the positive ones will be simply substracted. More info at https://www.bitergia.net/redmine/issues/6699

In [58]:
people_data = execute_query(cursor, people_query)
people_list = list(people_data)
people_df = pandas.DataFrame(people_list, columns=["patch_subject", "patch_message_id", "patch_sent_date", "patch_sender", "patch_sender_domain", "patchserie_id", "patchserie_subject", "patchserie_sent_date", "patch_comment_sender", "patch_comment_domain", "patch_comment_date", "patch_time2comment"])

In [59]:
INDEX = 'xen-patchseries-peoplefocused'

# Building the ES connection
es = Elasticsearch()
# Creating the openstack-votes index
es.indices.create(index=INDEX)


Out[59]:
{u'acknowledged': True}

In [60]:
people_df["patch_time2comment"] = (people_df["patch_time2comment"] / 3600.0) / 24.0
columns = people_df.columns.values.tolist()
print columns
for row in people_df.itertuples():
    # Let's insert into ES each tuple found in the dataset

    uniq_id = row[0]
    doc = to_json(row, columns)
    result = es.index(index=INDEX, doc_type='patch', id = uniq_id, body=doc)


['patch_subject', 'patch_sent_date', 'patch_sender', 'patch_sender_domain', 'patchserie_id', 'patchserie_subject', 'patchserie_sent_date', 'patch_comment_sender', 'patch_comment_domain', 'patch_comment_date', 'patch_time2comment']

In [ ]: