In [ ]:

    

import sqlite3
import pandas
import json
import re
import os
import math
import glob
from qumulo.rest_client import RestClient
from IPython.core.display import display, HTML, Javascript, display_javascript


    

In [ ]:

    

%%javascript

requirejs.config({
    paths: {
        'd3': 'https://cdnjs.cloudflare.com/ajax/libs/d3/4.7.4/d3',
        'd3-color': 'https://d3js.org/d3-color.v1.min',
        'd3-interpolate': 'https://d3js.org/d3-interpolate.v1.min',
        'd3-chromatic': 'https://d3js.org/d3-scale-chromatic.v1.min'
    }
});


require(['d3', 'd3-color', 'd3-interpolate', 'd3-chromatic'], function(d3, d3_color, d3_interpolate, d3_chromatic) {
    window.d3 = Object.assign({}, d3, d3_color, d3_interpolate, d3_chromatic);
});


    

In [ ]:

    

def read_dir(path, level, root_cap, thresh=0.001):
    try:
        the_dir = rc.fs.read_dir_aggregates(path=path)
    except:
        return
    if float(the_dir['total_capacity']) / root_cap > thresh:
        yield {"path": path, 
               "level": level, 
               "cap": the_dir['total_capacity'],
               "cap_perc": float(the_dir['total_capacity']) / root_cap
              }
    for d in the_dir['files']:
        if (float(d['capacity_usage']) / root_cap) > thresh and d['type'] == 'FS_FILE_TYPE_DIRECTORY':
            for d in read_dir(re.sub('//', '/', path + '/') + d['name'], level+1, root_cap):
                yield d
        elif (float(d['capacity_usage']) / root_cap) > thresh:
            file_d = {"path": re.sub('//', '/', path + '/') + d['name'], 
                   "level": level, 
                   "cap": d['capacity_usage'],
                   "cap_perc": float(d['capacity_usage']) / root_cap
                  }
            yield file_d


    

In [ ]:

    

def delete_tree(path):
    # print("deleting: %s" % (path, ))
    # out = rc.fs.delete_tree(path)
    return "not actually deleting: %s" % (path, )


    

In [ ]:

    

conf = json.loads(open('/mnt/product/qumulo-historical-data/config.json').read())
clusters = {}
for cl in conf['clusters']:
    clusters[cl['cluster']] = cl


    

In [ ]:

    

def get_all_data(cluster_name, start_dir='/'):
    global rc

    cluster_conf = clusters[cluster_name]
    rc = RestClient(cluster_name, 8000)
    rc.login(cluster_conf['user'], cluster_conf['pass'])

    dfs = []

    for db_file in glob.glob('/mnt/product/qumulo-historical-data/data/%s/*' % (cluster_name,)):
        print("Read sqlite db file: %s" % (db_file, ))
        cn = sqlite3.connect(db_file)
        try:
            dfs.append(pandas.read_sql('SELECT * FROM iops_tput_path_hour', cn))
        except:
            print("table doesn't exist")

    df_activity = pandas.concat(dfs).groupby(['path']) \
        .agg({'total_iops':'sum', 
              'total_data':'sum',
              'read_data':'sum',
              'write_data':'sum'}) \
        .reset_index()
    df_activity.set_index(['path'], inplace=True)

    # this does a pretty large tree walk, with no parallelization. Can take a few minutes on large clusters.

    root_dir = rc.fs.read_dir_aggregates(start_dir)
    root_cap = int(root_dir['total_capacity'])
    print("Begin the recursive walking.")
    df_capacity = pandas.DataFrame(read_dir(start_dir, 1, root_cap))
    print("Completed the recursive walking.")

    df = pandas.merge(df_capacity, df_activity.reset_index(), how='left').fillna(0)
    df.to_csv('capacity.csv')

    # df[(df['cap_perc'] > 0.01) & (df['write_data'] <= 0) & (df['level'] <= 4)].sort_values(['cap_perc'], ascending=False)


    

In [ ]:

    

%%html

<style>

svg{
    shape-rendering: crispEdges;
}

tspan {
    font-size: 10px;
}

.node rect{
    stroke: rgba(255, 255, 255, 0.2);
    stroke-width: 1;
}

.node.node--hover rect{
    stroke: rgba(255, 255, 255, 0.9);
    stroke-width: 1;
}

#id2{
    position: absolute;
    top: 12px;
    left: 12px;
    border: 1px solid #555;
    z-index: 100;
    background-color: white;
    -webkit-box-shadow: 0px 9px 29px 0px rgba(66,66,66,1);
    -moz-box-shadow: 0px 9px 29px 0px rgba(66,66,66,1);
    box-shadow: 0px 9px 29px 0px rgba(66,66,66,1);
}

#close_button{
    position: absolute;
    top: 10px;
    right: 10px;
    background-color: #cccccc;
    font-weight: bold;
    border: 1px solid #333333;
    border-radius: 18px;
    height: 36px;
    width: 36px;
    font-size: 24px;
    z-index: 101;
    text-align: center;
    cursor: pointer;
}

text.light{
    fill: white;
}
</style>


    

In [ ]:

    

%%javascript

window.draw_treemap = function(the_metric){
    var w = jQuery(window).width()-24;
    var h = jQuery(window).height()-24;
    var svg = d3.select("body").append("svg").attr("id", "id2")
    svg.attr("width", w).attr("height", h);

    var max_depth = 5;
    svg.selectAll("*").remove();

    jQuery(document).keypress(function(e) {
        console.log(e.which);
        if(e.which == 92){
            jQuery("svg#id2").remove();
        }
    })
    
    console.clear();
    d3.csv('capacity.csv', function(data){
        data = data.filter(function(d){return d.level <= max_depth})
        data.forEach(function(d){
            d.path = '[root]' + (d.path =='/'?'':d.path);
            d.cap = +d.cap;
            d.cap_perc = +d.cap_perc;
            d.level = +d.level;
            d.read_data = +d.read_data;
            d.write_data = +d.write_data;
            d.total_iops = +d.total_iops;
            d.total_data = +d.total_data;
        });
        var stratify = d3.stratify()
            .id(function(d) { return d.path; })
            .parentId(function(d) { return d.path.substring(0, d.path.lastIndexOf("/")); });

        var root = stratify(data)

        root.eachBefore(function(d){
            if('children' in d){
                d.children.forEach(function(child){
                    d.data.cap_perc -= child.data.cap_perc;
                })
            }
        })

        root
            .sum(function(d) { return d.cap_perc; })
            .sort(function(a, b) { return b.height - a.height || b.value - a.value; });

        var offset = 3;

        var close_button = d3.select("body").append("div")
            .attr("id", "close_button")
            .text("X")

        var svg = d3.select("svg"),
            width = +svg.attr("width"),
            height = +svg.attr("height");

        var format = function(dd){
            if(dd >= 0.0001){
                return (dd*100).toFixed("2")
            }else{
                return ""
            }
        };

        var domain = [0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 10, 50];
        var range = [d3.interpolateRdYlBu(0.99), 
                     d3.interpolateRdYlBu(0.8),
                     d3.interpolateRdYlBu(0.6),
                     d3.interpolateRdYlBu(0.4),
                     d3.interpolateRdYlBu(0.3),
                     d3.interpolateRdYlBu(0.2),
                     d3.interpolateRdYlBu(0.01),
                    ]
        var color = d3.scaleLinear().domain(domain).range(range);

        var stratify = d3.stratify()
            .parentId(function(d) { return d.id.substring(0, d.id.lastIndexOf("/")); });

        var treemap = d3.treemap()
            .tile(d3.treemapResquarify.ratio(1.6))
            .size([w, h])
            .paddingInner(0)
            .paddingOuter(offset)
            .paddingTop(function(d) { return d.depth < 4 ? 19 : offset; })
            .round(true);

        treemap(root);

        var cell = svg
        .selectAll(".node")
        .data(root.descendants())
        .enter().append("g")
          .attr("transform", function(d) { return "translate(" + d.x0 + "," + d.y0 + ")"; })
          .attr("class", "node")
          .each(function(d) { d.node = this; })
          .on("mouseover", hovered(true))
          .on("click", clicked())
          .on("mouseout", hovered(false));

        cell.append("rect")
          .attr("id", function(d) { return "rect-" + d.id; })
          .attr("width", function(d) { return d.x1 - d.x0; })
          .attr("height", function(d) { return d.y1 - d.y0; })
          .style("fill", function(d) { 
              // if(d.data.total_data == 0){
              //    return "rgb(80, 120, 240)"
              // }
              return color(d.data[the_metric] / d.data.cap);
          })
          .style("opacity", function(d){
            return 1.0;
//             if(d.data.total_data > 0){
//                 return 0.1;
//             }else{
//                 return 1.0;
//             }
          })
            ;

        cell.append("clipPath")
          .attr("id", function(d) { return "clip-" + d.id; })
        .append("use")
          .attr("xlink:href", function(d) { return "#rect-" + d.id + ""; });

        var label = cell.append("text")
          .attr("class", function(d){ return (d.data[the_metric] / d.data.cap < 0.0001?"light":"dark")})
          .attr("clip-path", function(d) { return "url(#clip-" + d.id + ")"; })

        label
        .filter(function(d) { return d.children; })
        .selectAll("tspan")
          .data(function(d) { return d.id.substring(d.id.lastIndexOf("/") + 1).split(/(?=[A-Z][^A-Z])/g).concat("\xa0" + format(d.data[the_metric]/d.data.cap) + "\xa0" + (d.data.cap/1000000000000).toFixed(2)+"TB"); })
        .enter().append("tspan")
          .attr("x", function(d, i) { return i ? null : 4; })
          .attr("y", 13)
          .text(function(d) { return d.replace("[root]/", "/"); })

        label
        .filter(function(d) { return !d.children; })
        .selectAll("tspan")
          .data(function(d) { return d.id.substring(d.id.lastIndexOf("/") + 1).split(/(?=[A-Z][^A-Z])/g).concat(format(d.data[the_metric]/d.data.cap) + "\xa0" + (d.data.cap/1000000000000).toFixed(2)+"TB"); })
        .enter().append("tspan")
          .attr("x", 4)
          .attr("y", function(d, i) { return 13 + i * 10; })
          .text(function(d) { return d.replace("[root]/", "/"); })

        cell.append("title")
          .text(function(d) { return d.id + "\n" + format(d.data[the_metric]/d.data.cap) + "\n" + (d.data.cap/1000000000000).toFixed(2)+"TB"; });

        jQuery("#close_button").click(function(e){
        jQuery("svg#id2").remove();
        setTimeout(function(){
            jQuery("#close_button").remove();
        }, 100)
        })

    });
}

function hovered(hover) {
  return function(d) {
    d3.selectAll(d.ancestors().map(function(d) { return d.node; }))
        .classed("node--hover", hover)
      .select("rect")
        .attr("width", function(d) { return d.x1 - d.x0 - hover; })
        .attr("height", function(d) { return d.y1 - d.y0 - hover; });
  };
}

function handle_output(out_obj, out){
    console.log("Command completed!")
    console.log(out_obj.content.data['text/plain'])
}

function clicked(){
    return function(d){
        prompt("Tree delete qq command", "qq fs_delete_tree --path \"" + d.id.replace("[root]", "") + "\"")
        var kernel = IPython.notebook.kernel;
        var callbacks = {iopub : {'output' : handle_output}};
        kernel.execute('del_path = "' + d.id.replace("[root]", "") + '"');
        kernel.execute('delete_tree(del_path)', callbacks, {silent:false});
    }
}


    

In [ ]:

    

get_all_data('gravytrain')
jso = Javascript("draw_treemap('write_data')")
display_javascript(jso)


    

In [ ]: