In [1]:
import logging
reload(logging)
logging.basicConfig(
    format='%(asctime)-9s %(levelname)-8s: %(message)s',
    datefmt='%I:%M:%S')

# Enable logging at INFO level
logging.getLogger().setLevel(logging.INFO)
# Uncomment the following lines to enabled CGroups verbose logging
logging.getLogger('cgroups').setLevel(logging.INFO)
logging.getLogger('cgroups.cpuset').setLevel(logging.INFO)

In [2]:
import json
import operator

import devlib
import trappy
import bart

from bart.sched.SchedMultiAssert import SchedMultiAssert
from wlgen import RTA, Periodic

Target connection


In [3]:
import os
os.environ['ANDROID_HOME'] = '/ext/android-sdk-linux/'

In [4]:
from env import TestEnv

my_conf = {

#     # JUNO Linux
#     "platform"    : "linux",
#     "board"       : "juno",
#     "host"        : "192.168.0.1",
#     "username"    : "root",
#     "password"    : "",
#     "exclude_modules" : ['hwmon'],

    # JUNO Android
    "platform"    : "android",
    "board"       : "juno",
    "host"        : "192.168.0.1",
    "exclude_modules" : ['hwmon'],
    

    # RT-App calibration values
    "rtapp-calib" : {
        '0': 363, '1': 138, '2': 139, '3': 352, '4': 353, '5': 361
    },

    # List of additional devlib modules to install 
    "modules" : ['cgroups', 'bl', 'cpufreq'],
    
    # List of additional binary tools to install
    "tools" : ['rt-app', 'trace-cmd'],
    
    # FTrace events to collect
    "ftrace" : {
         "events" : [
             "sched_switch"
         ],
         "buffsize" : 10240
    }
}

te = TestEnv(my_conf)
target = te.target

# Report target connection
logging.info('Connected to %s target', target.abi)
print "DONE"


05:21:13  INFO    :         Target - Using base path: /home/derkling/Code/lisa
05:21:13  INFO    :         Target - Loading custom (inline) target configuration
05:21:13  INFO    :         Target - Devlib modules to load: ['bl', 'cpufreq', 'cgroups']
05:21:13  INFO    :         Target - Connecting Android target [192.168.0.1:5555]
05:21:16  INFO    :         Target - Initializing target workdir:
05:21:16  INFO    :         Target -    /data/local/tmp/devlib-target
05:21:21  INFO    :         Target - Topology:
05:21:21  INFO    :         Target -    [[0, 3, 4, 5], [1, 2]]
05:21:21  INFO    :       Platform - Loading default EM:
05:21:21  INFO    :       Platform -    /home/derkling/Code/lisa/libs/utils/platforms/juno.json
05:21:23  INFO    :         FTrace - Enabled tracepoints:
05:21:23  INFO    :         FTrace -   sched_switch
05:21:23  INFO    :    EnergyMeter - HWMON module not enabled
05:21:23  WARNING :    EnergyMeter - Energy sampling disabled by configuration
05:21:23  WARNING :         Target - Using configuration provided RTApp calibration
05:21:23  INFO    :         Target - Using RT-App calibration values:
05:21:23  INFO    :         Target -    {"0": 363, "1": 138, "2": 139, "3": 352, "4": 353, "5": 361}
05:21:23  INFO    :        TestEnv - Set results folder to:
05:21:23  INFO    :        TestEnv -    /home/derkling/Code/lisa/results/20160428_172123
05:21:23  INFO    :        TestEnv - Experiment results available also in:
05:21:23  INFO    :        TestEnv -    /home/derkling/Code/lisa/results_latest
05:21:23  INFO    : Connected to arm64 target
DONE

List available Controllers


In [5]:
logging.info('%14s - Available controllers:', 'CGroup')
ssys = target.cgroups.list_subsystems()
for (n,h,g,e) in ssys:
    logging.info('%14s -    %10s (hierarchy id: %d) has %d cgroups',
                 'CGroup', n, h, g)


05:21:25  INFO    :         CGroup - Available controllers:
05:21:25  INFO    :         CGroup -        cpuset (hierarchy id: 3) has 6 cgroups
05:21:25  INFO    :         CGroup -           cpu (hierarchy id: 2) has 3 cgroups
05:21:25  INFO    :         CGroup -       cpuacct (hierarchy id: 1) has 37 cgroups
05:21:25  INFO    :         CGroup -     schedtune (hierarchy id: 4) has 1 cgroups
05:21:25  INFO    :         CGroup -       freezer (hierarchy id: 5) has 1 cgroups
05:21:25  INFO    :         CGroup -         debug (hierarchy id: 6) has 1 cgroups

Example of CPUSET controller usage


In [6]:
# Get a reference to the CPUSet controller
cpuset = target.cgroups.controller('cpuset')

In [7]:
# Get the list of current configured CGroups for that controller
cgroups = cpuset.list_all()
logging.info('Existing CGropups:')
for cg in cgroups:
    logging.info('  %s', cg)


05:21:27  INFO    : Existing CGropups:
05:21:27  INFO    :   /
05:21:27  INFO    :   /LITTLE
05:21:27  INFO    :   /system-background
05:21:27  INFO    :   /background
05:21:27  INFO    :   /foreground
05:21:27  INFO    :   /foreground/boost

In [8]:
# Dump the configuraiton of each controller
for cgname in cgroups:
    #print cgname
    cgroup = cpuset.cgroup(cgname)
    attrs = cgroup.get()
    #print attrs
    cpus = attrs['cpus']
    logging.info('%s:%-15s cpus: %s', cpuset.kind, cgroup.name, cpus)


05:21:28  INFO    : cpuset:/               cpus: 0-5
05:21:28  INFO    : cpuset:/LITTLE         cpus: 0,3-5
05:21:29  INFO    : cpuset:/system-background cpus: 0,3-5
05:21:29  INFO    : cpuset:/background     cpus: 0
05:21:29  INFO    : cpuset:/foreground     cpus: 0-5
05:21:29  INFO    : cpuset:/foreground/boost cpus: 1-2

In [9]:
# Create a LITTLE partition
cpuset_littles = cpuset.cgroup('/LITTLE')

In [10]:
# Check the attributes available for this control group
print "LITTLE:\n", json.dumps(cpuset_littles.get(), indent=4)


LITTLE:
{
    "/sys/fs/cgroup/devlib_cpuset/LITTLE/cpuset.*": " No such file or directory", 
    "memory_pressure": "0", 
    "memory_spread_page": "0", 
    "notify_on_release": "0", 
    "sched_load_balance": "1", 
    "cpus": "0,3-5", 
    "effective_mems": "0", 
    "mem_hardwall": "0", 
    "cpu_exclusive": "0", 
    "mem_exclusive": "0", 
    "effective_cpus": "0,3-5", 
    "mems": "0", 
    "memory_migrate": "0", 
    "sched_relax_domain_level": "-1", 
    "memory_spread_slab": "0"
}

In [11]:
# Tune CPUs and MEMs attributes
#   they must be initialize for the group to be usable
cpuset_littles.set(cpus=target.bl.littles, mems=0)
print "LITTLE:\n", json.dumps(cpuset_littles.get(), indent=4)


LITTLE:
{
    "/sys/fs/cgroup/devlib_cpuset/LITTLE/cpuset.*": " No such file or directory", 
    "memory_pressure": "0", 
    "memory_spread_page": "0", 
    "notify_on_release": "0", 
    "sched_load_balance": "1", 
    "cpus": "0,3-5", 
    "effective_mems": "0", 
    "mem_hardwall": "0", 
    "cpu_exclusive": "0", 
    "mem_exclusive": "0", 
    "effective_cpus": "0,3-5", 
    "mems": "0", 
    "memory_migrate": "0", 
    "sched_relax_domain_level": "-1", 
    "memory_spread_slab": "0"
}

In [12]:
# Define a periodic big (80%) task
task = Periodic(
    period_ms=100,
    duty_cycle_pct=80,
    duration_s=5).get()

# Create one task per each CPU in the target
tasks={}
for tid in enumerate(target.core_names):
    tasks['task{}'.format(tid[0])] = task

# Configure RTA to run all these tasks
rtapp = RTA(target, 'simple', calibration=te.calibration())
rtapp.conf(kind='profile', params=tasks, run_dir=target.working_directory);


05:21:34  INFO    :          WlGen - Setup new workload simple
05:21:34  INFO    :          RTApp - Workload duration defined by longest task
05:21:34  INFO    :          RTApp - Default policy: SCHED_OTHER
05:21:34  INFO    :          RTApp - ------------------------
05:21:34  INFO    :          RTApp - task [task0], sched: using default policy
05:21:34  INFO    :          RTApp -  | calibration CPU: 1
05:21:34  INFO    :          RTApp -  | loops count: 1
05:21:34  INFO    :          RTApp - + phase_000001: duration 5.000000 [s] (50 loops)
05:21:34  INFO    :          RTApp - |  period   100000 [us], duty_cycle  80 %
05:21:34  INFO    :          RTApp - |  run_time  80000 [us], sleep_time  20000 [us]
05:21:34  INFO    :          RTApp - ------------------------
05:21:34  INFO    :          RTApp - task [task1], sched: using default policy
05:21:34  INFO    :          RTApp -  | calibration CPU: 1
05:21:34  INFO    :          RTApp -  | loops count: 1
05:21:34  INFO    :          RTApp - + phase_000001: duration 5.000000 [s] (50 loops)
05:21:34  INFO    :          RTApp - |  period   100000 [us], duty_cycle  80 %
05:21:34  INFO    :          RTApp - |  run_time  80000 [us], sleep_time  20000 [us]
05:21:34  INFO    :          RTApp - ------------------------
05:21:34  INFO    :          RTApp - task [task2], sched: using default policy
05:21:34  INFO    :          RTApp -  | calibration CPU: 1
05:21:34  INFO    :          RTApp -  | loops count: 1
05:21:34  INFO    :          RTApp - + phase_000001: duration 5.000000 [s] (50 loops)
05:21:34  INFO    :          RTApp - |  period   100000 [us], duty_cycle  80 %
05:21:34  INFO    :          RTApp - |  run_time  80000 [us], sleep_time  20000 [us]
05:21:34  INFO    :          RTApp - ------------------------
05:21:34  INFO    :          RTApp - task [task3], sched: using default policy
05:21:34  INFO    :          RTApp -  | calibration CPU: 1
05:21:34  INFO    :          RTApp -  | loops count: 1
05:21:34  INFO    :          RTApp - + phase_000001: duration 5.000000 [s] (50 loops)
05:21:34  INFO    :          RTApp - |  period   100000 [us], duty_cycle  80 %
05:21:34  INFO    :          RTApp - |  run_time  80000 [us], sleep_time  20000 [us]
05:21:34  INFO    :          RTApp - ------------------------
05:21:34  INFO    :          RTApp - task [task4], sched: using default policy
05:21:34  INFO    :          RTApp -  | calibration CPU: 1
05:21:34  INFO    :          RTApp -  | loops count: 1
05:21:34  INFO    :          RTApp - + phase_000001: duration 5.000000 [s] (50 loops)
05:21:34  INFO    :          RTApp - |  period   100000 [us], duty_cycle  80 %
05:21:34  INFO    :          RTApp - |  run_time  80000 [us], sleep_time  20000 [us]
05:21:34  INFO    :          RTApp - ------------------------
05:21:34  INFO    :          RTApp - task [task5], sched: using default policy
05:21:34  INFO    :          RTApp -  | calibration CPU: 1
05:21:34  INFO    :          RTApp -  | loops count: 1
05:21:34  INFO    :          RTApp - + phase_000001: duration 5.000000 [s] (50 loops)
05:21:34  INFO    :          RTApp - |  period   100000 [us], duty_cycle  80 %
05:21:34  INFO    :          RTApp - |  run_time  80000 [us], sleep_time  20000 [us]

In [13]:
# Test execution of all these tasks into the LITTLE cluster
trace = rtapp.run(ftrace=te.ftrace, cgroup=cpuset_littles.name, out_dir=te.res_dir)


05:21:42  INFO    :          WlGen - Workload execution START:
05:21:42  INFO    :          WlGen -    cgroups_run_into /LITTLE '/data/local/tmp/bin/rt-app /data/local/tmp/devlib-target/simple_00.json'
05:21:58  INFO    :          WlGen - Pulling trace file into [/home/derkling/Code/lisa/results/20160428_172123/simple_00.dat]...

In [14]:
# Check tasks residency on little clsuter
trappy.plotter.plot_trace(trace)



In [15]:
# Compute and visualize tasks residencies on LITTLE clusterh CPUs
s = SchedMultiAssert(trappy.Run(trace), te.topology, execnames="task")
residencies = s.getResidency('cluster', target.bl.littles, percent=True)
print json.dumps(residencies, indent=4)


{
    "2903": {
        "residency": 100.0, 
        "task_name": "rt-app"
    }, 
    "2904": {
        "residency": 100.0, 
        "task_name": "rt-app"
    }, 
    "2905": {
        "residency": 100.0, 
        "task_name": "rt-app"
    }, 
    "2906": {
        "residency": 100.0, 
        "task_name": "rt-app"
    }, 
    "2907": {
        "residency": 100.0, 
        "task_name": "rt-app"
    }, 
    "2908": {
        "residency": 100.0, 
        "task_name": "rt-app"
    }
}
/home/derkling/Code/lisa/libs/trappy/trappy/__init__.py:50: UserWarning: The Run object is deprecated.  Use trappy.FTrace instead
  warnings.warn("The Run object is deprecated.  Use trappy.FTrace instead")

In [16]:
# Assert that ALL tasks have always executed only on LITTLE cluster
s.assertResidency('cluster', target.bl.littles,
                  99.9, operator.ge, percent=True, rank=len(residencies))


Out[16]:
True

Example of CPU controller usage


In [17]:
# Get a reference to the CPU controller
cpu = target.cgroups.controller('cpu')

In [18]:
# Create a big partition on that CPUS
cpu_littles = cpu.cgroup('/LITTLE')

In [19]:
# Check the attributes available for this control group
print "LITTLE:\n", json.dumps(cpu_littles.get(), indent=4)


LITTLE:
{
    "rt_period_us": "1000000", 
    "shares": "512", 
    "rt_runtime_us": "0"
}

In [20]:
# Set a 1CPU equivalent bandwidth for that CGroup
# cpu_littles.set(cfs_period_us=100000, cfs_quota_us=50000)
cpu_littles.set(shares=512)
print "LITTLE:\n", json.dumps(cpu_littles.get(), indent=4)


LITTLE:
{
    "rt_period_us": "1000000", 
    "shares": "512", 
    "rt_runtime_us": "0"
}

In [21]:
# Test execution of all these tasks into the LITTLE cluster
trace = rtapp.run(ftrace=te.ftrace, cgroup=cpu_littles.name)


05:22:19  INFO    :          WlGen - Workload execution START:
05:22:19  INFO    :          WlGen -    cgroups_run_into /LITTLE '/data/local/tmp/bin/rt-app /data/local/tmp/devlib-target/simple_00.json'
05:22:35  INFO    :          WlGen - Pulling trace file into [.//simple_00.dat]...

In [22]:
# Check tasks residency on little clsuter
trappy.plotter.plot_trace(trace)