ChromeOS_TestThat


Run Tests on ChromeOS using test_that

This notebook shows how to run tests on a Chromebook using test_that. At the end of the test results are collected as a dictionary as well as trace events if you specify so in the target configuration.

NOTE: if you want to receive a token from the benchmark before starting trace collection and power measurement, you need to modify the benchmark such that a specific UDP packet is sent to the host machine. As a reference, consider the following python script and convert it to the language in which the benchmark is written accordingly:

import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto("POWER", (<TARGET_IP>, 1234))
sock.close()

You should put this right before the instruction that starts the execution of the actual workload in the benchmark.

In case of Acquarium, you should put this in

<CROS_PATH>/trunk/src/third_party/autotest/files/client/site_tests/graphics_WebGLAquarium/graphics_WebGLAcquarium.py:run_fish_test()

Being CROS_PATH the ChromeOS chroot.


In [1]:
import logging
from conf import LisaLogging
LisaLogging.setup()

In [2]:
import getpass
from subprocess import Popen, PIPE

import os
import pandas as pd
import scipy.integrate as integrate
import re
import json

# Support to access the remote target
import devlib
from env import TestEnv

import trappy

import socket

from time import sleep

import netifaces as ni

Target Configuration


In [3]:
# Define your path to the ChromeOS installation folder
CROS_BASE = "/data/chromiumos"

In [4]:
# Setup a target configuration
my_conf = {
    
    # Target platform and board
    "platform"    : 'linux',
    
    # Target board IP/MAC address
    "host"        : '192.168.0.1',
    
    # Login credentials
    "username"    : 'root',
    "password"    : 'test0000',
    
    # Tools required by the experiments
    "tools"   : [ 'trace-cmd' ], 
    
    # FTrace events to collect for all the tests configuration which have
    # the "ftrace" flag enabled
    "ftrace"  : {
        "events" : [
           "cpu_frequency",
           "cpu_idle",
           "sched_switch"
        ],
        "buffsize" : 10 * 1024,
    },
}

In [5]:
# Initialize a test environment using:
# the provided target configuration (my_conf)
te = TestEnv(my_conf)
target = te.target


2016-04-27 11:06:35,781 INFO    :         Target - Using base path: /home/pippo/work/lisa
2016-04-27 11:06:35,782 INFO    :         Target - Loading custom (inline) target configuration
2016-04-27 11:06:35,782 DEBUG   :         Target - Target configuration {'username': 'root', 'ftrace': {'buffsize': 10240, 'events': ['cpu_frequency', 'cpu_idle', 'sched_switch']}, 'platform': 'linux', 'host': '192.168.0.1', 'board': 'oak', 'password': 'test0000', 'tools': ['trace-cmd']}
2016-04-27 11:06:35,783 INFO    :         Target - Devlib modules to load: ['bl', 'cpufreq']
2016-04-27 11:06:35,783 INFO    :         Target - Connecting linux target:
2016-04-27 11:06:35,784 INFO    :         Target -   username : root
2016-04-27 11:06:35,785 INFO    :         Target -       host : 192.168.0.1
2016-04-27 11:06:35,785 INFO    :         Target -   password : test0000
2016-04-27 11:06:35,786 DEBUG   :         Target - Setup LINUX target...
2016-04-27 11:06:35,795 DEBUG   : Logging in root@192.168.0.1
2016-04-27 11:06:37,318 DEBUG   : id
2016-04-27 11:06:37,727 DEBUG   : if [ -e '/root/devlib-target/bin' ]; then echo 1; else echo 0; fi
2016-04-27 11:06:38,133 DEBUG   : ls -1 /root/devlib-target/bin
2016-04-27 11:06:38,541 DEBUG   : cat /proc/cpuinfo
2016-04-27 11:06:38,851 DEBUG   : Installing module bl
2016-04-27 11:06:38,952 DEBUG   : /root/devlib-target/bin/busybox uname -m
2016-04-27 11:06:39,359 DEBUG   : if [ -e '/sys/devices/system/cpu/cpufreq' ]; then echo 1; else echo 0; fi
2016-04-27 11:06:39,665 DEBUG   : Installing module cpufreq
2016-04-27 11:06:39,666 DEBUG   :         Target - Checking target connection...
2016-04-27 11:06:39,666 DEBUG   :         Target - Target info:
2016-04-27 11:06:39,667 DEBUG   :         Target -       ABI: arm64
2016-04-27 11:06:39,668 DEBUG   :         Target -      CPUs: CpuInfo(['A53', 'A53', 'A72', 'A72'])
2016-04-27 11:06:39,668 DEBUG   :         Target -  Clusters: [0, 0, 1, 1]
2016-04-27 11:06:39,770 DEBUG   : sudo -- sh -c 'mount -o remount,rw /'
2016-04-27 11:06:40,574 INFO    :         Target - Initializing target workdir:
2016-04-27 11:06:40,575 INFO    :         Target -    /root/devlib-target
2016-04-27 11:06:40,676 DEBUG   : mkdir -p /root/devlib-target
2016-04-27 11:06:41,085 DEBUG   : mkdir -p /root/devlib-target/bin
2016-04-27 11:06:41,393 DEBUG   : /usr/bin/scp -r   /home/pippo/work/lisa/libs/devlib/devlib/bin/arm64/busybox root@192.168.0.1:/root/devlib-target/bin/busybox
2016-04-27 11:06:41,804 DEBUG   : chmod a+x /root/devlib-target/bin/busybox
2016-04-27 11:06:42,112 DEBUG   : /usr/bin/scp -r   /home/pippo/work/lisa/libs/devlib/devlib/bin/scripts/shutils root@192.168.0.1:/root/devlib-target/bin/shutils
2016-04-27 11:06:42,360 DEBUG   : chmod a+x /root/devlib-target/bin/shutils
2016-04-27 11:06:42,668 DEBUG   : /usr/bin/scp -r   /home/pippo/work/lisa/tools/arm64/trace-cmd root@192.168.0.1:/root/devlib-target/bin/trace-cmd
2016-04-27 11:06:42,976 DEBUG   : chmod a+x /root/devlib-target/bin/trace-cmd
2016-04-27 11:06:43,285 DEBUG   :         Target - Check for module [bl]...
2016-04-27 11:06:43,286 DEBUG   :         Target - Check for module [cpufreq]...
2016-04-27 11:06:43,287 INFO    :         Target - Topology:
2016-04-27 11:06:43,287 INFO    :         Target -    [[0, 1], [2, 3]]
2016-04-27 11:06:43,389 DEBUG   : sudo -- sh -c 'cat '\''/sys/devices/system/cpu/online'\'''
2016-04-27 11:06:44,293 DEBUG   : cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies
2016-04-27 11:06:44,702 DEBUG   : sudo -- sh -c 'cat '\''/sys/devices/system/cpu/online'\'''
2016-04-27 11:06:45,606 DEBUG   : cat /sys/devices/system/cpu/cpu2/cpufreq/scaling_available_frequencies
2016-04-27 11:06:45,914 DEBUG   :       Platform - Trying to load default EM from /home/pippo/work/lisa/libs/utils/platforms/oak.json
2016-04-27 11:06:45,915 DEBUG   :       Platform - Platform descriptor initialized
{'nrg_model': None, 'clusters': {'big': [2, 3], 'little': [0, 1]}, 'cpus_count': 4, 'freqs': {'big': [507000, 702000, 1001000, 1209000, 1404000, 1612000, 1807000, 2106000], 'little': [507000, 702000, 1001000, 1105000, 1209000, 1300000, 1508000, 1703000]}, 'topology': [[0, 1], [2, 3]]}
2016-04-27 11:06:45,932 DEBUG   : /usr/bin/scp -r   /home/pippo/work/lisa/libs/devlib/devlib/bin/arm64/trace-cmd root@192.168.0.1:/root/devlib-target/bin/trace-cmd
2016-04-27 11:06:46,296 DEBUG   : chmod a+x /root/devlib-target/bin/trace-cmd
2016-04-27 11:06:46,704 DEBUG   : cat /sys/kernel/debug/tracing/available_events
2016-04-27 11:06:47,036 INFO    :         FTrace - Enabled tracepoints:
2016-04-27 11:06:47,037 INFO    :         FTrace -   cpu_frequency
2016-04-27 11:06:47,038 INFO    :         FTrace -   cpu_idle
2016-04-27 11:06:47,038 INFO    :         FTrace -   sched_switch
2016-04-27 11:06:47,039 DEBUG   : No RT-App workloads, skipping calibration
2016-04-27 11:06:47,040 INFO    :        TestEnv - Set results folder to:
2016-04-27 11:06:47,040 INFO    :        TestEnv -    /home/pippo/work/lisa/results/20160427_110647
2016-04-27 11:06:47,040 INFO    :        TestEnv - Experiment results available also in:
2016-04-27 11:06:47,041 INFO    :        TestEnv -    /home/pippo/work/lisa/results_latest

In [6]:
def get_host_ip():
    """Returns the IP of the local host"""
    ifs = ni.interfaces()
    for interface in ifs:
        if interface == 'lo':
            continue

        addresses = ni.ifaddresses(interface)
        if addresses.has_key(socket.AF_INET):
            return addresses[socket.AF_INET][0]['addr']

HOST_IP = get_host_ip()

Set ChromeOS paths


In [7]:
CROS_SDK_BIN_PATH = CROS_BASE + "/chromium/tools/depot_tools/cros_sdk"
username = !id -un
CROS_PATH = CROS_BASE + "/chroot/home/" + username[0]

Test-specific Parser Functions


In [8]:
def parse_graphics_WebGLAquarium(results_dir):
    results_file = os.path.join(
        CROS_PATH,
        os.path.basename(results_dir),
        'results-1-graphics_WebGLAquarium/graphics_WebGLAquarium/results/keyval'
    )
    
    data = {}
    with open(results_file) as data_file:
        for line in data_file:
            if line.strip():
                key, val = line.split('=')
                data[key] = float(val)
        
    return data

In [9]:
parse_results = {
    # Acquarium
    'graphics_WebGLAquarium' : parse_graphics_WebGLAquarium
}

Run a Test


In [10]:
def CrosSdkSession(password):
    """
    Create cros_sdk session. The user will be asked to type his password.
    
    :param password: host machine password
    :type password: str
    """
    cros_sdk_session = Popen(['sudo -Sk {}'.format(CROS_SDK_BIN_PATH)],
                             bufsize=1,
                             stdin=PIPE,
                             stdout=PIPE,
                             stderr=PIPE,
                             cwd=CROS_PATH,
                             shell=True)
    cros_sdk_session.stdin.write(password)
    cros_sdk_session.stdin.write('\n')
    return cros_sdk_session

def test_that(password, te, test, pwr_time_s, get_token=False):
    """
    Run a specific test using the test_that command.
    
    :param password: host machine password
    :type password: str
    
    :param te: Test Environment object
    :type te: env.TestEnv
    
    :param test: name of the test to be run
    :type test: str
    
    :param pwr_time_s: power measurement duration in seconds
    :type pwr_time_s: int
    
    :param get_token: if True wait for token before collecting traces
    :type get_token: bool
    """
    
    results_dir = "~/results-dir"
    pwr_file = "~/power.txt"
    
    test_cmd = 'test_that -b oak {} --results_dir {} {}\n'.format(te.ip,
                                                                  results_dir,
                                                                  test)
    pwr_cmd = 'dut-control -t {} -y dvfs1_mw dvfs2_mw > {}\n'.format(pwr_time_s,
                                                                     pwr_file)
    
    # Create cros_sdk session
    cros_sdk_session = CrosSdkSession(password)
        
    logging.info('#### Start %s execution', test)
    cros_sdk_session.stdin.write(test_cmd)
    
    if get_token:
        # Setup socket to get token from target
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock.bind((HOST_IP, 1234))
        logging.debug('     Waiting for token...')
        while True:
            data, addr = sock.recvfrom(32)
            if addr[0] == te.ip and data == "POWER":
                break
        sock.close()    
        logging.debug('     Token received....')
    
    logging.debug('     Start trace collection')
    # Check if trace events need to be collected
    if te.ftrace:
        te.ftrace.start()
        
    sleep(5)
    # Start measuring power
    cros_sdk_session.stdin.write(pwr_cmd)
    
    # communicate will close the session when the command terminates
    cros_sdk_session.communicate()
    logging.info('#### Completed %s execution', test)
    
    if te.ftrace:
        te.ftrace.stop()
        te.ftrace.get_trace(os.path.join(te.res_dir, 'trace.dat'))
    
    # Parse results using test-specific parser
    results = parse_results[test](results_dir)
    
    # Copy results to our Test Environment results directory
    with open(os.path.join(te.res_dir, 'results.json'), 'w') as outfile:
        json.dumps(results, outfile)
    
    return {
        "results"  : results,
        "pwr_file" : pwr_file
    }

In [11]:
# ask user for host password
password = getpass.getpass()


········

In [12]:
# Run Acquarium and collect results
acquarium_res = test_that(password,
                          te,
                          'graphics_WebGLAquarium',
                          10,
                          get_token=True)


2016-04-27 11:06:53,601 INFO    : #### Start graphics_WebGLAquarium execution
2016-04-27 11:06:53,603 DEBUG   :      Waiting for token...
2016-04-27 11:07:23,040 DEBUG   :      Token received....
2016-04-27 11:07:23,041 DEBUG   :      Start trace collection
2016-04-27 11:07:23,142 DEBUG   : sudo -- sh -c 'echo 10240 > '\''/sys/kernel/debug/tracing/buffer_size_kb'\'''
2016-04-27 11:07:23,845 DEBUG   : sudo -- sh -c 'cat '\''/sys/kernel/debug/tracing/buffer_size_kb'\'''
2016-04-27 11:07:24,750 DEBUG   : sudo -- sh -c '/root/devlib-target/bin/trace-cmd reset'
2016-04-27 11:07:26,170 DEBUG   : sudo -- sh -c '/root/devlib-target/bin/trace-cmd start -e cpu_frequency -e cpu_idle -e sched_switch'
2016-04-27 11:07:27,539 DEBUG   : sudo -- sh -c 'echo TRACE_MARKER_START > '\''/sys/kernel/debug/tracing/trace_marker'\'''
2016-04-27 11:07:28,141 DEBUG   : Trace CPUFreq frequencies
2016-04-27 11:07:28,242 DEBUG   : sudo -- sh -c '/root/devlib-target/bin/shutils cpufreq_trace_all_frequencies'
2016-04-27 11:08:45,869 INFO    : #### Completed graphics_WebGLAquarium execution
2016-04-27 11:08:45,870 DEBUG   : Trace CPUFreq frequencies
2016-04-27 11:08:45,971 DEBUG   : sudo -- sh -c '/root/devlib-target/bin/shutils cpufreq_trace_all_frequencies'
2016-04-27 11:08:46,875 DEBUG   : sudo -- sh -c 'echo TRACE_MARKER_STOP > '\''/sys/kernel/debug/tracing/trace_marker'\'''
2016-04-27 11:08:47,578 DEBUG   : sudo -- sh -c '/root/devlib-target/bin/trace-cmd stop'
2016-04-27 11:08:48,482 DEBUG   : sudo -- sh -c '/root/devlib-target/bin/trace-cmd extract -o /root/devlib-target/trace.dat'
2016-04-27 11:08:50,337 DEBUG   : /usr/bin/scp -r   root@192.168.0.1:/root/devlib-target/trace.dat /home/pippo/work/lisa/results/20160427_110647/trace.dat

In [13]:
# Print Acquarium results
df = pd.DataFrame.from_dict(acquarium_res['results'], orient="index")
df.columns = ['Values']
df


Out[13]:
Values
avg_render_time_0050_fishes{perf} 0.007754
std_render_time_0050_fishes{perf} 0.003022
avg_render_time_1000_fishes{perf} 0.015975
meminfo_MemUsed{perf} 1052424.000000
std_interframe_time_1000_fishes{perf} 0.036034
avg_fps_1000_fishes{perf} 38.576854
std_interframe_time_0050_fishes{perf} 0.029933
std_render_time_1000_fishes{perf} 0.002286
meminfo_SwapUsed{perf} 0.000000
avg_interframe_time_1000_fishes{perf} 0.025922
avg_interframe_time_0050_fishes{perf} 0.017457
avg_fps_0050_fishes{perf} 57.282652

Plot trace data using TRAPpy


In [ ]:
if te.ftrace:
    trappy.plotter.plot_trace(te.res_dir)

Compute Energy


In [15]:
# Collect SERVO trace
servo_trace_file = os.path.join(CROS_PATH,
                                os.path.basename(acquarium_res['pwr_file']))
time = []
dvfs1_wm = []
dvfs2_wm = []
with open(servo_trace_file, 'r') as f:
    for l in f:
        if l.startswith("@@"):
            continue

        info = re.split(r'[ :]+', l)
        if info[1] == "dvfs1_mw":
            time.append(float(info[0]))
            dvfs1_wm.append(float(info[2]))
        else:
            dvfs2_wm.append(float(info[2]))

# Create dataframes for power data from SERVO board
big_pwr = pd.DataFrame(dvfs1_wm, index=time, columns=['Power'])
little_pwr = pd.DataFrame(dvfs2_wm, index=time, columns=['Power'])

x = big_pwr.index.get_values()
y = big_pwr.Power.get_values()
bnrg = integrate.simps(y, x=x)

x = little_pwr.index.get_values()
y = little_pwr.Power.get_values()
lnrg = integrate.simps(y, x=x)
results = {
    'big Cluster' : bnrg,
    'LITTLE Cluster' : lnrg,
    'Total' : bnrg + lnrg,
}

df = pd.DataFrame.from_dict(results, orient="index")
df.columns = ['Energy [mJ]']
df


Out[15]:
Energy [mJ]
Total 4573.172353
LITTLE Cluster 1574.512282
big Cluster 2998.660071