In [0]:

    

#@title Licensed under the Apache License, Version 2.0 (the "License"); { display-mode: "form" }
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


    

In [0]:

    

import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.core.util import test_log_pb2


    

In [0]:

    

def load_benchmarks(path, columns, extra_cols=None):
  results = []

  for f in tf.io.gfile.glob(path):
    with tf.io.gfile.GFile(f, 'rb') as infile:
      serialized_entry = infile.read()
      benchmark_item = test_log_pb2.BenchmarkEntries.FromString(
          serialized_entry)
      entry, = benchmark_item.entry
      extras = entry.extras
      
      names = extras['name'].string_value
      if names.startswith('('):
        names = tuple(str(n) for n in names[1:-2].split(','))
      else:
        names = (names,)

      all_times = None
      all_times = extras['all_times'].string_value[1:-1].split(', ')
      all_times = list(map(float, all_times))
      
      extra_col_values = ()
      if extra_cols:
        for c in extra_cols:
          extra_col_values += (extras[c].double_value,)
      
      for time in all_times:
        results.append(names + (time,) + extra_col_values)

  if extra_cols:
    columns += extra_cols
        
  return pd.DataFrame(results, columns=columns)


    

In [0]:

    

data = load_benchmarks(
    '/tmp/autograph/sysml2019_benchmarks/BeamSearchBenchmark.*',
    ('benchmark', 'max_seq_len', 'vocab_size', 'time'))

data.groupby(['benchmark', 'max_seq_len', 'vocab_size']).agg([np.mean, np.std])


    

In [0]:

    

data = load_benchmarks('/tmp/autograph/sysml2019_benchmarks/MAMLBenchmark.*',
                       ('benchmark', 'meta_steps', 'time'))

data.groupby(['benchmark', 'meta_steps']).agg([np.mean, np.std])


    

In [0]:

    

data = load_benchmarks('/tmp/autograph/sysml2019_benchmarks/LBFGSBenchmark.*',
                       ('benchmark', 'batch_size', 'time'))

data.groupby(['benchmark', 'batch_size']).agg([np.mean, np.std])


    

In [0]:

    

data = load_benchmarks('/tmp/autograph/sysml2019_benchmarks/MAMLBenchmark.*',
                       ('benchmark', 'meta_steps', 'time'))

data.groupby(['benchmark', 'meta_steps']).agg([np.mean, np.std])


    

In [0]:

    

data = load_benchmarks(
    '/tmp/autograph/sysml2019_benchmarks/MNISTBenchmark.*',
    ('benchmark', 'time'),
    extra_cols=('iter_volume',))

data['examples_per_sec'] = data['iter_volume'].values / data['time'].values
data = data[['benchmark', 'examples_per_sec']]

data.groupby(['benchmark']).agg([np.mean, np.std])


    

In [0]:

    

data = load_benchmarks(
    '/tmp/autograph/sysml2019_benchmarks/RNNBenchmark.*',
    ('benchmark', 'batch_size', 'max_seq_len', 'time'),
    extra_cols=('iter_volume',))

data['examples_per_sec'] = data['iter_volume'].values / data['time'].values 
data = data[['benchmark', 'batch_size', 'max_seq_len', 'examples_per_sec']]

data.groupby(['benchmark', 'batch_size', 'max_seq_len']).agg([np.mean, np.std])


    

In [0]:

    

data = load_benchmarks(
    '/tmp/autograph/sysml2019_benchmarks/Seq2SeqBenchmark.*',
    ('benchmark', 'max_seq_len', 'vocab_size', 'teacher_forcing', 'time'))

data.groupby(['benchmark', 'max_seq_len', 'vocab_size', 'teacher_forcing']).agg([np.mean, np.std])