This notebook is where all national/NERC figures (not maps) are created. It contains code to import generation and CO₂ intensity results and uses some functions from scr to create figures.
The first part of this notebook imports and combines data. Subsequent sections generate different types of plots. Summary statistics of CO₂ intensity are available at the bottom of this notebook.
In [1]:
%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.lines as mlines
from matplotlib.ticker import StrMethodFormatter
import pandas as pd
import seaborn as sns
import numpy as np
import os
from os.path import join
from statsmodels.tsa.tsatools import detrend
# import statsmodels.formula.api as sm
import itertools
import sys
import calendar
import datetime
# sns.set()
idx = pd.IndexSlice
try:
from pathlib import Path
except:
pass
In [2]:
file_date = '2018-03-06'
In [3]:
# Load the "autoreload" extension
%load_ext autoreload
# always reload modules marked with "%aimport"
%autoreload 1
In [4]:
# add the 'src' directory as one where we can import modules
src_dir = join(os.getcwd(), os.pardir, 'src')
sys.path.append(src_dir)
In [5]:
%aimport Analysis.index
from Analysis.index import facility_co2, adjust_epa_emissions, group_fuel_cats
from Analysis.index import reduce_emission_factors, add_datetime
%aimport util.utils
from util.utils import rename_cols
%aimport Plots.plot
from Plots.plot import region_facet_grid, rolling_corr_plot, monthly_fuel_gen
In [6]:
cwd = os.getcwd()
path = join(cwd, '..', 'Data storage', 'National data',
'National index {}.csv'.format(file_date))
national_index = pd.read_csv(path, parse_dates=['datetime'])
national_index['nerc'] = 'USA'
rename_cols(national_index)
In [7]:
national_index.head()
Out[7]:
In [8]:
path = join(cwd, '..', 'Data storage',
'final NERC data',
'NERC gen emissions and index {}.csv'.format(file_date))
nerc_index = pd.read_csv(path, parse_dates=['datetime'])
nerc_index.rename(columns={'index': 'index (g/kwh)'}, inplace=True)
In [9]:
nerc_index.head()
Out[9]:
In [10]:
path = join(cwd, '..', 'Data storage',
'National data', 'National percent gen {}.csv'.format(file_date))
national_per_gen = pd.read_csv(path)
add_datetime(national_per_gen)
national_per_gen['nerc'] = 'USA'
national_per_gen.rename(columns={'generation (mwh)': '% generation'},
inplace=True)
In [11]:
national_per_gen.head()
Out[11]:
In [12]:
path = join(cwd, '..', 'Data storage',
'final NERC data', 'NERC percent gen {}.csv'.format(file_date))
nerc_per_gen = pd.read_csv(path, parse_dates=['datetime'])
In [13]:
nerc_per_gen.head()
Out[13]:
In [14]:
path = join(cwd, '..', 'Data storage',
'National data', 'National generation {}.csv'.format(file_date))
national_gen = pd.read_csv(path)
add_datetime(national_gen)
national_gen['nerc'] = 'USA'
In [15]:
national_gen.tail()
Out[15]:
In [16]:
index_cols = ['nerc', 'generation (mwh)', 'datetime']
nerc_gen = pd.merge(nerc_per_gen, nerc_index[index_cols],
on=['nerc', 'datetime'])
nerc_gen.rename(columns={'generation (mwh)' : 'total generation (mwh)'},
inplace=True)
nerc_gen['generation (mwh)'] = (nerc_gen['% generation']
* nerc_gen['total generation (mwh)'])
nerc_gen.set_index(['nerc', 'fuel category', 'datetime'], inplace=True)
nerc_gen.head()
Out[16]:
In [17]:
total = nerc_gen.groupby(['nerc', 'datetime'])['% generation',
'generation (mwh)'].sum()
total['fuel category'] = 'Total'
total.set_index('fuel category', append=True, inplace=True)
total = total.reorder_levels(['nerc', 'fuel category', 'datetime'])
total.head()
Out[17]:
In [18]:
nerc_gen = pd.concat([nerc_gen, total])
nerc_gen.sort_index(inplace=True)
In [19]:
nerc_gen.head()
Out[19]:
In [20]:
path = join(cwd, '..', 'Data storage', 'Derived data',
'Plant Capacity', 'monthly capacity by fuel.csv')
capacity = pd.read_csv(path, index_col=['nerc', 'fuel category', 'datetime'],
parse_dates=['datetime'])
capacity.drop(['year', 'month'], axis=1, inplace=True)
In [21]:
capacity.head()
Out[21]:
In [22]:
path = join(cwd, '..', 'Data storage', 'Derived data',
'Plant Capacity', 'monthly natural gas split.csv')
ng_capacity_type = pd.read_csv(path)
ng_capacity_type.head()
Out[22]:
In [23]:
ng_type_avg = (ng_capacity_type.reset_index()
.groupby(['nerc', 'year'])
.mean())
ng_type_avg.head()
Out[23]:
In [24]:
index = pd.concat([national_index, nerc_index])
In [25]:
index.head()
Out[25]:
In [26]:
per_gen = pd.concat([national_per_gen, nerc_per_gen])
per_gen.year = per_gen['datetime'].dt.year
per_gen.month = per_gen['datetime'].dt.month
In [27]:
per_gen.head()
Out[27]:
In [28]:
national_gen.head()
Out[28]:
In [29]:
cols = ['nerc', 'fuel category', 'datetime']
gen = pd.concat([national_gen.set_index(cols), nerc_gen])
gen.sort_index(inplace=True)
gen.drop(['% generation', 'total generation (mwh)'], axis=1,
inplace=True)
In [30]:
gen.head()
Out[30]:
In [34]:
sns.set_style('white', {'axes.linewidth': 1.5,
'axes.grid': True})
sns.set_context('paper', font_scale=1.0)
In [32]:
sns.palplot(sns.color_palette('tab10', n_colors=10))
In [33]:
fuel_order = [5, -1, 1, 0, 2, -2, 4, -3]
fuel_palette = [sns.color_palette('tab10', n_colors=10)[x]
for x in fuel_order]
sns.palplot(fuel_palette)
In [35]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
temp = (gen.reset_index()
.groupby(['nerc', 'datetime'])
.sum()
.reset_index())
temp['Month'] = temp['datetime'].dt.month
temp['Year'] = temp['datetime'].dt.year
temp['generation (mwh)'] /= 1E6
# temp.rename(columns={'month': 'Month',
# 'year': 'Year'}, inplace=True)
with sns.plotting_context('paper', font_scale=1):
g = sns.factorplot(x='Month', y='generation (mwh)', hue='Year',
data=temp, col='nerc', col_wrap=3, col_order=order,
palette='viridis_r', scale=0.5, size=2, sharey=False)
axes = g.axes.flatten()
for ax, title in zip(axes, order):
ax.set_title(title)
ax.set_ylim(0, None)
if title in ['USA', 'RFC', 'FRCC']:
ax.set_ylabel('Million MWh')
path = join(cwd, '..', 'Figures', 'Monthly generation {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [56]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
temp = index.reset_index()
temp.rename(columns={'month': 'Month',
'year': 'Year'}, inplace=True)
with sns.plotting_context('paper', font_scale=1):
g = sns.factorplot(x='Month', y='index (g/kwh)', hue='Year',
data=temp, col='nerc', col_wrap=3, col_order=order,
palette='viridis_r', scale=0.5, size=2)
axes = g.axes.flatten()
for ax, title in zip(axes, order):
ax.set_title(title)
ax.set_ylim(0, 1050)
if title in ['USA', 'RFC', 'FRCC']:
ax.set_ylabel('g $\mathregular{CO_2 \ kWh^{-1}}$')
path = join(cwd, '..', 'Figures',
'Monthly index (12-months) {}.pdf'.format(file_date))
# plt.savefig(path, bbox_inches='tight')
In [47]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
FG_kwargs = dict(hue='nerc',
col='nerc',
col_wrap=3,
aspect=1.2,
hue_order=order,
palette='tab10')
region_facet_grid(df=index.reset_index(), plot_function=plt.plot, x_axis='datetime',
add_legend=False, y_axis='index (g/kwh)', col_order=order,
suptitle='', FG_kwargs=FG_kwargs, x_label='Year',
y_label='g CO$_2$/kWh', context='talk', font_scale=1)
path = join(cwd, '..', 'Figures', 'SI', 'Monthly index by region.pdf')
plt.savefig(path, bbox_inches='tight')
In [29]:
index.head()
Out[29]:
In [48]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
FG_kwargs = dict(hue='nerc',
col='nerc',
col_wrap=3,
aspect=1.2,
hue_order=order,
palette='tab10')
detrend_index = index.copy()
detrend_index.set_index(['nerc', 'datetime'], inplace=True)
detrend_index.sort_index(inplace=True)
for nerc in order:
detrend_index.loc[idx[nerc, :], 'index (g/kwh)'] = detrend(detrend_index.loc[idx[nerc, :], 'index (g/kwh)'])
region_facet_grid(df=detrend_index.reset_index(), plot_function=plt.plot, x_axis='datetime',
add_legend=True, y_axis='index (g/kwh)', col_order=order,
suptitle='', FG_kwargs=FG_kwargs, x_label='Year',
context='talk', font_scale=1)
In [49]:
def diff_detrend(series, n=1):
d = series - series.shift(n)
return d
In [50]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
FG_kwargs = dict(hue='nerc',
col='nerc',
col_wrap=3,
aspect=1.2,
hue_order=order,
palette='tab10')
detrend_index = index.copy()
detrend_index.set_index(['nerc', 'datetime'], inplace=True)
detrend_index.sort_index(inplace=True)
for nerc in order:
detrend_index.loc[idx[nerc, :], 'index (g/kwh)'] = diff_detrend(detrend_index.loc[idx[nerc, :], 'index (g/kwh)'])
region_facet_grid(df=detrend_index.reset_index(), plot_function=plt.plot, x_axis='datetime',
add_legend=True, y_axis='index (g/kwh)', col_order=order,
suptitle='', FG_kwargs=FG_kwargs, x_label='Year',
context='talk', font_scale=1)
In [37]:
from statsmodels.tsa.seasonal import seasonal_decompose
In [53]:
wecc = index.loc[index['nerc'] == 'WECC', ['index (g/kwh)', 'datetime']]
wecc.set_index('datetime', inplace=True)
wecc.head()
Out[53]:
In [54]:
sd = seasonal_decompose(wecc)
In [55]:
sd.plot()
Out[55]:
These three things are all the same:
WECC dataWECC data
In [56]:
fig, ax = plt.subplots(nrows=3, squeeze=True)
(wecc - sd.trend).plot(ax=ax[0], legend=False)
(sd.seasonal + sd.resid).plot(ax=ax[1], legend=False)
(wecc - wecc.rolling(12, center=True).mean()).plot(ax=ax[2], legend=False)
plt.tight_layout()
In [57]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
FG_kwargs = dict(hue='nerc',
col='nerc',
col_wrap=3,
aspect=1.2,
hue_order=order,
palette='tab10')
temp = index.copy()
temp.set_index('datetime', inplace=True)
df_list = []
for nerc in order:
# Subtract the rolling mean from the raw data in each region
sd = (temp.loc[temp['nerc'] == nerc, 'index (g/kwh)']
- (temp.loc[temp['nerc'] == nerc, 'index (g/kwh)']
.rolling(12, center=True)
.mean()))
# Have to make the series into a df to add the 'nerc' column
sd = pd.DataFrame(sd)
sd['nerc'] = nerc
df_list.append(sd)
detrend_index = pd.concat(df_list)
region_facet_grid(df=detrend_index.reset_index(), plot_function=plt.plot, x_axis='datetime',
add_legend=False, y_axis='index (g/kwh)', col_order=order,
suptitle='', FG_kwargs=FG_kwargs, x_label='Year',
context='talk', font_scale=1)
In [38]:
temp = index.copy()
temp.set_index('datetime', inplace=True)
df_list = []
for nerc in order:
# Subtract the rolling mean from the raw data in each region
sd = (temp.loc[temp['nerc'] == nerc, 'index (g/kwh)']
- (temp.loc[temp['nerc'] == nerc, 'index (g/kwh)']
.rolling(12, center=True)
.mean()))
# Have to make the series into a df to add the 'nerc' column
sd = pd.DataFrame(sd)
sd['nerc'] = nerc
df_list.append(sd)
detrend_index = pd.concat(df_list)
In [39]:
index.head()
Out[39]:
In [40]:
detrend_index.head()
Out[40]:
In [41]:
combined = index.copy()
combined.set_index(['nerc', 'datetime'], inplace=True)
df = detrend_index.reset_index(drop=False).set_index(['nerc', 'datetime'])
df.columns = ['detrend index']
In [42]:
combined = pd.concat([index.set_index(['nerc', 'datetime']), df])
In [43]:
combined.head()
Out[43]:
In [45]:
def plot_trend(x, y, **kws):
'Plot function for the FacetGrid to plot all three data series'
trend = y.rolling(12, center=True).mean()
plt.plot(x, y, **kws)
plt.plot(x, trend, lw=2, c='0.4')
detrended = y - trend
plt.plot(x, detrended, **kws)
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
FG_kwargs = dict(hue='nerc',
col='nerc',
col_wrap=3,
aspect=1.2,
hue_order=order,
palette='tab10',
size=2)
region_facet_grid(df=index.reset_index(), plot_function=plot_trend, x_axis='datetime',
add_legend=False, y_axis='index (g/kwh)', col_order=order,
suptitle='', FG_kwargs=FG_kwargs, x_label='Year',
y_label='g CO$_2$/kWh', context='paper', font_scale=1)
path = join(cwd, '..', 'Figures', 'SI',
'Monthly index trend and detrended by region {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [66]:
gen.head()
Out[66]:
In [105]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
fuel_order = ['Coal', 'Natural Gas', 'Nuclear', 'Hydro', 'Wind', 'Solar',
'Other', 'Other Renewables']
temp = per_gen.copy()
temp.rename(columns={'fuel category': 'Fuel Category'}, inplace=True)
FG_kwargs = dict(hue='Fuel Category',
col='nerc',
col_wrap=3,
aspect=1.2,
hue_order=fuel_order,
palette=fuel_palette,
size=2)
g = region_facet_grid(df=temp, plot_function=plt.plot, x_axis='datetime',
add_legend=True, y_axis='% generation', col_order=order,
suptitle='', FG_kwargs=FG_kwargs, x_label='Year',
y_label='Fraction Generation',
context='paper', font_scale=1)
path = join(cwd, '..', 'Figures', 'Generation by fuel_full {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [145]:
capacity['Capacity Factor'] = (gen.loc[:, 'generation (mwh)']
/ capacity.loc[:, 'possible gen'])
In [146]:
capacity.tail()
Out[146]:
In [113]:
# Monthly coal CF over the last 2 years
capacity.loc[idx['USA', 'Coal', :], :].tail(n=24)
Out[113]:
In [147]:
# Check annual capacity factors
# need to add a year column for groupby
capacity['year'] = capacity.index.get_level_values('datetime').year
gen['year'] = gen.index.get_level_values('datetime').year
annual_cf = (gen.groupby(['nerc', 'fuel category', 'year'])['generation (mwh)'].sum()
.div(capacity.groupby(['nerc', 'fuel category', 'year'])['possible gen'].sum()))
In [148]:
# Annual coal CF since 2014
annual_cf.unstack().loc[idx[:, 'Coal'], 2014:]
Out[148]:
In [111]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
temp = capacity.loc[idx[:, 'Coal', :], :].reset_index()
temp['Month'] = temp['datetime'].dt.month
temp['Year'] = temp['datetime'].dt.year
with sns.plotting_context('paper', font_scale=1):
g = sns.factorplot(x='Month', y='Capacity Factor', hue='Year',
data=temp, col='nerc', col_wrap=3, col_order=order,
palette='viridis_r', scale=0.5, size=2)
axes = g.axes.flatten()
for ax, title in zip(axes, order):
ax.set_title(title)
# ax.set_ylim(0, 1050)
if title in ['USA', 'RFC', 'FRCC']:
ax.set_ylabel('Capacity Factor\n(Coal)')
path = join(cwd, '..', 'Figures', 'SI', 'Monthly Coal CF {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [112]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
temp = capacity.loc[idx[:, 'Natural Gas', :], :].reset_index()
temp['Month'] = temp['datetime'].dt.month
temp['Year'] = temp['datetime'].dt.year
with sns.plotting_context('paper', font_scale=1):
g = sns.factorplot(x='Month', y='Capacity Factor', hue='Year',
data=temp, col='nerc', col_wrap=3, col_order=order,
palette='viridis_r', scale=0.5, size=2)
axes = g.axes.flatten()
for ax, title in zip(axes, order):
ax.set_title(title)
# ax.set_ylim(0, 1050)
if title in ['USA', 'RFC', 'FRCC']:
ax.set_ylabel('Capacity Factor\n(Natural Gas)')
path = join(cwd, '..', 'Figures', 'SI', 'Monthly Natural Gas CF {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [67]:
order = ['MRO', 'RFC', 'SERC', 'FRCC', 'SPP', 'WECC', 'NPCC', 'TRE']
# Melt the wide dataframe into tidy format for plotting
temp = (ng_type_avg.reset_index()
.melt(id_vars=['nerc', 'year'],
value_vars=['ngcc fraction', 'turbine fraction', 'other fraction'],
var_name='type', value_name='fraction capacity'))
# Use the new tableau colorblind10 palette
with plt.style.context('tableau-colorblind10'):
g = sns.factorplot(x='year', y='fraction capacity', data=temp, hue='nerc',
row='type', hue_order=order,
scale=1, aspect=1.4, ci=0).set_xticklabels(rotation=35)
g.set_axis_labels(y_var='Fraction Capacity')
#rename each of the subplot titles
axs = g.axes.ravel()
for ax, title in zip(axs, ['NGCC', 'Turbine', 'Other']):
ax.set_title(title)
path = join(cwd, '..', 'Figures', 'SI', 'Natural gas capacity by type.pdf')
plt.savefig(path, bbox_inches='tight')
In [44]:
def monthly_fuel_gen(gen_df, fuel, folder, file_type='pdf', dpi=350, save=False):
"""
Make a FacetGrid plot of monthly generation for a single fuel category
inputs:
gen_df (dataframe): monthly generation for all fuels
fuel (string): name of the fuel category to plot
folder (path): folder where the plot should be saved
file_type (string): file format (e.g. pdf, png, etc)
dpi (int): dots per inch resolution for saved file (if not pdf)
save (bool): if True, save the file
"""
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
temp = gen.loc[idx[:, fuel, :], :].reset_index()
temp['Month'] = temp['datetime'].dt.month
temp['Year'] = temp['datetime'].dt.year
temp['million mwh'] = temp['generation (mwh)'] / 1e6
with sns.plotting_context('paper', font_scale=1):
g = sns.factorplot(x='Month', y='million mwh', hue='Year', sharey=False,
data=temp, col='nerc', col_wrap=3, col_order=order,
palette='viridis_r', scale=0.5, size=2)
axes = g.axes.flatten()
for ax, title in zip(axes, order):
ax.set_title(title)
ax.set_ylim(0, None)
# ax.set_ylim(0, 1050)
if title in ['USA', 'RFC', 'FRCC']:
ax.set_ylabel('Million MWh\n({})'.format(fuel))
path = join(folder, 'Monthly {} gen {}.{}'.format(fuel, file_date, file_type))
if save:
plt.savefig(path, bbox_inches='tight', dpi=dpi)
In [46]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Coal', folder=folder_path, file_type='pdf', save=True)
In [62]:
gen.head()
Out[62]:
In [67]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Natural Gas', folder=folder_path, file_date=file_date,
file_type='pdf', save=False)
In [45]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Natural Gas', folder=folder_path, file_type='pdf', save=True)
In [47]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Nuclear', folder=folder_path, file_type='pdf', save=True)
In [48]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Wind', folder=folder_path, file_type='pdf', save=True)
In [49]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Solar', folder=folder_path, file_type='pdf', save=True)
In [50]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Other', folder=folder_path, file_type='pdf', save=True)
In [51]:
folder_path = join(cwd, '..', 'Figures', 'SI')
monthly_fuel_gen(gen, fuel='Other Renewables', folder=folder_path, file_type='pdf', save=True)
In [108]:
order = ['USA', 'SPP', 'MRO', 'RFC', 'SERC', 'TRE', 'FRCC', 'WECC', 'NPCC']
years = range(2001, 2017, 3)
temp = gen.loc[idx[:, 'Coal', :], :].reset_index()
temp['Month'] = temp['datetime'].dt.month
temp['Year'] = temp['datetime'].dt.year
temp['coal generation'] = temp['generation (mwh)'] / 1e6
temp = temp.loc[temp['Year'].isin(years), :]
total_gen = (gen.reset_index()
.groupby(['nerc', 'datetime'])
.sum()
.reset_index())
total_gen.rename(columns={'generation (mwh)': 'total generation'}, inplace=True)
total_gen['total generation'] /= 1e6
temp = temp.merge(total_gen, on=['nerc', 'datetime'])
def scatter_coal_total(x, y, *args, **kwargs):
plt.plot(x, y, 'o', markersize=6, markeredgewidth=1, markerfacecolor='None', **kwargs)
line_kws = dict(linewidth=1.5)
sns.regplot(x, y, x_ci=None, lowess=True, n_boot=1, line_kws=line_kws,
scatter=False, **kwargs)
with sns.plotting_context('paper', font_scale=1):
g = sns.FacetGrid(data=temp, hue='Year', sharey=False, sharex=False,
col='nerc', col_wrap=3, col_order=order,
palette='viridis_r', size=2)
g.map(scatter_coal_total, 'total generation', 'coal generation')
g.add_legend()
axes = g.axes.flatten()
for ax, title in zip(axes, order):
ax.set_title(title)
# ax.set_ylim(0, None)
# ax.set_ylim(0, 1050)
# if title in ['USA', 'RFC', 'FRCC']:
# ax.set_ylabel('Million MWh Coal')
path = join(cwd, '..', 'Figures', 'Total vs coal gen {}.pdf'.format(file_date))
# plt.savefig(path, bbox_inches='tight')
In [112]:
g._legend_data['2001'].get_linewidth()
Out[112]:
In [159]:
per_gen.loc[(per_gen['fuel category'].isin(['Wind', 'Coal', 'Natural Gas'])) &
(per_gen.nerc.isin(order)) &
(per_gen.year >= 2005)].head()
Out[159]:
In [160]:
index.head()
Out[160]:
In [161]:
capacity.head()
Out[161]:
In [75]:
order = ['SPP', 'TRE', 'MRO']
fuels = ['Coal', 'Natural Gas', 'Nuclear', 'Hydro', 'Wind', 'Solar']
per_gen.year = per_gen.year.astype(int)
temp_df = per_gen.loc[(per_gen['fuel category'].isin(['Wind', 'Coal', 'Natural Gas'])) &
(per_gen.nerc.isin(order)) &
(per_gen.year >= 2005)].copy()
def early_late(x):
'Add classifications to each month based on the year'
if x <= 2008:
return '2005-2008'
elif x <= 2012:
return '2009-2012'
elif x <= 2016:
return '2013-2017'
else:
return ValueError
temp_df.loc[:, 'timeframe'] = temp_df.loc[:, 'year'].map(early_late)
merge_cols = ['nerc', 'month', 'year']
temp_df = pd.merge(temp_df, index.reset_index()[merge_cols + ['generation (mwh)']],
on=merge_cols)
temp_df['generation'] = (temp_df.loc[:, '% generation']
* temp_df.loc[:, 'generation (mwh)'] / 1e6)
# temp_df = temp_df.merge(capacity.reset_index(), how='left',
# on=['nerc', 'fuel category', 'datetime'])
if 'year_x' in temp_df.columns:
temp_df.rename(columns={'year_x': 'year'}, inplace=True)
In [76]:
temp_df.head()
Out[76]:
In [45]:
# a is the list of colors
# Matching fuel colors to the generation plot above
a = fuel_palette[0:2]
a.append(fuel_palette[4])
In [77]:
sns.tsplot(temp_df.loc[temp_df.nerc=='MRO'], time='month', unit='year', value='generation',
condition='fuel category', err_style='unit_traces')
Out[77]:
In [117]:
from matplotlib.lines import Line2D
order = ['SPP', 'TRE', 'MRO']
col_order = ['2005-2008', '2009-2012', '2013-2017']
with sns.plotting_context(font_scale=1):
g = sns.FacetGrid(temp_df, col='timeframe', row='nerc',
col_order=col_order, row_order=order,# aspect=1.2,
size=2)
# g.map_dataframe(sns.tsplot, time='month', unit='year', value='generation',
# condition='fuel category', err_style='unit_traces',
# color=a)
titles = ['{}, {}'.format(a, b) for (a, b) in
list(itertools.product(order, col_order))]
axes = g.axes.flatten()
# This is hacky, but I'm having trouble gettign tsplot to work with FacetGrid
# As a workaround, manually pass data from g.facet_data() - which returns a
# tuple - to tsplot for every axis of the figure. Then manually set the axis
# labels and create the legend.
for ax, title, data in zip(axes, titles, g.facet_data()):
sns.tsplot(data[-1], time='month', unit='year', value='generation',
condition='fuel category', err_style='unit_traces',
color=a, ax=ax, legend=False)
ax.set_title(title)
ax.set_xticks([1, 4, 7, 10])
ax.set_xlabel('')
ax.set_ylabel('')
if '2005' in title:
ax.set_ylabel('Million MWh')
if 'MRO' in title:
ax.set_xlabel('Month')
# Need to use tight_layout to get the facet spacing correct
g.fig.tight_layout()
# Create legend lines
legend_lines = {fuel: Line2D([0], [0], color=c, lw=1.5, label=fuel)
for c, fuel in zip(a, ['Coal', 'Natural Gas', 'Wind'])}
g.add_legend(legend_data=legend_lines, title='Fuel Category')
path = join(cwd, '..', 'Figures',
'_Wind-Coal-NG variability tsplot {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [98]:
g._legend_data
Out[98]:
In [66]:
order = ['SPP', 'TRE', 'MRO']
col_order = ['2005-2008', '2009-2012', '2013-2017']
with sns.plotting_context(font_scale=1):
g = sns.FacetGrid(temp_df, col='timeframe', row='nerc',
col_order=col_order, row_order=order, aspect=1.2,
size=2)
g.map_dataframe(sns.tsplot, time='month', unit='year', value='generation',
condition='fuel category', err_style='unit_traces',
color=a).add_legend(title='Fuel Category')
g.set_axis_labels('Month', 'Million MWh')
titles = ['{}, {}'.format(a, b) for (a, b) in
list(itertools.product(order, col_order))]
axes = g.axes.flatten()
for ax, title in zip(axes, titles):
ax.set_title(title)
ax.set_xticks([1, 4, 7, 10])
path = join(cwd, '..', 'Figures',
'Wind-Coal-NG variability tsplot {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [126]:
def rolling_norm_std_plot(temp_df, window=24):
rolling_var_list = []
for nerc in ['TRE', 'SPP', 'MRO']:
for fuel in ['Coal', 'Natural Gas', 'Wind']:
_df = temp_df.loc[(temp_df['nerc'] == nerc) &
(temp_df['fuel category'] == fuel),
['nerc', 'fuel category', 'datetime', 'generation']]
# I'm using std to mean variability
_df['variability'] = (_df['generation']
.rolling(window, center=True)
.std())
_df['variance'] = (_df['generation']
.rolling(window, center=True)
.var())
_df['mean'] = (_df['generation']
.rolling(window, center=True)
.mean())
_df['norm variability'] = _df['variability'] / _df['mean']
_df['dispersion'] = _df['variance'] / _df['mean']
rolling_var_list.append(_df)
rolling_var_df = pd.concat(rolling_var_list)
g = sns.FacetGrid(rolling_var_df, hue='fuel category', col='nerc',
col_order=['SPP', 'TRE', 'MRO'], palette=a, size=2)
g.map(plt.plot, 'datetime', 'norm variability').add_legend(title='Fuel Category')
g.set_xticklabels(rotation=25)
g.set_axis_labels('Year', 'Normalized Variability')
axes = g.axes.flatten()
for ax, nerc in zip(axes, ['SPP', 'TRE', 'MRO']):
ax.set_title(nerc)
In [128]:
rolling_norm_std_plot(temp_df, window=24)
path = join(cwd, '..', 'Figures',
'Wind-Coal-NG norm rolling variability {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [57]:
rolling_norm_std_plot(temp_df, window=12)
In [58]:
rolling_norm_std_plot(temp_df, window=36)
In [48]:
# Pairs of adjacent NERC regions
region_pairs = [
# ('WECC', 'MRO'),
('WECC', 'SPP'),
('WECC', 'TRE'),
('TRE', 'SPP'),
('TRE', 'SERC'),
('SPP', 'MRO'),
('SPP', 'SERC'),
('SERC', 'FRCC'),
('SERC', 'MRO'),
('RFC', 'SERC'),
('RFC', 'NPCC'),
('RFC', 'MRO'),
('MRO', 'NPCC'),
('MRO', 'WECC')
]
In [49]:
index.reset_index(inplace=True)
index.set_index(['nerc', 'datetime'], inplace=True)
index.sort_index(inplace=True)
In [51]:
rolling_corr_plot(index, region_pairs=region_pairs, window=48,
detrend_series=True, seasonal=True, fill_alpha=0.4)
path = join(cwd, '..', 'Figures',
'__Rolling correlation - seasonal detrend 48m centered {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [54]:
rolling_corr_plot(index, region_pairs=region_pairs, window=48,
detrend_series=True, seasonal=True, fill_alpha=0.4)
path = join(cwd, '..', 'Figures',
'_Rolling correlation - seasonal detrend 48m centered {}.pdf'.format(file_date))
plt.savefig(path, bbox_inches='tight')
In [85]:
rolling_corr_plot(index, region_pairs=region_pairs, window=12,
detrend_series=True, seasonal=True,
sup_title='12-Month Window')
path = join(cwd, '..', 'Figures', 'SI',
'Rolling correlation - seasonal detrend 12m centered.pdf')
plt.savefig(path, bbox_inches='tight')
In [86]:
rolling_corr_plot(index, region_pairs=region_pairs, window=24,
detrend_series=True, seasonal=True,
sup_title='24-Month Window')
path = join(cwd, '..', 'Figures', 'SI',
'Rolling correlation - seasonal detrend 24m centered.pdf')
plt.savefig(path, bbox_inches='tight')
In [87]:
rolling_corr_plot(index, region_pairs=region_pairs, window=36,
detrend_series=True, seasonal=True,
sup_title='36-Month Window')
path = join(cwd, '..', 'Figures', 'SI',
'Rolling correlation - seasonal detrend 36m centered.pdf')
plt.savefig(path, bbox_inches='tight')
In [88]:
rolling_corr_plot(index, region_pairs=region_pairs, window=72,
detrend_series=True, seasonal=True,
sup_title='72-Month Window')
path = join(cwd, '..', 'Figures', 'SI',
'Rolling correlation - seasonal detrend 72m centered.pdf')
plt.savefig(path, bbox_inches='tight')
In [129]:
index.head()
Out[129]:
In [130]:
annual_index = (index#.drop('index', axis=1)
#.reset_index()
.groupby(['year', 'nerc'])
.sum())
annual_index['index (g/kwh)'] = (annual_index['final co2 (kg)']
/ annual_index['generation (mwh)'])
annual_index.sort_index(inplace=True)
In [131]:
annual_index.head()
Out[131]:
In [132]:
annual_index.loc[idx[:, 'USA'], :]
Out[132]:
In [151]:
regions = [
'TRE',
'SERC',
'WECC',
'USA',
'RFC',
'FRCC',
'NPCC',
'SPP',
'MRO'
]
index2001 = annual_index.loc[2001, 'index (g/kwh)'][regions]
index2017 = annual_index.loc[2017, 'index (g/kwh)'][regions]
table_df = pd.DataFrame(data={2001:index2001, 2017:index2017})
table_df['Reduction'] = table_df[2001] - table_df[2017]
table_df['Percent Reduction'] = table_df['Reduction'] / table_df[2001]
table_df.sort_values('Reduction', inplace=True)
Export for use in the NERC map
In [153]:
path = join(cwd, '..', 'Data storage', 'final NERC data',
'Summary table {}.csv'.format(file_date))
table_df.to_csv(path)
Annual CO₂ intensity in 2001 and 2017, and change in intensity
In [152]:
table_df['Annual Reduction'] = table_df['Reduction'] / (2017 - 2001)
table_df
Out[152]:
In [ ]: