Algorithm Performance¶
Setup¶
In [1]:
from importlib import import_module
import json
from pathlib import Path
In [2]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import plotnine as p
In [3]:
import bookgender.datatools as dt
from bookgender.config import data_dir
from bookgender.nbutils import *
In [4]:
init_figs('AlgoPerf')
Load Performance Data¶
We need to collect all of the accuracy output files.
Algorithm Eval Performance¶
In [5]:
perf_data = pd.read_csv('data/rec-perf.csv')
perf_data.head()
Out[5]:
We would also like to bootstrap confidence intervals - this requires per-user statistics.
Algorithm Tune Performance¶
In [6]:
tune_data = pd.read_json('data/rec-tune.json', orient='records', lines=True)
tune_data.head()
Out[6]:
Rerank Performance¶
In [7]:
rerank_data = pd.read_csv('data/rerank-perf.csv')
rerank_data.head()
Out[7]:
Organize Runs¶
We have data sets, and we have implicit/explicit configurations. For some data sets, these are encoded in the data set; for others, in the algorithm.
We need to split that out.
In [8]:
def normalize_runs(frame):
frame = frame.assign(Implicit = frame['DataSet'].str.endswith('-I') | frame['Algorithm'].str.endswith('-imp'))
frame['Algorithm'] = frame['Algorithm'].str.replace('-imp', '')
frame['DataSet'] = frame['DataSet'].str.replace('-[IE]', '')
frame['Algorithm'] = frame['Algorithm'].str.replace('wrls', 'als').str.upper()
frame['Algorithm'] = frame['Algorithm'].str.replace(r'(\w)\w+-(\w)\w+', r'\1\2')
return frame
In [9]:
perf_data = normalize_runs(perf_data)
And look at that data:
In [10]:
perf_data
Out[10]:
In [11]:
tune_data = normalize_runs(tune_data)
tune_data
Out[11]:
Accuracy Plots¶
Plot the MRR for each data set!
In [12]:
merged_mrr = pd.concat([
tune_data[['DataSet', 'Algorithm', 'Implicit', 'MRR']].assign(Stage='Tune'),
perf_data[['DataSet', 'Algorithm', 'Implicit', 'MRR']].assign(Stage='Eval')
], ignore_index=True)
merged_mrr['Mode'] = 'Explicit'
merged_mrr.loc[merged_mrr['Implicit'], 'Mode'] = 'Implicit'
merged_mrr.head()
Out[12]:
In [13]:
sns.catplot(x='Algorithm', y='MRR', row='Mode', col='DataSet', hue='Stage', data=merged_mrr, kind='bar',
sharey=False, margin_titles=True, aspect=1.5, height=2.2)
Out[13]:
In [14]:
make_plot(merged_mrr, p.aes(x='Algorithm', y='MRR', fill='Stage'),
p.geom_bar(stat='identity', position='dodge'),
p.facet_grid('Mode ~ DataSet', scales='free_y'),
p.scale_fill_brewer('qual', 'Dark2'),
file='rec-perf.pdf', width=7, height=4, legend_position='top', legend_title=p.element_blank())
Out[14]:
In [15]:
print(perf_data[['DataSet', 'Algorithm', 'Implicit', 'MRR']].set_index(['DataSet', 'Implicit', 'Algorithm']).unstack().to_latex())
And hit rate:
In [16]:
sns.catplot(x='Algorithm', y='HR', row='Implicit', col='DataSet', data=perf_data, kind='bar',
sharey=False, margin_titles=True)
Out[16]:
Rerank Loss¶
In [17]:
rerank_data = normalize_runs(rerank_data)
rerank_data.head()
Out[17]:
In [18]:
rr_mrr = pd.concat([
perf_data[['DataSet', 'Algorithm', 'Implicit', 'MRR', 'HR']].assign(Strategy='Raw'),
rerank_data[['DataSet', 'Algorithm', 'Implicit', 'MRR', 'HR', 'Strategy']]
], ignore_index=True)
rr_mrr.head()
rr_mrr['Mode'] = 'Explicit'
rr_mrr.loc[rr_mrr['Implicit'], 'Mode'] = 'Implicit'
rr_mrr['Strategy'] = rr_mrr['Strategy'].astype('category').cat.reorder_categories(['Raw', 'SingleEQ', 'GreedyEQ', 'GreedyReflect'])
rr_mrr.head()
Out[18]:
In [19]:
sns.catplot(x='Algorithm', y='MRR', row='Mode', col='DataSet', hue='Strategy', data=rr_mrr, kind='bar',
sharey=False, margin_titles=True, aspect=1.5, height=1.8)
# plt.savefig(fig_dir / 'rerank-perf.pdf')
Out[19]:
In [20]:
make_plot(rr_mrr, p.aes(x='Algorithm', y='MRR', fill='Strategy'),
p.geom_bar(stat='identity', position='dodge'),
p.facet_grid('Mode ~ DataSet', scales='free_y'),
p.scale_fill_brewer('qual', 'Dark2'),
file='rerank-perf.pdf', width=7, height=4, legend_position='top', legend_title=p.element_blank())
Out[20]:
In [21]:
penalty = (perf_data.set_index(['DataSet', 'Implicit', 'Algorithm']).MRR - \
rerank_data.set_index(['DataSet', 'Implicit', 'Algorithm', 'Strategy']).MRR) / perf_data.set_index(['DataSet', 'Implicit', 'Algorithm']).MRR
In [22]:
print(penalty.unstack().to_latex(float_format=lambda f: '{:.2f}%'.format(f*100)))
In [ ]: