This notebook presents a comparison of ARFI lesion reads compared to whole mount histology reads, comparing index lesions, clinically-significant lesion and benign confounders, including atrophy and BPH.
This notebook replaces the arfi_histology_analysis.py script and the corresponding arfi_histology_analysis.md output that were becoming really cumbersome with the different subanalyses.
In [1]:
%pylab inline
%load_ext autoreload
%autoreload 2
# graphicsMode = 'paper'
graphicsMode = 'presentation'
saveFigType = {'paper': 'eps',
'presentation': 'png'
}
from lesion_analysis import LesionAnalysis
invivo_path = '/luscinia/ProstateStudy/invivo'
from IPython.display import display, Image
import matplotlib.pyplot as plt
class countTotalPercent:
"""
calculate percentage (p) from 2 input ints, count (c) and total (t)
used for printed output below
"""
def __init__(self, c, t):
self.c = c
self.t = t
self.p = 100*float(c)/float(t)
# open LaTeX source file with variables to read into Results section directly so that they don't have to get manually updated
# for each subsequent re-anlaysis iteration
analysis_results = open('analysis_results.tex', 'w')
analysis_results.write('% DO NOT MANUALLY EDIT THIS FILE!\n')
analysis_results.write('% GENERATED BY arfi_histology_lesion_analysis.ipynb\n')
def write_analysis_result(varname, value, analysis_results=analysis_results):
"""
write a new LaTeX command with specified variable name and value
"""
if type(value) is float:
value = str(round(value, 1))
analysis_results.write('\\newcommand{\%s}{%s}\n' % (varname, value))
In [2]:
ValidPatientNums = []
for n, p in enumerate(range(56, 107)):
P = LesionAnalysis(p, invivo_path)
if P.valid:
ValidPatientNums.append(p)
print "%i of %i study subjects valid" % (len(ValidPatientNums), n+1)
print "Valid study subjects: %s" % ValidPatientNums
Index lesions were considered matched if they existed in the nearest-neighbor set of the histology-identified region. An exact region match is not presented since there is so much uncertainty with the exact region for both the ARFI image and the histology. There was at least one case that did not have an index lesion because the only PCA did not meet the threshold for clinical significance.
In [3]:
IndexMatch = []
IndexMiss = []
for p in ValidPatientNums:
P = LesionAnalysis(p, invivo_path)
if 'index' in P.histology:
if P.histology['index'] is not None:
if P.index_match['nn']:
IndexMatch.append((p, P.histology['index']['location']))
else:
IndexMiss.append((p, P.histology['index']['location']))
ARFI_Index = countTotalPercent(len(IndexMatch), len(IndexMatch)+len(IndexMiss))
print "ARFI nearest-neighbor index PCA sensitivity: %i/%i (%.f%%)" % (ARFI_Index.c, ARFI_Index.t, ARFI_Index.p)
write_analysis_result('arfiIndexSensitivity', ARFI_Index.p)
print "ARFI-detected index PCA study subjects: %s" % IndexMatch
print "ARFI-missed study subjects: %s" % IndexMiss
# posterior index lesion sub-analysis
TotalIndexPosteriorMatch = len([i for i, j in enumerate(IndexMatch) if j[1] == 'posterior'])
TotalIndexPosteriorMiss = len([i for i, j in enumerate(IndexMiss) if j[1] == 'posterior'])
TotalIndexPosterior = TotalIndexPosteriorMatch + TotalIndexPosteriorMiss
ARFI_posterior = countTotalPercent(TotalIndexPosteriorMatch, TotalIndexPosterior)
print "ARFI POSTERIOR index PCA lesion sensitivity: %i/%i (%.f%%)" % (ARFI_posterior.c, ARFI_posterior.t, ARFI_posterior.p)
write_analysis_result('arfiPosteriorIndexSensitivity', ARFI_posterior.p)
# anterior index lesion sub-analysis
TotalIndexAnteriorMatch = len([i for i, j in enumerate(IndexMatch) if j[1] == 'anterior'])
TotalIndexAnteriorMiss = len([i for i, j in enumerate(IndexMiss) if j[1] == 'anterior'])
TotalIndexAnterior = TotalIndexAnteriorMatch + TotalIndexAnteriorMiss
ARFI_anterior = countTotalPercent(TotalIndexAnteriorMatch, TotalIndexAnterior)
print "ARFI ANTERIOR index PCA lesion sensitivity: %i/%i (%.f%%)" % (ARFI_anterior.c, ARFI_anterior.t, ARFI_anterior.p)
write_analysis_result('arfiAnteriorIndexSensitivity', ARFI_anterior.p)
# generate some pie charts
labels = ['ARFI-Localized Posterior', 'ARFI-Localized Anterior', 'ARFI-Not Localized Anterior', 'ARFI-Not Localized Posterior']
pie_colors = {'presentation': ['orchid', 'sage', 'sage', 'orchid'],
'paper': ['lightgray', 'darkgray', 'darkgray', 'lightgray']
}
colors = pie_colors[graphicsMode]
plt.figure(figsize=(12,12))
slices = [ARFI_posterior.c, ARFI_anterior.c, ARFI_anterior.t-ARFI_anterior.c, ARFI_posterior.t-ARFI_posterior.c]
patches, texts = plt.pie(slices, colors=colors)
plt.axis('equal')
plt.title('ARFI Imaging Index Lesion\nNearest-Neighbor Localization', fontsize=24, fontweight='bold')
for i in range(0,len(texts)):
texts[i].set_fontsize(18)
patches[2].set_hatch('x')
patches[3].set_hatch('x')
plt.legend(patches, labels, fontsize=18, loc='upper center', frameon=False, bbox_to_anchor=(0.525, 0.95))
#plt.text(-0.5, 0.4, '79% of all index lesions detected', fontsize=18, color='black')
plt.text(-0.5, 0.3, '%.1f%% of posterior index lesions localized' % ARFI_posterior.p, fontsize=18, color='black')
plt.annotate('%.f%% of anterior index lesions\nlocalized' % ARFI_anterior.p,
xy=(0.075, -0.55),
xytext=(-0.9,-0.1),
fontsize=18,
arrowprops=dict(facecolor='black'))
savefig('ARFI_IndexLesionSensitivity.%s' % saveFigType[graphicsMode], dpi=600)
In [4]:
for p in [i for i,j in IndexMiss]:
montage = '%s/Patient%s/Histology/Images/montage.png' % (invivo_path, p)
print montage
display(Image(filename=montage))
In [5]:
ClinSigMatch = []
ClinSigMatchCases = []
ClinSigMissCases = []
ClinSigMiss = []
ClinSigMatchIOS = []
ClinSigMissIOS = []
for p in ValidPatientNums:
P = LesionAnalysis(p, invivo_path)
if P.clin_sig_match is not None:
for yay, locat, ios in P.clin_sig_match:
if yay is True:
ClinSigMatch.append(locat)
ClinSigMatchCases.append(p)
ClinSigMatchIOS.append(ios)
elif yay is False:
ClinSigMissCases.append(p)
ClinSigMiss.append(locat)
ClinSigMissIOS.append(ios)
ClinSigMatchCases = set(ClinSigMatchCases)
ClinSigMissCases = set(ClinSigMissCases)
ARFItotal = len(ClinSigMatch) + len(ClinSigMiss)
ARFIclinsig = countTotalPercent(len(ClinSigMatch), ARFItotal)
ARFIposterior = countTotalPercent(len([i for i in ClinSigMatch if i == 'posterior']), len(ClinSigMatch))
ARFIanterior = countTotalPercent(len([i for i in ClinSigMatch if i == 'anterior']), len(ClinSigMatch))
ARFImissposterior = countTotalPercent(len([i for i in ClinSigMiss if i == 'posterior']), len(ClinSigMiss))
ARFImissanterior = countTotalPercent(len([i for i in ClinSigMiss if i == 'anterior']), len(ClinSigMiss))
# print and write interesting results to include in the manuscript
print "%i/%i (%.f%%) of read lesions were clinically significant" % \
(ARFIclinsig.c, ARFIclinsig.t, ARFIclinsig.p)
write_analysis_result('arfiClinSig', ARFIclinsig.p)
print "\tStudy subjects: %s" % ClinSigMatchCases
print "\t%i/%i (%.f%%) of read clinically-significant lesions were posterior" % \
(ARFIposterior.c, ARFIposterior.t, ARFIposterior.p)
write_analysis_result('arfiClinSigPosterior', ARFIposterior.p)
print "\t%i/%i (%.f%%) of read clinically-significant lesions were anterior" % \
(ARFIanterior.c, ARFIanterior.t, ARFIanterior.p)
write_analysis_result('arfiClinSigAnterior', ARFIanterior.p)
print "\t%i/%i (%.f%%) of read clinically-insignificant lesions were posterior" % \
(ARFImissposterior.c, ARFImissposterior.t, ARFImissposterior.p)
write_analysis_result('arfiClinInsigPosterior', ARFImissposterior.p)
print "\t%i/%i (%.f%%) of read clinically-insignificant lesions were anterior" % \
(ARFImissanterior.c, ARFImissanterior.t, ARFImissanterior.p)
write_analysis_result('arfiClinInsigAnterior', ARFImissanterior.p)
print ClinSigMatchIOS
print ClinSigMissIOS
In [6]:
ARFIatrophy = []
ARFIbph = []
ARFI_FalsePositive = []
NoHistLesion = []
for p in ValidPatientNums:
P = LesionAnalysis(p, invivo_path)
if P.false_positive:
ARFI_FalsePositive.append([P.pnum, P.false_positive])
print ARFI_FalsePositive
ARFI_ClinInsigPCA = [z for x, y in ARFI_FalsePositive for z in y if 'pca' in z]
if not ARFI_ClinInsigPCA:
ARFI_ClinInsigPCA = None
print "Number of non-clinically significant PCA: %s" % len(ARFI_ClinInsigPCA)
for i, j in ARFI_FalsePositive:
if 'atrophy' in j:
ARFIatrophy.append(i)
elif 'bph' in j:
ARFIbph.append(i)
if not ARFIatrophy:
ARFIatrophy = None
print "Number of study subjects with ATROPHY in ROS: %s" % len(ARFIatrophy)
if not ARFIbph:
ARFIbph = None
print "Number of study subjects with BPH in ROS: %s" % ARFIbph
for i, j in ARFI_FalsePositive:
if 'no lesion' in j:
NoHistLesion.append(i)
if not NoHistLesion:
NoHistLesion = None
print "No histologic lesions: %s" % NoHistLesion
In [7]:
for i in ARFIatrophy:
montage = '%s/Patient%s/Histology/Images/montage.png' % (invivo_path, i)
print montage
display(Image(filename=montage))
In [8]:
# lets make another pie chart for the ARFI ROSs
labels = ['Clinically-Significant PCa', 'Clinically-Insignificant PCa', 'Atrophy']
ros_colors = {'paper': ['lightgray', 'darkgray', 'dimgray'],
'presentation': ['orchid', 'plum', 'coral']
}
colors = ros_colors[graphicsMode]
plt.figure(figsize=(12,12))
slices = [ARFIclinsig.c, len(ARFI_ClinInsigPCA), len(ARFIatrophy)]
patches, texts, autotexts = plt.pie(slices, colors=colors, autopct='%1.1f%%')
plt.axis('equal')
plt.title('ARFI Regions of Suspicion', fontsize=24, fontweight='bold')
for i in range(0,len(texts)):
texts[i].set_fontsize(18)
for i in range(0,len(autotexts)):
autotexts[i].set_fontsize(18)
plt.legend(patches, labels, fontsize=18, loc='upper center', frameon=False, bbox_to_anchor=(0.525, 0.95))
# TODO: IOS histogram in clin-sig and clin-insig slices
a = plt.axes([0.285, 0.525, 0.1, 0.1], axisbg='white')
n, b, p = hist(ClinSigMatchIOS, rwidth=2.0, color='black')
xlabel('IOS')
ylabel('n')
setp(a, xlim=(0.5, 3.5), ylim=(0, 10), xticks=[1, 2, 3])
a = plt.axes([0.675, 0.235, 0.1, 0.1], axisbg='white')
n, b, p = hist(ClinSigMissIOS, rwidth=2.0, color='black')
xlabel('IOS')
ylabel('n')
setp(a, xlim=(0.5, 3.5), ylim=(0, 5), xticks=[1, 2, 3])
savefig('ARFI_ROS.%s' % saveFigType[graphicsMode], dpi=600)
In [9]:
ClinSigDetected = []
for p in ValidPatientNums:
P = LesionAnalysis(p, invivo_path)
if P.clin_sig_sensitivity:
ClinSigDetected.append(P.clin_sig_sensitivity)
histtotal = len([j for i in ClinSigDetected for j in i])
histclinsig = countTotalPercent(len([j for i in ClinSigDetected for j in i if j[0]]), histtotal)
histposterior = countTotalPercent(len([j for i in ClinSigDetected for j in i if j[1] == 'posterior']), histtotal)
histanterior = countTotalPercent(len([j for i in ClinSigDetected for j in i if j[1] == 'anterior']), histtotal)
histPosteriorMatch = countTotalPercent(len([j for i in ClinSigDetected for j in i if j[0] and j[1] == 'posterior']), histposterior.c)
histAnteriorMatch = countTotalPercent(len([j for i in ClinSigDetected for j in i if j[0] and j[1] == 'anterior']), histanterior.c)
print "%i/%i (%.f%%) of clinically-significant histology lesions were detected" % (histclinsig.c, histclinsig.t, histclinsig.p)
write_analysis_result('histClinSigDetect', histclinsig.p)
print "\tLocations: %i/%i (%.f%%) posterior, %i/%i (%.f%%) anterior" % \
(histposterior.c, histposterior.t, histposterior.p,histanterior.c, histanterior.t, histanterior.p)
write_analysis_result('histClinSigDetectPosterior', histposterior.p)
write_analysis_result('histClinSigDetectAnterior', histanterior.p)
print "\t%i/%i (%.f%%) of posterior clinically-significant lesions ARFI-detected" % (histPosteriorMatch.c, histPosteriorMatch.t, histPosteriorMatch.p)
print "\t%i/%i (%.f%%) of anterior clinically-significant lesions ARFI-detected" % (histAnteriorMatch.c, histAnteriorMatch.t, histAnteriorMatch.p)
write_analysis_result('histPosteriorMatch', histPosteriorMatch.p)
write_analysis_result('histAnteriorMatch', histAnteriorMatch.p)
# lets make another pie chart for the clinically-significant lesions
labels = ['ARFI-Detected Posterior', 'ARFI-Detected Anterior', 'ARFI-Missed Anterior', 'ARFI-Missed Posterior']
colors = pie_colors[graphicsMode]
plt.figure(figsize=(12,12))
slices = [histPosteriorMatch.c, histAnteriorMatch.c, histAnteriorMatch.t-histAnteriorMatch.c, histPosteriorMatch.t-histPosteriorMatch.c]
patches, texts = plt.pie(slices, colors=colors)
plt.axis('equal')
plt.title('ARFI Imaging Detection of\nClinically-Significant PCa Lesions', fontsize=24, fontweight='bold')
for i in range(0,len(texts)):
texts[i].set_fontsize(18)
#for i in range(0,len(autotexts)):
# autotexts[i].set_fontsize(18)
patches[2].set_hatch('x')
patches[3].set_hatch('x')
plt.legend(patches, labels, fontsize=18, frameon=False, bbox_to_anchor=(0.75, 0.95))
#plt.text(-0.75, 0.4, '%.f%% of all clinically-significant PCa lesions detected' % histclinsig.p, fontsize=18, color='black')
plt.text(-0.75, 0.3, '%.f%% of posterior clinically-significant PCa lesions detected' % histPosteriorMatch.p, fontsize=18, color='black')
plt.annotate('%.f%% of anterior clinically-\nsignificant PCa lesions\ndetected' % histAnteriorMatch.p,
xy=(-0.4, -0.6),
xytext=(-0.9,-0.1),
fontsize=18,
arrowprops=dict(facecolor='black'))
savefig('HistClinSigDetection.%s' % saveFigType[graphicsMode], dpi=600)
In [10]:
analysis_results.close()
In [11]:
%%bash
cat analysis_results.tex
In [ ]: