In [12]:
import sys
sys.path.append('/Users/shams/logic-lab/TextPreprocessing/')
import __preprocessing as pre
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
import numpy as np
In [7]:
DATAFILE = '/Users/shams/querent-datascience/Diagnosis_ICD/data/icd_10_2017.csv'
COLUMNS = ['icd_code', 'xyz', 'short_desc', 'long_desc']
HEADERLINE = None
DOCUMENT_COLUMN = 'long_desc'
df_data = pd.DataFrame.from_csv(DATAFILE, header=HEADERLINE)
if len(COLUMNS):
df_data.columns = COLUMNS
df_data.head()
Out[7]:
icd_code
xyz
short_desc
long_desc
0
1
A00
0
Cholera
Cholera
2
A000
1
Cholera due to Vibrio cholerae 01, biovar chol...
Cholera due to Vibrio cholerae 01, biovar chol...
3
A001
1
Cholera due to Vibrio cholerae 01, biovar eltor
Cholera due to Vibrio cholerae 01, biovar eltor
4
A009
1
Cholera, unspecified
Cholera, unspecified
5
A01
0
Typhoid and paratyphoid fevers
Typhoid and paratyphoid fevers
In [28]:
df_data[DOCUMENT_COLUMN] = df_data[DOCUMENT_COLUMN].apply(pre._text_preprocessing)
documents = df_data[DOCUMENT_COLUMN].tolist()
In [29]:
vocab = list(set([i for sub_doc in documents for i in sub_doc.strip().split()]))
vectorizer = TfidfVectorizer(
stop_words=None,
norm=None,
min_df=0,
sublinear_tf=True,
token_pattern=r'(?u)\b\w+\b',
vocabulary=vocab
)
In [30]:
def tf_idf(documents):
return vectorizer.fit_transform(documents)
In [31]:
x = tf_idf(documents=documents)
In [32]:
vectorizer.vocabulary
Out[32]:
['scombroid',
'umbilicus',
'volleyball',
'predominant',
'continua',
'trichinellosis',
'palpitations',
'papillomata',
'institutional',
'aneurysms',
'capitate',
'respirator',
'chemicals',
'need',
'volar',
'dorsopathies',
'fascia',
'sibling',
'gastroenteritis',
'chagas',
'tonic',
'opening',
'tarantula',
'blepharophimosis',
'bell',
'systems',
'erythrocytosis',
'undescended',
'ecchymoses',
'erb',
'sequestration',
'divorce',
'interphalangeal',
'clavicle',
'aldrich',
'provider',
'weapons',
'fibroplasia',
'episodic',
'antiestrogens',
'postcricoid',
'infectiosum',
'manic',
'bizarre',
'apparent',
'stimulator',
'forced',
'explosives',
'node',
'dislocation',
'slow',
'injury',
'railway',
'due',
'lymphomas',
'paraurethral',
'cryptosporidiosis',
'thromboembolism',
'antiparkinsonism',
'endosseous',
'angioimmunoblastic',
'panlobular',
'gila',
'adolescence',
'landscaping',
'trichodermal',
'comitancies',
'landlord',
'mucinosa',
'deaf',
'erythrasma',
'proliferative',
'pole',
'childhood',
'official',
'concentric',
'chain',
'rotation',
'parachutist',
'skateboarder',
'fancier',
'thromboangiitis',
'richardson',
'continuous',
'areolar',
'live',
'paintball',
'defiant',
'progressiva',
'subdermal',
'curvature',
'eltor',
'instructor',
'conus',
'lumbosacral',
'under',
'cluster',
'nephrogenic',
'cholesteatosis',
'dioxide',
'arthus',
'smoking',
'cortex',
'ruber',
'aspirin',
'currently',
'pattern',
'bradycardia',
'spatial',
'abnormality',
'toes',
'wear',
'aural',
'filariases',
'localization',
'pigmentosum',
'bronchitis',
'van',
'500',
'old',
'atlanto',
'weight',
'brucellosis',
'bumping',
'kaposi',
'rubella',
'exhaust',
'condyle',
'vulval',
'reticular',
'schizoaffective',
'abuse',
'sided',
'exfoliation',
'disjunction',
'athetoid',
'atmospheric',
'cranial',
'hip',
'concerns',
'conjunctivitis',
'previa',
'mass',
'ankle',
'coracoid',
'chest',
'thoracoscopic',
'chromosome',
'malignant',
'religious',
'phthiriasis',
'bicondylar',
'catheter',
'bullous',
'tetrachloroethylene',
'otosclerosis',
'emt',
'game',
'niemann',
'myiasis',
'hypersplenism',
'impingement',
'ichthyosis',
'ocular',
'relief',
'adults',
'jakob',
'magnum',
'previous',
'dune',
'lactation',
'paraphilia',
'periprosthetic',
'transvestic',
'pox',
'generalized',
'c6',
'penetration',
'hydroxylysine',
'alcohols',
'stepsibling',
'distortions',
'vaginitis',
'er',
'megacolon',
'recluse',
'1',
'bypass',
'sick',
'phenylketonuria',
'coarctation',
'projectile',
'helping',
'gianotti',
'antithrombotics',
'cyclosporiasis',
'nerves',
'wood',
'amnesia',
'monoplegia',
'movie',
'firework',
'choanal',
'uncomplicated',
'tophus',
'correspondence',
'curve',
'pneumoconiosis',
'acyl',
'myopathy',
'malunion',
'cor',
'childbirth',
'necrosis',
'hemoptysis',
'cyclical',
'inferior',
'effusion',
'gummata',
'pylorus',
'macroglossia',
'percent',
'supravesical',
'numbers',
'osteopathy',
'protozoal',
'ulna',
'promyelocytic',
'bursopathies',
'pregnancy',
'streptococcus',
'tubal',
'hellp',
'calcitonin',
'rolling',
'intracapsular',
'barr',
'gram',
'margins',
'vasodilators',
'staining',
'preparations',
'attempt',
'morgagnian',
'gigantism',
'cold',
'hydronephrosis',
'overactive',
'wild',
'economic',
'7',
'rhythmic',
'ascariasis',
'derailment',
'oliguria',
'scheie',
'phobia',
'gait',
'papilledema',
'miliary',
'glycols',
'ventricles',
'antifungal',
'muscles',
'member',
'johnson',
'bronchus',
'aftercare',
'days',
'interior',
'achilles',
'5q',
'28',
'postsurgical',
'yacht',
'cavernosum',
'truck',
'psoriatic',
'caregiver',
'epilepsy',
'allergic',
'suspected',
'spondyloepiphyseal',
'nucleoside',
'k',
'hordeolum',
'stevens',
'peritoneum',
'moniliformis',
'01',
'gallbladder',
'sandhoff',
'infected',
'reovirus',
'transmetacarpal',
'dam',
'pseudosarcomatous',
'ghost',
'far',
'sickle',
'erythroblastopenia',
'edentulous',
'serous',
'rumination',
'tarsometatarsal',
'c5',
'osteoarthritis',
'dysphoric',
'chills',
'methadone',
'cercarial',
'rapidly',
'aplasia',
'tachycardia',
'thirds',
'wiskott',
'striatus',
'spotting',
'hiccough',
'lobomycosis',
'calf',
'fibroadenosis',
'parkinson',
'calcaneofibular',
'incarcerated',
'sunlight',
'anhydrotic',
'klippel',
'threatening',
'reattachment',
'woodworking',
'cephalgias',
'twist',
'capillary',
'foot',
'lymphoid',
'displacement',
'ventilator',
'specifically',
'tentorial',
'mastoid',
'interference',
'subluxation',
'thyrotoxicosis',
'dsap',
'connections',
'using',
'gross',
'pigmentations',
'homologues',
'injectable',
'cecoureterocele',
'gastric',
'pierini',
'nile',
'strabismus',
'protozoa',
'sea',
'business',
'restraint',
'labyrinthitis',
'lepromatous',
'bandemia',
'infiltrative',
'lactating',
'pick',
'predominantly',
'away',
'nasofrontal',
'21',
'tidal',
'melena',
'pontiac',
'sphingolipidosis',
'syncope',
'iii',
'cerebellum',
'constitutional',
'methylglutaconic',
'penis',
'reverse',
'dilution',
'neuron',
'patellae',
'overfill',
'flatulence',
'pancreatic',
'thrown',
'overload',
'synovium',
'asymmetry',
'myhre',
'necatoriasis',
'rig',
'typhus',
'boarder',
'somatization',
'external',
'chemistry',
'1249',
'wilson',
'slaughter',
'electrical',
'delirium',
'nursemaid',
'paranoid',
'sirs',
'grafts',
'presentation',
'triceps',
'wave',
'miscellaneous',
'focal',
'unrepairable',
'artery',
'dermatosis',
'anonymous',
'manhandling',
'traumatica',
'trichuriasis',
'best',
'false',
'stereopsis',
'periventricular',
'dysreflexia',
'epithelium',
'wuchereria',
'arenaviral',
'fat',
'foregut',
'arm',
'incarceration',
'postmenopausal',
'chondrodysplasia',
'septo',
'branch',
'cowpox',
'handgun',
'curb',
'perfusion',
'hepatic',
'rough',
'places',
'subway',
'condensans',
'antithyroid',
'ureter',
'but',
'breath',
'leukonychia',
'rider',
'transsexualism',
'euthyroid',
'rearrangement',
'choroideremia',
'gottron',
'mitotic',
'penicillin',
'mechanical',
'reconstitution',
'glaucoma',
'consult',
'mother',
'retracted',
'ruled',
'hyposplenism',
'fold',
'spondylitis',
'luteum',
'aortopulmonary',
'string',
'modulators',
'posteruptive',
'dander',
'digging',
'krabbe',
'guillain',
'given',
'purine',
'colitis',
'c',
'ambient',
'noted',
'clear',
'linear',
'neoplasm',
'matter',
'repeat',
'dupuytren',
'foreskin',
'neurosyphilis',
'isosporiasis',
'delivered',
'building',
'conduction',
'utero',
'craniorachischisis',
'chlamydiae',
'mole',
'cigarettes',
'necrotizing',
'atrophicae',
'psychophysical',
'blighted',
'general',
'cystocele',
'tinnitus',
'meningeal',
'vitreous',
'exercises',
'steroids',
'psychological',
'aesthetic',
'products',
'ataxia',
'accommodative',
'narcissistic',
'enuresis',
'paravaccinia',
'cannabis',
'common',
'explanation',
'boiler',
'astrovirus',
'friendly',
'anhydrase',
'quinolones',
'cesarean',
'erythroid',
'microgenia',
'parieto',
'coronoid',
'underdevelopment',
'tsutsugamushi',
'hepatis',
'kyphoscoliotic',
'oligomenorrhea',
'games',
'process',
'antecedent',
'nyong',
'asymptomatic',
'hepatopulmonary',
'nightwear',
'wound',
'suicide',
'bladder',
'erysipelothrix',
'supravalvular',
'bursopathy',
'telogen',
'little',
'lability',
'new',
'desire',
'deforming',
'transfusions',
'campsite',
'tularemia',
'discoid',
'estrogens',
'genitals',
'mopping',
'substance',
'rotator',
'walked',
'myeloproliferative',
'spider',
'ebstein',
'undue',
'lesser',
'pay',
'black',
'radiculomyelitis',
'stage',
'ige',
'fecal',
'xerophthalmic',
'tobogganing',
'catalase',
'cyclic',
'blizzard',
'murmur',
'lungs',
'perforated',
'drip',
'phakomatosis',
'south',
'eccrine',
'weakness',
'sensations',
'mucolipidosis',
'kernicterus',
'ice',
'diaper',
'domestic',
'anemias',
'lacrosse',
'hydrocarbons',
'epidermal',
'bronchopneumonia',
'underfill',
'of',
'postconcussional',
'recent',
'steps',
'polyorchism',
'closed',
'friend',
'nystagmus',
'safe',
'80',
'state',
'vacuum',
'mid',
'pre',
'partner',
'memory',
'antithrombotic',
'methemoglobinemias',
'antidiabetic',
'streptobacillosis',
'factors',
'peripapillary',
'tracheostomy',
'hypopyon',
'antepartum',
'operating',
'defibrillator',
'niacin',
'musculoskeletal',
'malingerer',
'ramus',
'significant',
'its',
'underwater',
'processing',
'allograft',
'dagger',
'ossicles',
'ankylosing',
'chronicus',
'succinimides',
'torticollis',
'airgun',
'trochanteric',
'strain',
'stock',
'drum',
'cheerleading',
'visceral',
'heartburn',
'collecting',
'supporting',
'prostatodynia',
'apophyseal',
'amphibians',
'sclerosus',
'blindness',
'self',
'vibrio',
'symptomatic',
'health',
'left',
'azoospermia',
'spine',
'swan',
'carcinoid',
'urethrorectal',
'frontotemporal',
'pustular',
'pyemic',
'crepitant',
'hpv',
'phrenic',
'an',
'handle',
'hymenal',
'trisomy',
'hyponatremia',
'40',
'hypermetropia',
'profound',
'hyperemesis',
'progestogens',
'actinomycetoma',
'burns',
'nocardiosis',
'over',
'falciparum',
'malaria',
'circulating',
'linked',
'canis',
'metalworking',
'cholinergic',
'restorative',
'subinvolution',
'gain',
'hyperkeratosis',
'merrf',
'cells',
'attachment',
'balanitis',
'stepmother',
'law',
'dysphagia',
'canthus',
'louis',
'wart',
'endocrine',
'monkeypox',
'weights',
'paraparesis',
'profunda',
'spermatocele',
'bitot',
'ossification',
'hemorrhages',
'angiostrongyliasis',
'internum',
'enteropathogenic',
'direct',
'antenatal',
'skills',
'implantation',
'thrombocytopenic',
'autosomes',
'bulbous',
'restoration',
'cholesterol',
'hydantoin',
'city',
'skating',
'unequal',
'sequelae',
'dacryoadenitis',
'pupillary',
'neuronal',
'poorly',
'disability',
'dissection',
'hypertrichosis',
'klinefelter',
'exostosis',
'australis',
'western',
'continuity',
'jewelry',
'congestion',
'arrhythmias',
'nasolacrimal',
'multiplex',
'chiefly',
'terrorist',
'staphyloma',
'necrobiosis',
'toe',
'prowazekii',
'ligament',
'storm',
'torus',
'commissure',
'hidradenitis',
'indigenous',
'descriptions',
'canoeing',
'tool',
'infants',
'cocaine',
'carbamate',
'analogues',
'percussion',
'scid',
'tongue',
'arts',
'ptosis',
'otorrhagia',
'keloid',
'examinations',
'removed',
'inflamed',
'nutrient',
'sayre',
'inexplicable',
'menstruation',
'intraperitoneal',
'papillitis',
'intrasphincteric',
'elephantiasis',
'infundibular',
'insect',
'gastrostomy',
'deletions',
'epidural',
'alkalis',
'instrument',
'syngamiasis',
'chromhidrosis',
'golf',
'sleeping',
'latent',
'from',
'detachments',
'hypodermic',
'termination',
'ototoxic',
'di',
'percreta',
'1st',
'jaws',
'control',
'grilling',
'carcinomas',
'airway',
'trapped',
'acne',
'relapsing',
'inclusion',
'cerebrovascular',
'pseudobulbar',
'olfactory',
'nitidus',
'callosities',
'decline',
'exogenous',
'atrium',
'vasculogenic',
'coral',
'aversion',
'sarcoidosis',
'plaque',
'dysautonomia',
'waterhouse',
't7',
'retrobulbar',
'pericardium',
'beriberi',
'lactamase',
'transmission',
'neck',
'forest',
'gasoline',
'observation',
'neurostimulator',
'sinusitis',
'heterotropia',
'operation',
'postnasal',
'behind',
'sadomasochism',
'neuronitis',
'property',
'multiparity',
'histoplasmosis',
'aminoderivatives',
'all',
'exocervix',
'attack',
'epidemic',
'hypercholesterolemia',
'delusions',
'vasoconstriction',
'intermittent',
'folliculitis',
'demyelinating',
'german',
'marginal',
'heroin',
'perineum',
'protein',
'wide',
'centrally',
'cystic',
'glutaric',
'scars',
'opera',
'dysmorphism',
'pericarditis',
'osteochondritis',
'specified',
'regulation',
'reflux',
'vaginally',
'properly',
'49',
'moving',
'confirm',
'orchitis',
'15',
'mucocele',
'obesity',
'rafting',
'psychomotor',
'caloric',
'birds',
'sextuplets',
'thyroiditis',
'41',
'emergency',
'household',
'balantidiasis',
'cystoid',
'vesicant',
'friderichsen',
'up',
'epiphora',
'dilation',
'physiological',
'vesical',
'membranes',
'pollution',
'inflammations',
'designated',
'pool',
'postlaminectomy',
'pesticides',
'ulcer',
'senile',
'accessory',
'role',
'psychosocial',
'charge',
'unisystemic',
'grocery',
'head',
'mental',
'strychnine',
'mitochondrial',
'margin',
'onset',
'cartilagenous',
'states',
'ligation',
'lymphocytic',
'clubfoot',
'greases',
'reductase',
'valga',
'lifestyle',
'kitchen',
'sledding',
'tank',
'precordial',
'furuncle',
'au',
'retroperitoneal',
'synovial',
'efa',
'pillar',
'assisted',
'glenoid',
'xeroderma',
'mucinosis',
'1500',
'carpus',
'rickets',
'chromoabnormalities',
'gas',
'incontinentia',
'overlapping',
'bat',
'passenger',
'subcortical',
'fulminating',
'without',
'infrared',
'enlarged',
'sickness',
'universalis',
'extremely',
'detonation',
'peliosis',
'acetabuli',
'tornado',
'69',
'inattentive',
'receptor',
'pinched',
'shigellosis',
'pressurized',
'triploidy',
'medicaments',
'steroid',
'cycle',
'skeleton',
'chorioretinitis',
'dna',
'phosphorus',
'hurst',
'1749',
'disulfide',
'centric',
'uteri',
'underdosing',
'dyspareunia',
'weber',
'run',
'akathisia',
'staphylococcal',
...]
In [42]:
tf_idf = pd.DataFrame({'word': vectorizer.vocabulary, 'idf': vectorizer.idf_}, index=None)
tf_idf.sort_values('idf')
Out[42]:
idf
word
602
1.423939
of
5359
1.947641
encounter
1039
2.205512
unspecified
5666
2.441358
with
6068
2.458027
subsequent
4254
2.533637
fracture
2348
2.607479
other
2734
2.712999
for
663
2.790833
left
6033
2.805700
right
5748
2.841007
and
6534
2.890157
initial
5724
3.045978
non
4661
3.049199
sequela
7036
3.086962
displaced
3941
3.240511
or
2103
3.530065
open
6705
3.608092
type
4779
3.634326
in
51
3.700628
injury
6144
3.744582
healing
6109
3.822127
to
3821
4.000550
lower
2317
4.017833
level
5074
4.044331
at
4619
4.067648
by
1210
4.100713
injured
2459
4.176680
body
607
4.200721
closed
2215
4.226160
shaft
...
...
...
4537
11.756103
chyluria
1244
11.756103
125
1242
11.756103
snoring
4544
11.756103
najjar
1261
11.756103
inadvertent
4458
11.756103
squash
1262
11.756103
bagassosis
4499
11.756103
caruncle
1281
11.756103
leukoencephalitis
4461
11.756103
hirschsprung
1277
11.756103
pubocervical
4464
11.756103
typing
1276
11.756103
obliterans
4469
11.756103
emotions
4471
11.756103
guidance
4472
11.756103
vomit
4473
11.756103
anoxic
4500
11.756103
delusional
4475
11.756103
ectodermal
4477
11.756103
cyanosis
1269
11.756103
rivalry
4483
11.756103
marginatum
1268
11.756103
enterocele
4488
11.756103
neuropathies
4493
11.756103
shortness
4496
11.756103
downhill
4497
11.756103
dyspepsia
7116
11.756103
robertson
3660
12.449251
+
6825
12.449251
-
7117 rows × 2 columns
In [43]:
tf_idf.loc[tf_idf.word=='acute']
Out[43]:
idf
word
4343
5.723017
acute
In [ ]:
Content source: shams-sam/logic-lab
Similar notebooks: