In [1]:
from bs4 import BeautifulSoup
import codecs, pandas as pd, cPickle as pickle, csv
Read in MedlinePlus health topic data
In [2]:
soup = BeautifulSoup(codecs.open('../data/mplus_topics_2014-11-04.xml','r','utf-8').read())
Read in MeSH thesaurus and condition data from CTTI
In [3]:
mesh_terms = {}
for row in codecs.open('../data/mesh_thesaurus.txt','r','utf-8').readlines():
row_id, mesh_id, mesh_term = row.strip().split('|')
if mesh_term not in mesh_terms: mesh_terms[mesh_term] = []
mesh_terms[mesh_term].append(mesh_id)
conditions = set([row.strip().split('|')[2] for row in codecs.open('../data/condition_browse.txt','r','utf-8').readlines()])
interventions = set([row.strip().split('|')[2] for row in codecs.open('../data/intervention_browse.txt','r','utf-8').readlines()])
Create lookup of lay term to MeSH term where they exist
In [5]:
mesh_lookup = {}
others = []
for t in soup.find_all("health-topic",language="English"):
cur_mesh = t.find("mesh-heading").descriptor.text
if cur_mesh in conditions:
if cur_mesh not in mesh_lookup: mesh_lookup[cur_mesh] = set()
mesh_lookup[cur_mesh] |= set([t.attrs["title"]] + [a.text for a in t.find_all("also-called")]) # can take out "also-called" tags for a tigher match
else:
others.append(cur_mesh)
Clean up lookup dictionary
In [6]:
for m in mesh_lookup.keys():
cur_set = mesh_lookup[m].copy()
for s in mesh_lookup[m]:
if m.lower() == s.lower(): cur_set -= set([s])
if len(cur_set) == 0:
del(mesh_lookup[m])
else:
mesh_lookup[m] = cur_set
In [9]:
pickle.dump(mesh_lookup,open('../data/mesh_synonyms.pickle','wb'))
In [7]:
mesh_lookup
Out[7]:
{u'AIDS-Related Opportunistic Infections': {u'HIV/AIDS and Infections'},
u'Abnormalities, Drug-Induced': {u'Pregnancy and Medicines'},
u'Abortion, Spontaneous': {u'Miscarriage'},
u'Acne Vulgaris': {u'Acne'},
u'Adrenal Gland Diseases': {u'Adrenal Gland Disorders'},
u'Adrenal Gland Neoplasms': {u'Adrenal Gland Cancer'},
u'Airway Obstruction': {u'Choking'},
u'Alcohol Drinking': {u'Alcohol', u'Underage Drinking'},
u'Alcoholism': {u'Alcoholism and Alcohol Abuse'},
u'Alzheimer Disease': {u"Alzheimer's Caregivers"},
u'Anemia, Aplastic': {u'Aplastic Anemia'},
u'Anemia, Sickle Cell': {u'Sickle Cell Anemia'},
u'Aneurysm': {u'Aneurysms'},
u'Angina Pectoris': {u'Angina'},
u'Ankle Injuries': {u'Ankle Injuries and Disorders'},
u'Anus Diseases': {u'Anal Disorders'},
u'Anus Neoplasms': {u'Anal Cancer'},
u'Anxiety Disorders': {u'Anxiety'},
u'Arnold-Chiari Malformation': {u'Chiari Malformation'},
u'Arrhythmias, Cardiac': {u'Arrhythmia'},
u'Arthritis, Infectious': {u'Infectious Arthritis'},
u'Arthritis, Psoriatic': {u'Psoriatic Arthritis'},
u'Arthritis, Rheumatoid': {u'Rheumatoid Arthritis'},
u'Athletic Injuries': {u'Sports Injuries'},
u'Attention Deficit Disorder with Hyperactivity': {u'Attention Deficit Hyperactivity Disorder'},
u'Autistic Disorder': {u'Autism Spectrum Disorder'},
u'Autonomic Nervous System Diseases': {u'Autonomic Nervous System Disorders'},
u'Behcet Syndrome': {u"Behcet's Syndrome"},
u'Bell Palsy': {u"Bell's Palsy"},
u'Bile Duct Neoplasms': {u'Bile Duct Cancer'},
u'Bites and Stings': {u'Animal Bites'},
u'Blood Coagulation Disorders': {u'Bleeding Disorders'},
u'Blood Platelet Disorders': {u'Platelet Disorders'},
u'Body Weight Changes': {u'Body Weight'},
u'Bone Neoplasms': {u'Bone Cancer'},
u'Brachial Plexus Neuropathies': {u'Brachial Plexus Injuries'},
u'Brain Concussion': {u'Concussion'},
u'Brain Diseases, Metabolic, Inborn': {u'Genetic Brain Disorders'},
u'Brain Neoplasms': {u'Brain Cancer', u'Childhood Brain Tumors'},
u'Breast Neoplasms, Male': {u'Male Breast Cancer'},
u'Bronchial Diseases': {u'Bronchial Disorders'},
u'Bronchitis': {u'Acute Bronchitis'},
u'Bronchitis, Chronic': {u'Chronic Bronchitis'},
u'Callosities': {u'Corns and Calluses'},
u'Candidiasis': {u'Yeast Infections'},
u'Carcinoid Tumor': {u'Carcinoid Tumors'},
u'Carcinoma, Ductal, Breast': {u'Breast Cancer'},
u'Cardiomyopathies': {u'Cardiomyopathy'},
u'Carotid Artery Diseases': {u'Carotid Artery Disease'},
u'Cartilage Diseases': {u'Cartilage Disorders'},
u'Cat-Scratch Disease': {u'Cat Scratch Disease'},
u'Cerebellar Diseases': {u'Cerebellar Disorders'},
u'Cholelithiasis': {u'Gallstones'},
u'Chronic Disease': {u'Coping with Chronic Illness'},
u'Cicatrix': {u'Scars'},
u'Cleft Palate': {u'Cleft Lip and Palate'},
u'Cocaine-Related Disorders': {u'Cocaine'},
u'Coccidioidomycosis': {u'Valley Fever'},
u'Colitis, Ulcerative': {u'Ulcerative Colitis'},
u'Colonic Diseases, Functional': {u'Irritable Bowel Syndrome'},
u'Color Vision Defects': {u'Color Blindness'},
u'Colorectal Neoplasms': {u'Colorectal Cancer'},
u'Communicable Diseases': {u'Infectious Diseases'},
u'Communication Disorders': {u'Speech and Communication Disorders'},
u'Complex Regional Pain Syndromes': {u'Complex Regional Pain Syndrome'},
u'Condylomata Acuminata': {u'Genital Warts'},
u'Congenital Abnormalities': {u'Birth Defects'},
u'Conjunctivitis': {u'Pinkeye'},
u'Connective Tissue Diseases': {u'Connective Tissue Disorders'},
u'Contusions': {u'Bruises'},
u'Corneal Diseases': {u'Corneal Disorders'},
u'Craniocerebral Trauma': {u'Head Injuries', u'Traumatic Brain Injury'},
u'Craniofacial Abnormalities': {u'Facial Injuries and Disorders'},
u'Creutzfeldt-Jakob Syndrome': {u'Creutzfeldt-Jakob Disease'},
u'Crohn Disease': {u"Crohn's Disease"},
u'Cushing Syndrome': {u"Cushing's Syndrome"},
u'Cystitis, Interstitial': {u'Interstitial Cystitis'},
u'Deafness': {u'Hearing Disorders and Deafness'},
u'Death': {u'End of Life Issues'},
u'Deglutition Disorders': {u'Swallowing Disorders'},
u'Dental Caries': {u'Tooth Decay'},
u'Depression, Postpartum': {u'Postpartum Depression'},
u'Diabetes Mellitus': {u'Diabetes', u'Diabetes in Children and Teens'},
u'Diabetes Mellitus, Type 1': {u'Diabetes Type 1'},
u'Diabetes Mellitus, Type 2': {u'Diabetes Type 2'},
u'Diabetic Cardiomyopathies': {u'Diabetes Complications',
u'Diabetic Heart Disease'},
u'Diabetic Nephropathies': {u'Diabetic Kidney Problems'},
u'Diabetic Neuropathies': {u'Diabetic Nerve Problems'},
u'Diabetic Retinopathy': {u'Diabetic Eye Problems'},
u'Digestive System Diseases': {u'Digestive Diseases'},
u'Dislocations': {u'Shoulder Injuries and Disorders'},
u'Diverticulum, Colon': {u'Diverticulosis and Diverticulitis'},
u'Dizziness': {u'Dizziness and Vertigo'},
u'Dyspepsia': {u'Indigestion'},
u'Ear Diseases': {u'Ear Disorders'},
u'Embolism and Thrombosis': {u'Blood Clots'},
u'Emergencies': {u'Winter Weather Emergencies'},
u'Encopresis': {u'Bowel Incontinence'},
u'Endocrine System Diseases': {u'Endocrine Diseases'},
u'Enterocolitis, Pseudomembranous': {u'Clostridium Difficile Infections'},
u'Enuresis': {u'Bedwetting'},
u'Eosinophilia': {u'Eosinophilic Disorders'},
u'Eructation': {u'Gas'},
u'Erythema Infectiosum': {u'Fifth Disease'},
u'Escherichia coli Infections': {u'E. Coli Infections'},
u'Esophageal Diseases': {u'Esophagus Disorders'},
u'Esophageal Neoplasms': {u'Esophageal Cancer'},
u'Exanthema': {u'Rashes'},
u'Eye Neoplasms': {u'Eye Cancer'},
u'Eyelid Diseases': {u'Eyelid Disorders'},
u'Fatigue Syndrome, Chronic': {u'Chronic Fatigue Syndrome'},
u'Femoral Neck Fractures': {u'Hip Injuries and Disorders'},
u'Fibroma': {u'Benign Tumors'},
u'Finger Injuries': {u'Finger Injuries and Disorders'},
u'Fistula': {u'Fistulas'},
u'Folic Acid Deficiency': {u'Folic Acid'},
u'Food Hypersensitivity': {u'Food Allergy'},
u'Foot Diseases': {u'Foot Injuries and Disorders'},
u'Friedreich Ataxia': {u"Friedreich's Ataxia"},
u'Gait Disorders, Neurologic': {u'Walking Problems'},
u'Gallbladder Neoplasms': {u'Gallbladder Cancer'},
u'Gambling': {u'Compulsive Gambling'},
u'Gastroenteritis': {u'Norovirus Infections'},
u'Gastroesophageal Reflux': {u'GERD'},
u'Gastrointestinal Hemorrhage': {u'Gastrointestinal Bleeding'},
u'Gaucher Disease': {u"Gaucher's Disease"},
u'Genetic Diseases, Inborn': {u'Genetic Disorders'},
u'Giardiasis': {u'Giardia Infections'},
u'Gingival Diseases': {u'Gum Disease'},
u'Glucosephosphate Dehydrogenase Deficiency': {u'G6PD Deficiency'},
u'HIV Infections': {u'HIV/AIDS',
u'HIV/AIDS in Women',
u'Living with HIV/AIDS'},
u'Halitosis': {u'Bad Breath'},
u'Hand Injuries': {u'Hand Injuries and Disorders'},
u'Head and Neck Neoplasms': {u'Head and Neck Cancer'},
u'Hearing Disorders': {u'Hearing Problems in Children'},
u'Heart Arrest': {u'Cardiac Arrest'},
u'Heart Defects, Congenital': {u'Congenital Heart Defects'},
u'Heart Diseases': {u'Cardiac Rehabilitation',
u'Heart Disease in Women',
u'Heart Diseases--Prevention'},
u'Heat Stress Disorders': {u'Heat Illness'},
u'Helicobacter Infections': {u'Helicobacter Pylori Infections'},
u'Hemangioma': {u'Birthmarks'},
u'Hematologic Diseases': {u'Blood Disorders'},
u'Hemophilia A': {u'Hemophilia'},
u'Hemorrhage': {u'Bleeding'},
u'Hemorrhagic Fevers, Viral': {u'Hemorrhagic Fevers'},
u'Hepatolenticular Degeneration': {u'Wilson Disease'},
u'Hernia, Hiatal': {u'Hiatal Hernia'},
u'Heroin Dependence': {u'Heroin'},
u'Herpes Genitalis': {u'Genital Herpes'},
u'Herpes Labialis': {u'Cold Sores'},
u'Hiccup': {u'Hiccups'},
u'Huntington Disease': {u"Huntington's Disease"},
u'Hyperbilirubinemia': {u'Jaundice'},
u'Hyperlipoproteinemia Type IV': {u'Triglycerides'},
u'Hypersensitivity': {u'Allergy'},
u'Hypertension, Pregnancy-Induced': {u'High Blood Pressure in Pregnancy'},
u'Hypertension, Pulmonary': {u'Pulmonary Hypertension'},
u'Hypotension': {u'Low Blood Pressure'},
u'Hypotrichosis': {u'Hair Loss'},
u'Ileal Diseases': {u'Small Intestine Disorders'},
u'Ileal Neoplasms': {u'Intestinal Cancer'},
u'Infant, Newborn, Diseases': {u'Common Infant and Newborn Problems',
u'Uncommon Infant and Newborn Problems'},
u'Infant, Premature, Diseases': {u'Premature Babies'},
u'Infertility, Female': {u'Female Infertility'},
u'Infertility, Male': {u'Male Infertility'},
u'Influenza in Birds': {u'Bird Flu'},
u'Influenza, Human': {u'Flu'},
u'Intracranial Aneurysm': {u'Brain Aneurysm'},
u'Ischemic Attack, Transient': {u'Transient Ischemic Attack'},
u'Joint Diseases': {u'Joint Disorders'},
u'Kidney Diseases, Cystic': {u'Kidney Cysts'},
u'Kidney Failure, Chronic': {u'Kidney Failure'},
u'Kidney Neoplasms': {u'Kidney Cancer'},
u'Klinefelter Syndrome': {u"Klinefelter's Syndrome"},
u'Knee Injuries': {u'Knee Injuries and Disorders'},
u'Lacrimal Apparatus Diseases': {u'Tears'},
u'Language Disorders': {u'Speech and Language Problems in Children'},
u'Latex Hypersensitivity': {u'Latex Allergy'},
u'Leg Length Inequality': {u'Leg Injuries and Disorders'},
u'Leukemia': {u'Childhood Leukemia'},
u'Leukemia, Lymphoid': {u'Acute Lymphocytic Leukemia',
u'Chronic Lymphocytic Leukemia'},
u'Leukemia, Myelogenous, Chronic, BCR-ABL Positive': {u'Chronic Myeloid Leukemia'},
u'Leukemia, Myeloid, Acute': {u'Acute Myeloid Leukemia'},
u'Leukodystrophy, Metachromatic': {u'Leukodystrophies'},
u'Lice Infestations': {u'Lice'},
u'Listeriosis': {u'Listeria Infections'},
u'Liver Cirrhosis': {u'Cirrhosis'},
u'Liver Neoplasms': {u'Liver Cancer'},
u'Lung Diseases': {u'Pulmonary Rehabilitation'},
u'Lung Diseases, Interstitial': {u'Interstitial Lung Diseases'},
u'Lung Neoplasms': {u'Lung Cancer'},
u'Lupus Erythematosus, Discoid': {u'Lupus'},
u'Memory Disorders': {u'Memory'},
u'Meniere Disease': {u"Meniere's Disease"},
u'Metabolic Diseases': {u'Metabolic Disorders'},
u'Metabolic Syndrome X': {u'Metabolic Syndrome'},
u'Migraine Disorders': {u'Migraine'},
u'Mouth Diseases': {u'Mouth Disorders'},
u'Mouth Neoplasms': {u'Oral Cancer'},
u'Mucocutaneous Lymph Node Syndrome': {u'Kawasaki Disease'},
u'Muscle Cramp': {u'Muscle Cramps'},
u'Muscular Atrophy, Spinal': {u'Spinal Muscular Atrophy'},
u'Muscular Diseases': {u'Muscle Disorders'},
u'Muscular Dystrophies': {u'Muscular Dystrophy'},
u'Mycoses': {u'Fungal Infections'},
u'Myocardial Infarction': {u'Heart Attack'},
u'Neoplasms': {u'Cancer',
u'Cancer in Children',
u'Cancer--Living with Cancer'},
u'Nephrolithiasis': {u'Kidney Stones'},
u'Nervous System Diseases': {u'Neurologic Diseases'},
u'Nervous System Malformations': {u'Brain Malformations'},
u'Neuralgia, Postherpetic': {u'Shingles'},
u'Neurodegenerative Diseases': {u'Degenerative Nerve Diseases'},
u'Neurofibromatoses': {u'Neurofibromatosis'},
u'Neuroma, Acoustic': {u'Acoustic Neuroma'},
u'Neuromuscular Diseases': {u'Neuromuscular Disorders'},
u'Nevus': {u'Moles'},
u'Nose Diseases': {u'Nose Injuries and Disorders'},
u'Nose Neoplasms': {u'Nasal Cancer'},
u'Obstetric Labor Complications': {u'Childbirth Problems'},
u'Ocular Motility Disorders': {u'Eye Movement Disorders'},
u'Osteitis Deformans': {u"Paget's Disease of Bone"},
u'Otitis Media': {u'Ear Infections'},
u'Ovarian Diseases': {u'Ovarian Disorders'},
u'Ovarian Neoplasms': {u'Ovarian Cancer'},
u'Pancreatic Neoplasms': {u'Pancreatic Cancer'},
u'Papillomavirus Infections': {u'HPV'},
u'Parathyroid Diseases': {u'Parathyroid Disorders'},
u'Parkinson Disease': {u"Parkinson's Disease"},
u'Pericardial Effusion': {u'Pericardial Disorders'},
u'Peritoneal Diseases': {u'Peritoneal Disorders'},
u'Persistent Vegetative State': {u'Coma'},
u'Pharyngeal Diseases': {u'Throat Disorders'},
u'Pharyngeal Neoplasms': {u'Throat Cancer'},
u'Pharyngitis': {u'Sore Throat'},
u'Phenylketonurias': {u'Phenylketonuria'},
u'Phobic Disorders': {u'Phobias'},
u'Pigmentation Disorders': {u'Skin Pigmentation Disorders'},
u'Pituitary Diseases': {u'Pituitary Disorders'},
u'Pituitary Neoplasms': {u'Pituitary Tumors'},
u'Pleural Diseases': {u'Pleural Disorders'},
u'Poliomyelitis': {u'Polio and Post-Polio Syndrome'},
u'Porphyrias': {u'Porphyria'},
u'Postoperative Complications': {u'After Surgery'},
u'Prediabetic State': {u'Prediabetes'},
u'Pregnancy Complications, Infectious': {u'Infections and Pregnancy'},
u'Pregnancy in Diabetics': {u'Diabetes and Pregnancy'},
u'Pregnancy, Ectopic': {u'Ectopic Pregnancy'},
u'Prehypertension': {u'High Blood Pressure'},
u'Pressure Ulcer': {u'Pressure Sores'},
u'Primary Ovarian Insufficiency': {u'Premature Ovarian Failure'},
u'Prostatic Diseases': {u'Prostate Diseases'},
u'Prostatic Hyperplasia': {u'Enlarged Prostate (BPH)'},
u'Prostatic Neoplasms': {u'Prostate Cancer'},
u'Pruritus': {u'Itching'},
u'Pulmonary Atelectasis': {u'Collapsed Lung'},
u'Pulmonary Disease, Chronic Obstructive': {u'COPD'},
u'Raynaud Disease': {u"Raynaud's Disease"},
u'Rectal Diseases': {u'Rectal Disorders'},
u'Renal Insufficiency, Chronic': {u'Chronic Kidney Disease'},
u'Respiratory Insufficiency': {u'Respiratory Failure'},
u'Restless Legs Syndrome': {u'Restless Legs'},
u'Retinal Diseases': {u'Retinal Disorders'},
u'Rh Isoimmunization': {u'Rh Incompatibility'},
u'Rhinitis, Allergic, Seasonal': {u'Hay Fever'},
u'Salivary Gland Diseases': {u'Salivary Gland Disorders'},
u'Salivary Gland Neoplasms': {u'Salivary Gland Cancer'},
u'Sarcoma': {u'Soft Tissue Sarcoma'},
u'Sarcoma, Kaposi': {u"Kaposi's Sarcoma"},
u'Scleroderma, Localized': {u'Scleroderma'},
u'Self-Injurious Behavior': {u'Self-harm'},
u'Sexual Dysfunction, Physiological': {u'Sexual Problems in Men'},
u'Skin Diseases': {u'Skin Conditions'},
u'Skin Diseases, Infectious': {u'Skin Infections'},
u'Skin Neoplasms': {u'Skin Cancer'},
u'Sleep Apnea Syndromes': {u'Sleep Apnea'},
u'Sleep Initiation and Maintenance Disorders': {u'Insomnia'},
u'Smoking': {u'Smoking and Youth'},
u'Spinal Dysraphism': {u'Spina Bifida'},
u'Spinal Fractures': {u'Spine Injuries and Disorders'},
u'Spondylitis, Ankylosing': {u'Ankylosing Spondylitis'},
u'Staphylococcal Infections': {u'MRSA'},
u'Stomach Diseases': {u'Stomach Disorders'},
u'Stomach Neoplasms': {u'Stomach Cancer'},
u'Stomatitis, Aphthous': {u'Canker Sores'},
u'Stress Disorders, Post-Traumatic': {u'Post-Traumatic Stress Disorder'},
u'Stress, Psychological': {u'Stress'},
u'Stroke': {u'Stroke Rehabilitation'},
u'Substance-Related Disorders': {u'Drug Abuse',
u'Drugs and Young People',
u'Impaired Driving',
u'Prescription Drug Abuse'},
u'Sudden Infant Death': {u'Sudden Infant Death Syndrome'},
u'Sunburn': {u'Sun Exposure'},
u'Supranuclear Palsy, Progressive': {u'Progressive Supranuclear Palsy'},
u'Syncope': {u'Fainting'},
u'Temporomandibular Joint Disorders': {u'Temporomandibular Joint Dysfunction'},
u'Tendinopathy': {u'Tendinitis'},
u'Testicular Diseases': {u'Testicular Disorders'},
u'Testicular Neoplasms': {u'Testicular Cancer'},
u'Thoracic Injuries': {u'Chest Injuries and Disorders'},
u'Thymus Neoplasms': {u'Thymus Cancer'},
u'Thyroid Neoplasms': {u'Thyroid Cancer'},
u'Tick-Borne Diseases': {u'Tick Bites'},
u'Tinea': {u'Tinea Infections'},
u'Tinea Pedis': {u"Athlete's Foot"},
u'Tissue Adhesions': {u'Adhesions'},
u'Tooth Diseases': {u'Tooth Disorders'},
u'Tracheal Diseases': {u'Tracheal Disorders'},
u'Trichomonas Vaginitis': {u'Trichomoniasis'},
u'Ulna Fractures': {u'Arm Injuries and Disorders'},
u'Ureteral Diseases': {u'Ureteral Disorders'},
u'Urinary Bladder Diseases': {u'Bladder Diseases'},
u'Urinary Bladder Neoplasms': {u'Bladder Cancer'},
u'Urinary Bladder, Overactive': {u'Overactive Bladder'},
u'Urination Disorders': {u'Urine and Urination'},
u'Urticaria': {u'Hives'},
u'Usher Syndromes': {u'Usher Syndrome'},
u'Uterine Cervical Diseases': {u'Cervix Disorders'},
u'Uterine Cervical Neoplasms': {u'Cervical Cancer'},
u'Uterine Hemorrhage': {u'Vaginal Bleeding'},
u'Uterine Neoplasms': {u'Uterine Cancer'},
u'Vaginal Neoplasms': {u'Vaginal Cancer'},
u'Vaginismus': {u'Sexual Problems in Women'},
u'Venous Thrombosis': {u'Deep Vein Thrombosis'},
u'Virus Diseases': {u'Viral Infections'},
u'Vision Disorders': {u'Vision Impairment and Blindness'},
u'Vomiting': {u'Nausea and Vomiting'},
u'Vulvar Diseases': {u'Vulvar Disorders'},
u'Vulvar Neoplasms': {u'Vulvar Cancer'},
u'Wegener Granulomatosis': {u"Granulomatosis with Polyangiitis (Wegener's)"},
u'Weight Loss': {u'Weight Control'},
u'West Nile Fever': {u'West Nile Virus'},
u'Wilms Tumor': {u"Wilms' Tumor"},
u'Wrist Injuries': {u'Wrist Injuries and Disorders'},
u'Xerostomia': {u'Dry Mouth'}}
In [ ]:
Content source: jasonost/clinicaltrials
Similar notebooks: