In [229]:

    

import matplotlib.pyplot as plt
import csv
from textblob import TextBlob, Word
import pandas as pd
import sklearn
import pickle
import numpy as np
import scipy
import nltk.data
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC, LinearSVC
from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier 
from sklearn.model_selection import learning_curve, GridSearchCV, StratifiedKFold, cross_val_score, train_test_split 
sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
from nltk.tokenize import RegexpTokenizer
word_tokenizer = RegexpTokenizer('\s+', gaps=True)
from patsy import dmatrices
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.preprocessing import OneHotEncoder
from nltk.stem.porter import PorterStemmer

from stop_words import get_stop_words
stop_en = get_stop_words('en')
p_stemmer = PorterStemmer()
en_words = set(nltk.corpus.words.words())


from gensim import corpora, models
import gensim

import timeit
import re
import string
from string import whitespace, punctuation

from nltk.corpus import stopwords
stopw_en = stopwords.words('english')
print(stopw_en)
print(stop_en)
print(len(stopw_en))
print(len(stop_en))
all_stopw = set(stopw_en) | set(stop_en)
print(len(all_stopw))


    

    




['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won', 'wouldn']
['a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', "aren't", 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', "can't", 'cannot', 'could', "couldn't", 'did', "didn't", 'do', 'does', "doesn't", 'doing', "don't", 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', "hadn't", 'has', "hasn't", 'have', "haven't", 'having', 'he', "he'd", "he'll", "he's", 'her', 'here', "here's", 'hers', 'herself', 'him', 'himself', 'his', 'how', "how's", 'i', "i'd", "i'll", "i'm", "i've", 'if', 'in', 'into', 'is', "isn't", 'it', "it's", 'its', 'itself', "let's", 'me', 'more', 'most', "mustn't", 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same', "shan't", 'she', "she'd", "she'll", "she's", 'should', "shouldn't", 'so', 'some', 'such', 'than', 'that', "that's", 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', "there's", 'these', 'they', "they'd", "they'll", "they're", "they've", 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', "wasn't", 'we', "we'd", "we'll", "we're", "we've", 'were', "weren't", 'what', "what's", 'when', "when's", 'where', "where's", 'which', 'while', 'who', "who's", 'whom', 'why', "why's", 'with', "won't", 'would', "wouldn't", 'you', "you'd", "you'll", "you're", "you've", 'your', 'yours', 'yourself', 'yourselves']
153
174
207

In [5]:

    

set_tr = pickle.load(open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/set_tr_2','rb'))
# set_tr includes all unique highlight/text combos, plus text with highlights removed


    

In [ ]:

    




    

In [11]:

    

# print(set_tr)
# print(set_tr['textwohighlight'][0])


    

    




https://unsplash.com/?photo=pFqrYbhIAXs 35 Things You Need to Give Up to Be Successful 1. Wasting Five Minutes When you have five minutes of down-time, how do you spend that time? Most people use it as an excuse to rest or laze. By lazing for 5 five minute breaks each day, we waste 25 minutes daily. That’s 9,125 minutes per year (25 X 365). Sadly, my guess is we’re wasting far more time than that. I was once told by my 9th grade English teacher that if I read every time I had a break — even if the break was just for a minute or two — that I’d get a lot more reading done than expected. She was right. Every time I finished my work early, or had a spare moment, I’d pick up a book and read. How we spend our periodic five minute breaks is a determining factor to what we achieve in our lives. Every little bit adds up. Why can we justify wasting so much time? 2. Not Valuing One Dollar I was recently in Wal-Mart with my mother-in-law buying a few groceries. While we were in the check-out line, I pointed an item out to her I thought was interesting (honestly can’t remember what it is anymore). What stuck out to me is that she said, “One dollar. That’s a lot of money!” Why this surprised me is that my in-laws are not short of money. Actually, this happened while we were on a family trip (30+ people) at Disney World — the whole thing being paid for by them. Understanding the value of one dollar is the same as coming to appreciate the value of time. To thoughtlessly spend one dollar may not seem like a big deal, but it actually is. That frivolous spending compounded over a long enough time could be millions. It also reflects a lack of care about the details, which is where the true art and value lies. Additionally, most millionaires are “self-made”, 80 percent being first-generation rich, and 75 percent being self-employed. Not getting paid hourly challenges you to take more responsibility for every minute and every dollar. Consequently, a great majority of millionaires are extremely frugal — or at least highly mindful — with their money. 3. Believing Success Will Make You Happy “One of the enemies of happiness is adaptation” says Dr. Thomas Gilovich, a Cornell psychologist who has studied the relationship between money and happiness for over two decades. “We buy things to make us happy, and we succeed. But only for a while. New things are exciting to us at first, but then we adapt to them,” Gilovich further states. Actually, savoring the anticipation or idea of a desired outcome is generally more satisfying than the outcome itself. Once we get what we want — whether that’s wealth, health, or excellent relationships — we adapt and the excitement fades. Often, the experiences we’re seeking end up being underwhelming and even disappointing. I love watching this phenomena in our foster kids. They feel like they need a certain toy or the universe will explode. Their whole world revolves around getting this one thing. Yet, once we buy the toy for them, it’s not long before the joy fades and they want something else.   4. Believing You’re Not Up For the Challenge Just as we deceive ourselves into believing something will make us happier than it will, we also deceive ourselves into believing something will be harder than it will. The longer you procrastinate or avoid doing something, the more painful (in your head) it becomes. However, once you take action, the discomfort is far less severe than you imagined. Even to extremely difficult things, humans adapt. I recently sat on a plane with a lady who has 17 kids. Yes, you read that correctly. After having eight of her own, her and her husband felt inspired to foster four siblings whom they later adopted. A few years later, they took on another five foster siblings whom they also adopted. Of course, the initial shock to the system impacted her entire family. But they’re handling it. And believe it or not, you could handle it too, if you had to. The problem with dread and fear is that it holds people back from taking on big challenges. What you will find — no matter how big or small the challenge — is that you will adapt to it. When you consciously adapt to enormous stress, you evolve. 5. Pursuing “Happiness” “There is no way to happiness — happiness is the way.” — Thich Nhat Hanh Most people believe they must: First have something (e.g., money, time, or love) Before they can do what they want to do (e.g., travel the world, write a book, start a business, or have a romantic relationship) Which will ultimately allow them to be something (e.g., happy, peaceful, content, motivated, or in love). Paradoxically, this have — do — be paradigm must actually be reversed to experience happiness, success, or anything else you desire. First you be whatever it is you want to be (e.g., happy, compassionate, peaceful, wise, or loving) Then you start doing things from this space of being. Almost immediately, what you are doing will bring about the things you want to have. You attract what you are. If you want the things happy people have, you must be happy to get those things. If you want things wealthy people have, you must be and live wealthy to have those things. Results translate from attitudes and behaviors. Not the other ways around. 6. Undervaluing What You Have In an interview at the annual Genius Network Event in 2013, Tim Ferriss was asked, “With all of your various roles, do you ever get stressed out? Do you ever feel like you’ve taken on too much?” Ferriss responded: “Of course I get stressed out. If anyone says they don’t get stressed out they’re lying. But one thing that mitigates that is taking time each morning to declare and focus on the fact that ‘I have enough.’ I have enough. I don’t need to worry about responding to every email each day. If they get mad that’s their problem.” Ferriss was later asked during the same interview: “After having read The 4-Hour Workweek, I got the impression that Tim Ferriss doesn’t care about money. You talked about how you travel the world without spending any money. Talk about the balance and ability to let go of caring about making money.” Ferriss responded: “It’s totally okay to have lots of nice things. If it is addiction to wealth, like in Fight Club, “The things you own end up owning you,” and it becomes a surrogate for things like long-term health and happiness — connection — then it becomes a disease state. But if you can have nice things, and not fear having them taken away, then it’s a good thing. Because money is a really valuable tool.” If you appreciate what you already have, than more will be a good thing in your life. If you feel the need to have more to compensate for something missing in your life, you’ll always be left wanting — no matter how much you acquire or achieve. 7. Downplaying Your Current Position It’s easy to talk about how hard our lives are. It’s easy to talk about how unfair life is. And that we got the short-end of the stick. But does this kind of talking really help anyone? When we judge our situation as worse than someone else’s, we are ignorantly and incorrectly saying, “You’ve got it easy. You’re not like me. Success should come easy to you because you haven’t had to deal with what I’ve gone through.” This paradigm has formally become known as the victim mentality, and it generally leads to feelings of entitlement. The world owes you nothing. Life isn’t meant to be fair. However, the world has also given you everything you need. The truth is, you have every advantage in the world to succeed. And by believing this in your bones, you’ll feel an enormous weight of responsibility to yourself and the world. You’ve been put in a perfect position to succeed. Everything in the universe has brought you to this point so you can now shine and change the world.The world is your oyster. Your natural state is to thrive. All you have to do is show up. 8. Compartmentalizing Your Life Human beings are holistic — when you change a part of any system you simultaneously change the whole. You can’t change a part without fundamentally changing everything. Every pebble of thought — no matter how inconsequential — creates endless ripples of consequence. This idea, coined the butterfly effect by Edward Lorenz came from the metaphorical example of a hurricane being influenced by minor signals — such as the flapping of the wings of a distant butterfly — several weeks earlier. Little things become big things. When one area of your life is out of alignment, every area of your life suffers. You can’t compartmentalize a working system. Although it’s easy to push certain areas — like your health and relationships — to the side, you unwittingly infect your whole life. Eventually and always, the essentials you procrastinate or avoid will catch up to your detriment. Conversely, when you improve one area of your life, all other areas are positively influenced. As James Allen wrote in As a Man Thinketh, “When a man makes his thoughts pure, he no longer desires impure food.” We are holistic systems. Humanity as a whole is the same way. Everything you do effects the whole world, for better or worse. So I invite you to ask: “Am I part of the cure? Or am I part of the disease?” — Coldplay 9. Competing With Other People “All failed companies are the same: they failed to escape competition.” — Peter Thiel Competition is extremely costly to maximum product reach and wealth creation. It becomes a battle of who can slightly out-do the other for cheaper and cheaper. It’s a race to the bottom for all parties involved. Instead of trying to compete with other people or businesses, it’s better to do something completely novel or to focus on a tightly defined niche. Once you’ve established yourself as an authority over something, you can set your own terms — rather than reactively responding to the competition. Thus, you want to monopolize the space in which you create value. Competing with others leads people to spend every day of their lives pursuing goals that aren’t really their own — but what society has deemed important. You could spend your whole life trying to keep up, but will probably have a shallow life. Or, you can define success for yourself based on your own values and detach yourself from the noise. 10. Trying To Have It All Every decision has opportunity cost. When you choose one thing, you simultaneously don’t choose several others. When someone says you can have it all, they are lying. They are almost certainly not practicing what they preach and are trying to sell you on something. The truth is, you don’t want it all. And even if you did, reality simply doesn’t work that way. For example, I’ve come to terms with the fact that I want my family to be the center of my life. Spending time with my wife and three foster kids is my top priority. As a result, I can’t spend 12 or 15 hours a day working like some people. And that’s okay. I’ve made my choice. And that’s the point. We all need to choose what matters most to us, and own that. If we attempt to be everything, we’ll end up being nothing. Internal conflict is hell. Although the traditional view of creativity is that it is unstructured and doesn’t follow rules, creativity usually occurs by thinking inside the proverbial box, not outside of it. People flex their creative muscles when they constrain their options rather than broaden them. Hence, the more clearly defined and constraining your life’s objectives the better, because it allows you to sever everything outside those objectives. 11. Forgetting Where You Came From It’s easy when you achieve any level of success to believe you are solely responsible for that success. It’s easy to forget where you came from. It’s easy to forget all the sacrifices other people have made to get you where you are. It’s easy to see yourself as superior to other people. Burn all your bridges and you’ll have no human connection left. In that internal cave of isolation, you’ll lose your mind and identity, becoming a person you never intended to be. Humility, gratitude, and recognition of your blessings keeps your success in proper perspective. You couldn’t do what you’ve without the help of countless other people. You are extremely lucky to be able to contribute in the way you have. 12. Seeking Permission From Others My father-in-law is a highly successful real-estate investor. Throughout his career, he’s had hundreds of people ask him if they should “go into real-estate.” He tells every one of them the same thing: that they shouldn’t do it. In fact, he actually tries talking most of them out of it. And in most cases he succeeds. Why would he do that? “Those who are going to succeed will do so regardless of what I say,” my father-in-law told me. I know so many people who chase whatever worked for other people. They never truly decide what they want to do, and end up jumping from one thing to the next — trying to strike quick gold. And repetitively, they stop digging just a few feet from the gold after resigning the spot is barren. No one will ever give you permission to live your dreams. As Ryan Holiday has said in The Obstacle is the Way, “Stop looking for angels, and start looking for angles.” Rather than hoping for something external to change your circumstances, mentally reframe yourself and your circumstances. “When you change the way you see things, the things you see change.” — Wayne Dyer You are enough. You can do whatever you decide to do. Make the decision and forget what everyone else says or thinks about it. 13. Letting Others Determine How Much Money You Can Earn Most people “say” they want to be successful. But if they really wanted to, they’d be successful. I used to tell people, “I wish I played the piano.” Then someone said, “No you don’t. If you did, you’d make the time to practice.” I’ve since stopped saying that, because he was right. Life is a matter of priority and decision. And when it comes to money — in a free-market economy — you can make as much money as you choose. The question is, how much money do you really want to make? Instead of vegging on social media day-after-day, year-after-year, you could spend an hour or two each day building something of value — like yourself. In the book, Think and Grow Rich, Napoleon Hill invites readers to write down on a piece of paper the amount of money they want to make, and to put a time-line on it. This single act will challenge you to think and act in new ways to create the future of your wanting. For example, despite growing up so poor that for a time his family lived in their Volkswagen van on a relative’s lawn, Jim Carrey believed in his future. Every night in the late 1980’s, Carrey would drive atop a large hill that looked down over Los Angeles and visualize directors valuing his work. At the time, he was a broke and struggling young comic. One night in 1990, while looking down on Los Angeles and dreaming of his future, Carrey wrote himself a check for $10 million and put in the notation line “for acting services rendered.” He dated the check for Thanksgiving 1995 and stuck it in his wallet. He gave himself five years. And just before Thanksgiving of 1995, he got paid $10 million for Dumb and Dumber. 14. Ignoring the Vision You Have For Your Future “Create the highest, grandest vision possible for your life, because you become what you believe.” — Oprah Winfrey No matter where you are right now, you can have any future you want. But one thing is for certain, what you plant you must harvest. So, please plant with intention. Mental creation always precedes physical creation. The blueprint you design in your head becomes the life you build. Don’t let society tell you how your house should look. You are an artist and a creator. Your life can be exactly how you want it, whether or not it’s considered a “mansion” by others. Home is where your heart is. 15. Doing, Not Being There’s a parable of a wealthy parent who hesitated giving their unwise child an inheritance, knowing it would undoubtedly be squandered. The parent said to the child: “All that I have I desire to give you — not only my wealth, but also my position and standing among men. That which I have I can easily give you, but that which I am you must obtain for yourself. You will qualify for your inheritance by learning what I have learned and by living as I have lived. I will give you the laws and principles by which I have acquired my wisdom and stature. Follow my example, mastering as I have mastered, and you will become as I am, and all that I have will be yours.” Going through the motions is not enough. There isn’t a check-list of things you must do to be successful. You have to fundamentally change who you are to live at a higher level. You must shift from doing to being — so that what you do is a reflection of who you are, and who you’re becoming. Once you’ve experienced this change, success will be natural. Be, then do, then have (see point#3 above). “After you become a millionaire, you can give all of your money away because what’s important is not the million dollars; what’s important is the person you have become in the process of becoming a millionaire.” — Jim Rohn 16. Believing Money Is Evil “For better or worse, humans are holistic. Even the human body does best when its spiritual and physical sides are synchronized… People’s bodies perform best when their brains are on board with the program… Helping your mind to believe what you do is good, noble, and worthwhile in itself helps to fuel your energies and propel your efforts.” — Rabbi Daniel Lapin I know so many people who genuinely believe making money is immoral, and that people with money are evil. They believe those who seek profits force those weaker than them to buy their products. Money is not evil, but neutral. It is a symbol of perceived value. If I’m selling a pair of shoes for $20 and someone decides to buy them, they perceive the shoes to be worth more than the $20, or they wouldn’t buy them. I’m not forcing them to buy my shoes. It’s their choice. Thus, value exchange is win-win and based purely on perception. Value is subjective! If you offered that same person $20 for the shoes they just bought, they probably wouldn’t sell them. They see them as worth more than $20. But what if you offered $30? They still might not sell them. There is no “correct” price for goods and services. The correct price is the perceived worth from the customer. If the price is too high, the customer won’t exchange their money for it. We are extremely lucky to live in a society with a system of money. It allows us to borrow, lend, and leverage. Our ability to scale our work would be enormously limited in a bartering and trading system. Earning money is a completely moral pursuit when it is done with honesty and integrity. In fact, if you don’t feel moral about the work you’re doing, you should probably change your job. When you believe in the value you provide so much that you are doing people a disservice by not offering them your services, you’re on track to creating colossal value. Our work should be a reflection of us. It’s always their choice whether they perceive the value in what we’re offering or not. 17. Continuing to be Distracted “You cannot overestimate the unimportance of practically everything.” — Greg McKeown Almost everything is a distraction from what really matters. You really can’t put a price-tag on certain things. They are beyond a particular value to you. You’d give up everything, even your life, for those things. Your relationships and personal values don’t have a price-tag. And you should never exchange something priceless for a price. Keeping things in proper perspective allows you to remove everything non-essential from your life. It allows you to live simply and laser focused, and toavoid dead-end roads leading nowhere. 18. Not Realizing that Focus is Today’s I.Q. We live in the most distracted era of human history. The internet is a double-edged sword. Like money, the internet is neutral — and it can be used for good or bad based on who uses it. Sadly, most of us are simply not responsible enough for the internet. We waste hours every day staring idly at a screen. Millennials are particularly prone to distractions on the internet, but nowadays, everyone is susceptible. Our attention spans have shrunk to almost nothing. Our willpower has atrophied. We’ve developed some really bad habits that often require extreme interventions to reverse. There is a growing body of scientific evidence suggesting the internet — withits constant distractions and interruptions — is turning us into scattered and superficial thinkers. One of the biggest challenges to constant distraction is that it leads to “shallow” rather than “deep” thinking, and shallow thinking leads to shallow living. The Roman philosopher Seneca may have put it best 2,000 years ago: “To be everywhere is to be nowhere.” In his book, Deep Work: Rules for Focused Success in a Distracted World, Cal Newport differentiates “deep work,” from “shallow work.” Deep work is using your skills to create something of value. It takes thought, energy, time and concentration. Shallow work is all the little administrative and logistical stuff: email, meetings, calls, expense reports, etc. Most people aren’t moving toward their goals because they prioritize shallow work. “The ability to perform deep work is becoming increasingly rare at exactly the same time it is becoming increasingly valuable in our economy. As a consequence, the few who cultivate this skill, and then make it the core of their working life, will thrive.” — Cal Newport 19. Pursuing “Logical” Goals “You need to aim beyond what you are capable of. You need to develop a complete disregard for where your abilities end. If you think you’re unable to work for the best company in its sphere, make that your aim. If you think you’re unable to be on the cover of Time magazine, make it your business to be there. Make your vision of where you want to be a reality. Nothing is impossible.” — Paul Arden Most people’s goals are completely logical. They don’t require much imagination. They certainly don’t require faith, luck, magic, or miracles. Personally, I believe it’s sad how skeptical and secular many people are becoming. I find great pleasure in having faith in the spiritual. It provides context for life and meaning for personal growth. Having faith allows me to pursue that which others would call absurd, like walking on water and transcending death. Truly, with God all things are possible. There is absolutely nothing to fear. 20. Seeking Praise Rather than Criticism As a culture, we’ve become so fragile that we must combine honest feedback with 20 compliments. And when we get feedback, we do our best to disprove it. Psychologists call this confirmation bias — the tendency to search for, interpret, favor, and recall information that confirms our own beliefs, while giving excessively less consideration to alternative possibilities. It’s easy to get praise when you ask family and friends who will tell you exactly what you want to hear. Instead of seeking praise, your work will improve if you seek criticism. How could this be better? You will know your work has merit when someone cares enough to give unsolicited critique. If something is noteworthy, there will be haters. As Robin Sharma, author of The Monk Who Sold His Ferrari, has said, “haters confirm greatness.” When you really start showing up, the haters will be intimidated by you. Rather than being a reflection of what they could do,you become a reflection of what they are not doing. 21. Taking Rather Than Giving From a scarcity perspective, helping other people hurts you because you no longer have the advantage. This perspective sees the world as a giant pie. Every piece of the pie you have is pie I don’t have. So in order for you to win, I must lose. From an abundance perspective, there is not only one pie, but an infinite number of pies. If you want more, you make more. Thus, helping others actually helps you because it makes the system as a whole better. It also builds relationships and trust and confidence. I have a friend, Nate, who is doing some really innovative stuff at the real estate investing company he works for. He’s using strategies that no one else is using. And he’s killing it. He told me he considered keeping his strategies a secret. Because if other people knew about them, they’d use them and that’d mean less leads for him. But then he did the opposite. He told everyone in his company about what he was doing. He has even been giving tons of his leads away! This has never been seen before in his company. But Nate knows that once this strategy no longer works, he can come up with another one. And that’s what leadership and innovation is all about. And people have come to trust him. Actually, they’ve come to rely on him for developing the best strategies. Nate makes pies — for himself and several other people. And yes, he is also the top-selling and highest-earning in his company. It’s because he gives the most and doesn’t horde his ideas, resources, or information. 22. Creating What You Think You “Should” Create Many entrepreneurs design products to “scratch their own itch.” Actually, that’s how loads of problems are solved. You experience a difficulty and create a solution. Musicians and artists approach their work the same way. They create music they’d want to listen to, draw painting they’d want to see, and write books they wish were written. That’s how I personally approach my work. I write articles I myself would want to read. Your work should first and foremost resonate with yourself. If you don’t enjoy the product of your work, how can you expect other people to? 23. Looking For The Next Opportunity The perfect client, perfect opportunity, and perfect circumstances will almost never happen. Instead of wishing things were different, why not cultivate what’s right in front of you? Rather than waiting for the next opportunity, the one in your hands is the opportunity. Said another way, the grass is greener where you water it. I see so many people leave marriages because they believe better relationships are “out” there. In most cases, these people start new relationships and end them the same way the previous relationship ended. The problem isn’t your circumstances. The problem is you. You don’t find your soul-mate, you create your soul-mate through hard work. As Jim Rohn said, “Don’t wish it was easier, wish you were better. Don’t wish for less problems, wish for more skills. Don’t wish for less challenge, wish for more wisdom.” 24. Waiting To Start If you don’t purposefully carve time out every day to progress and improve — without question, your time will get lost in the vacuum of our increasingly crowded lives. Before you know it, you’ll be old and withered — wondering where all that time went. As Meredith Willson has said — “You pile up enough tomorrows, and you’ll find you are left with nothing but a lot of empty yesterdays.” I waited a few years too long to actively start writing. I was waiting for the right moment when I’d have enough time, money, and whatever else I thought I needed. I was waiting until I was somehow qualified or had permission to do what I wanted to do. But you are never pre-qualified. There is no degree for “Live your dreams.”You qualify yourself by showing up and working. You get permission by deciding. Life is short. Don’t wait for tomorrow for something you could do today. Your future self will either thank you or shamefully defend you. 25. Publishing Too Early At age 22, Tony Hsieh (now CEO of Zappos.com), graduated from Harvard. When Tony was 23 years old, six months after starting Linkexchange, he was offered one million dollars for the company. This was amazing to Tony because less than a year before, he was stoked to get a job at Oracle making 40K per year. After much thought and discussion with his partner, he rejected the offer believing he could continue to build Linkexchange into something bigger. His true love is in building and creating. A true pro gets paid, but doesn’t work for money. A true pro works for love. Five months later, Hsieh was offered 20 million dollars from Jerry Yang, cofounder of Yahoo!. This blew Tony away. His first thought was, “I’m glad I didn’t sell five months ago!” However, he held his cool and asked for a few days to consider the proposal. He would make this decisions on his terms. He thought about all the things he would do if he had all that money, knowing he would never have to work another day in his life. After reflecting, he could only devise a small list of things he wanted: A condo A TV and built-in home theater The ability to go on weekend mini-vacations whenever he wanted A new computer To start another company because he loves the idea of building and growing something. That was it. His passion and motivation wasn’t in having stuff. He concluded that he could already afford a TV, a new computer, and could already go on weekend mini-vacations whenever he wanted. He was only 23 years old, so he determined a condo could wait. Why would he sell Linkexchange just to build and grow another company? A year after Tony rejected the 20 million dollar offer, Linkexchange exploded. There were over 100 employees. Business was booming. Yet, Hsieh no longer enjoyed being there. The culture and politics had subtly changed in the process of rapid growth. Linkexchange was no longer Hsieh and a group of close friends building something they loved. They had hired a bunch of people in a hurry who didn’t have the same vision and motivations they had. Many of the new employees didn’t care about Linkexchange, or about building something they loved. Rather, they just wanted to get rich quick — purely self-interested. So he decided to sell the company on his terms. Microsoft purchased Linkexchange in 1998 for 265 million dollars when Hsieh was 25 years old. A similar concept emerged in a conversation I had about one year ago with Jeff Goins, best-selling author of The Art of Work. I asked his advice about publishing a book I want to write and he said, “Wait. Don’t jump the gun on this. I made that mistake myself. If you wait a year or two, you’ll get a 10x bigger advance, which will change the trajectory of your whole career.” Here’s how it works. With 20K email subscribers, a writer can get around a $20–40K book advance. But with 100–200K email subscribers, a writer can get around a $150–500K book advance. Wait a year or two and change the trajectory of your career (and life). This isn’t about procrastination. It’s about strategy. Timing — even a few seconds — could change your whole life. 26. Playing By The “Rules” “There is nothing that is a more certain sign of insanity than to do the same thing over and over and expect the results to be different.” — Albert Einstein Convention is where we’re at. Breaking convention is how we’ll evolve, which requires a gargantuan quantity of failure. If you don’t have the grit to fail 10,000 times, you’ll never invent your light bulb. As Seth Godin has said, “If I fail more than you do, I win.” Failure is something to be prized and praised. Failure is feedback. Failure is moving forward. It’s conscious and exerted effort toward something you’ve never done before. It’s incredible. “The person who doesn’t make mistakes is unlikely to make anything.” — Paul Arden 27. Not Setting Yourself Up To Win Before You Start “People may spend their whole lives climbing the ladder of success only to find, once they reach the top, that the ladder is leaning against the wrong wall.” — Thomas Merton Too many people are playing the wrong game — a losing game from the onset — and it hurts like hell. It’s how you ruin your life without even knowing it. More important than playing “the game” is how the game is set up. How you set up the game determines how you play. And it’s better to win first, then play. How does this work? Start from the end and work backwards. Rather than thinking about what’s plausible, or what’s expected, or what makes sense — start with what you want. Or as Covey put it in 7 Habits, “Begin with the end clearly in mind.” Once that’s nailed down, then dictate the daily, weekly, monthly, and yearly behaviors that will facilitate that. Jim Carrey wrote himself a $10 million check. Then he set out to earn it. He won the game first, then played. So can you. 28. Not Leveraging Every Small Win You Get No matter how small your wins along the way are, leverage your position! You have a high school diploma? Leverage your position! You know a guy who knows a guy who knows a guy? Leverage your position! You get an article featured on some unknown blog? Leverage your position! You have $100? Leverage your position! Sadly, most people can’t stop looking at the other side of the fence. They fail to realize the brilliant possibilities currently available to them. This is bad stewardship. There are people you already know who have information you need. There are people you already know who have capital you can use. There are people you already know who can connect you with people you should know. Instead of wanting more, how about you utilize what you already have? Until you do, more won’t help you. Actually, it will only continue hurting you until you learn to earn something for yourself. It’s easy to want other people to do it for you. But real success comes when you take ownership of your life. No one else cares more about your success (or health, or relationships, or time) than you do. Your current position is ripe with abundant opportunity. Leverage it. Once you gain another inch of position, leverage it for all it’s worth. Don’t wish for more. Wish you were better. And soon enough, you’ll find yourself in incredible positions and collaborating with your heroes. Success is based on choice. Success is based on having and maintaining a motivation worth fighting for. It’s based on believing what others might call a fantasy. It’s based on leveraging your position and maintaining the momentum of every step you take. 29. Viewing Your Work as “Work” The cool part about poetry is that to most poets, how their poems are performed is just as important — if not more important — than what is actually said. In a similar way, when you go to an event or to hear a speech, you’re usually going to see the speaker, not hear what they have to say. You already know what they have to say. No matter what type of work you are in, it will be better received if you see it as an art-form. You are performing for an audience. They want you just as much as they want your work — often more. 30. Letting Others Decide How Your Work Should Be Ryan Holiday, author of The Obstacle is the Way, explains what he calls “the moment,” which every skilled creative has experienced. “The moment,” is when your eyes are opened to the mechanics and behind-the-scenes of your craft. Until you have this moment, it all seems like magic to you. You have no idea how people create what they create. After you have this moment, you realize that everything is done by a person intentionally creating a particular experience. I was recently watching Lord of the Rings and it dawned on me that those movies would be completely different if they weren’t directed by Peter Jackson. Completely different! Every shot, every set, the lighting, the costumes, how the characters and landscapes look, and how the whole film feels and is portrayed. It all would have looked and felt completely different based on the experience a different director was trying to create. Thus, there is no right or wrong way. Rather, it’s about doing things your way. Until you experience this “moment,” you’ll continue attempting the correct or best way to do things. You’ll continue copying other people’s work. But if you persist, you’ll become disillusioned to those who were once your idols. They are people just like you and me. They’ve just made a decision to create in their own way. The idea of imitation will become abhorrent, freeing you to create as you see fit. You’ll emerge with your own voice and original work. You’ll be less troubled about how your work is received and more focused on creating something you believe in. 31. Seeking Retirement “To retire is to die.” — Pablo Casals The most powerful way to punch someone in the face is to aim a foot behind their face. That way, you have full momentum and power when you make contact. If you aim only for the face itself, by the time you reach it you’ll have already begun slowing down. Thus, your punch will not be as powerful as you intended it to be. Retirement is the same way. Most people planning for retirement begin slowing down in their 40’s and 50’s. The sad part is, as momentum-based beings, when you begin to slow down, you start a hard-to-reverse decaying process. Research has found that retirement often: Increases the difficulty of mobility and daily activities Increases the likelihood of becoming ill And decreases mental health But retirement is a 20th century phenomena. And actually, the foundations under girding this outdated notion make little sense in modern and future society. For instance, due to advances in health care, 65 is not considered old age anymore. When the Social Security system was designed, the planners chose age 65 because the average lifespan was age 63 at the time. Thus, the system was designed only for those who were really in need, not to create a culture of people being supported by others’ labor. Furthermore, the perception that people over 65 can’t provide meaningful work no longer makes sense either. Retirement became a thing when most work was manual labor — but today’s work is more knowledge-based. And if there’s anything lacking in today’s society, its wisdom, which people in their later years have spent a lifetime refining. Retirement should never be the goal. We are fully capable to work — in some capacity — until our final breath. My 92 year old grandfather, Rex, was a fighter pilot in WWII. In the past five years he’s written three books. He goes to bed every night at 8 P.M. and wakes up every morning at 4:30 A.M. He spends the first 2.5 hours of his day watching inspirational and instructional content on television. He then eats breakfast at 7 A.M. and spends his day reading, writing, connecting and serving people, and even doing physical labor around his son’s (my dad’s) house. He even walks around his neighborhood proselyting his faith and asking random strangers how he can help them. I have no intention of stopping or slowing down. Contrary to popular belief, humans are like wine and get better with age. 32. Downplaying The Importance Of Yesterday “The best time to plant a tree was 20 years ago. The second best time is now.” — Chinese Proverb Our present circumstances are a reflection of our past decisions. Although we have enormous power to change the trajectory of our lives here and now,we are where we are because of our past. While it’s popular to say the past doesn’t matter, that simply is not true. Today is tomorrow’s yesterday. What we do today will either enhance or diminish our future-present moments. But most people put things off until tomorrow. We thoughtlessly go into debt, forego exercise and education, and justify negative relationships. But at some point it all catches up. Like an airplane off-course, the longer we wait to correct the longer and harder it is to get back on-course. Time is absolutely marvelous. We get to anticipate the experiences we want to have — which is often more enjoyable than the experiences themselves. We get to have the experiences we long for. And then we get to remember and carry those experiences with us forever. The past, present, and future are uniquely important and enjoyable. 33. Believing You’re Way Behind In sports and all other forms of competition, people perform best when the game is close. Which is why big magic happens at the end of games, like on-sides kicks retrieved followed by 30 second touchdown drives. But when the contest is decidedly in one opponent’s favor, neither side acts with the same effort. When you’re winning big, it’s easy to get lax and overconfident. When you’re losing big, it’s easy to give up. Sadly, you probably perceive those at the top of your field “in a different league” altogether. But when you do this, you perform with less intensity than you would if you perceived the “game” to be closer. When you elevate your thinking — and see yourself on the same level as those at “the top” — you quickly become disillusioned by the fallibility of those you once perceived as immortal. They are just people. Most importantly, you will begin playing with an urgency that often surpasses even them. The game is close. The game is close. 34. Listening To Music That Destroys Your Joy and Productivity “Without music, life would be a mistake” — Friedrich Nietzsche You can also use music as a trigger for optimal performance. For example, Michael Phelps had a routine he did religiously before each swimming event involving music. He’s not alone. Many athletes use music before events to trigger relaxation from the pressure and even to psych themselves up. When asked by Time Magazine about his use of music prior to races, Phelps said it kept him focused and helped him “tune everything out, and take one step at a time.” When asked about the kind of music he listens to, he answered, “I listen to hip hop and rap.” Interestingly, research has found that high tempo music like hip hop can create strong arousal and performance readiness. Other evidence finds the intensity of the emotional response can linger long after the music has stopped. So, while Phelps is in the water swimming, he’s still hyped from his hip hop. Lastly, research has found that the types of music we listen to impact our level of spirituality. This last point is particularly important to me. Spirituality heavily influences everything I do, from how I interact with my family, to what and how I write, to how I develop and pursue my goals. 35. Call To Action If you want to focus on the right activities and get results 10x faster than most people, check out my morning checklist. Click here to get the checklist right now!

In [102]:

    

all_texts_processed = []

n = 0
for text in set_tr['text']:
    # combine sentences
    txt = ' '.join(text)
    # remove punctuation
    translator = str.maketrans('', '', string.punctuation)
    txt2 = re.sub(u'\u2014','',txt)
    txt3 = txt2.translate(translator)
    # split text into words
    tokens = word_tokenizer.tokenize(txt3.lower())
    # remove stop words
    nostop_tokens = [i for i in tokens if not i in all_stopw]
    # stem words
    stemmed = [p_stemmer.stem(i) for i in nostop_tokens]
    # append to processed texts
    all_texts_processed.append( stemmed )
    if n == 0:
#         print(txt)
#         print(tokens)
#         print(nostop_tokens)
        print(stemmed)
    n += 1
#     if n == 5:
#         break


    

    




['httpsunsplashcomphotopfqrybhiax', '35', 'thing', 'need', 'give', 'success', '1', 'wast', 'five', 'minut', 'five', 'minut', 'downtim', 'spend', 'time', 'peopl', 'use', 'excus', 'rest', 'laze', 'laze', '5', 'five', 'minut', 'break', 'day', 'wast', '25', 'minut', 'daili', 'that’', '9125', 'minut', 'per', 'year', '25', 'x', '365', 'sadli', 'guess', 'we’r', 'wast', 'far', 'time', 'told', '9th', 'grade', 'english', 'teacher', 'read', 'everi', 'time', 'break', 'even', 'break', 'minut', 'two', 'i’d', 'get', 'lot', 'read', 'done', 'expect', 'right', 'everi', 'time', 'finish', 'work', 'earli', 'spare', 'moment', 'i’d', 'pick', 'book', 'read', 'spend', 'period', 'five', 'minut', 'break', 'determin', 'factor', 'achiev', 'live', 'everi', 'littl', 'bit', 'add', 'justifi', 'wast', 'much', 'time', '2', 'valu', 'one', 'dollar', 'recent', 'walmart', 'motherinlaw', 'buy', 'groceri', 'checkout', 'line', 'point', 'item', 'thought', 'interest', 'honestli', 'can’t', 'rememb', 'anymor', 'stuck', 'said', '“one', 'dollar', 'that’', 'lot', 'money”', 'surpris', 'inlaw', 'short', 'money', 'actual', 'happen', 'famili', 'trip', '30', 'peopl', 'disney', 'world', 'whole', 'thing', 'paid', 'understand', 'valu', 'one', 'dollar', 'come', 'appreci', 'valu', 'time', 'thoughtlessli', 'spend', 'one', 'dollar', 'may', 'seem', 'like', 'big', 'deal', 'actual', 'frivol', 'spend', 'compound', 'long', 'enough', 'time', 'million', 'also', 'reflect', 'lack', 'care', 'detail', 'true', 'art', 'valu', 'lie', 'addit', 'millionair', '“selfmade”', '80', 'percent', 'firstgener', 'rich', '75', 'percent', 'selfemploy', 'get', 'paid', 'hourli', 'challeng', 'take', 'respons', 'everi', 'minut', 'everi', 'dollar', 'consequ', 'great', 'major', 'millionair', 'extrem', 'frugal', 'least', 'highli', 'mind', 'money', '3', 'believ', 'success', 'make', 'happi', '“one', 'enemi', 'happi', 'adaptation”', 'say', 'dr', 'thoma', 'gilovich', 'cornel', 'psychologist', 'studi', 'relationship', 'money', 'happi', 'two', 'decad', '“we', 'buy', 'thing', 'make', 'us', 'happi', 'succeed', 'new', 'thing', 'excit', 'us', 'first', 'adapt', 'them”', 'gilovich', 'state', 'actual', 'savor', 'anticip', 'idea', 'desir', 'outcom', 'gener', 'satisfi', 'outcom', 'get', 'want', 'whether', 'that’', 'wealth', 'health', 'excel', 'relationship', 'adapt', 'excit', 'fade', 'often', 'experi', 'we’r', 'seek', 'end', 'underwhelm', 'even', 'disappoint', 'love', 'watch', 'phenomena', 'foster', 'kid', 'feel', 'like', 'need', 'certain', 'toy', 'univers', 'explod', 'whole', 'world', 'revolv', 'around', 'get', 'one', 'thing', 'yet', 'buy', 'toy', 'it’', 'long', 'joy', 'fade', 'want', 'someth', 'els', 'appreci', 'current', 'won’t', 'make', 'life', 'better', '4', 'believ', 'you’r', 'challeng', 'deceiv', 'believ', 'someth', 'make', 'us', 'happier', 'also', 'deceiv', 'believ', 'someth', 'harder', 'longer', 'procrastin', 'avoid', 'someth', 'pain', 'head', 'becom', 'howev', 'take', 'action', 'discomfort', 'far', 'less', 'sever', 'imagin', 'even', 'extrem', 'difficult', 'thing', 'human', 'adapt', 'recent', 'sat', 'plane', 'ladi', '17', 'kid', 'ye', 'read', 'correctli', 'eight', 'husband', 'felt', 'inspir', 'foster', 'four', 'sibl', 'later', 'adopt', 'year', 'later', 'took', 'anoth', 'five', 'foster', 'sibl', 'also', 'adopt', 'cours', 'initi', 'shock', 'system', 'impact', 'entir', 'famili', 'they’r', 'handl', 'believ', 'handl', 'problem', 'dread', 'fear', 'hold', 'peopl', 'back', 'take', 'big', 'challeng', 'find', 'matter', 'big', 'small', 'challeng', 'adapt', 'conscious', 'adapt', 'enorm', 'stress', 'evolv', '5', 'pursu', '“happiness”', '“there', 'way', 'happi', 'happi', 'way”', 'thich', 'nhat', 'hanh', 'peopl', 'believ', 'must', 'first', 'someth', 'eg', 'money', 'time', 'love', 'want', 'eg', 'travel', 'world', 'write', 'book', 'start', 'busi', 'romant', 'relationship', 'ultim', 'allow', 'someth', 'eg', 'happi', 'peac', 'content', 'motiv', 'love', 'paradox', 'paradigm', 'must', 'actual', 'revers', 'experi', 'happi', 'success', 'anyth', 'els', 'desir', 'first', 'whatev', 'want', 'eg', 'happi', 'compassion', 'peac', 'wise', 'love', 'start', 'thing', 'space', 'almost', 'immedi', 'bring', 'thing', 'want', 'attract', 'want', 'thing', 'happi', 'peopl', 'must', 'happi', 'get', 'thing', 'want', 'thing', 'wealthi', 'peopl', 'must', 'live', 'wealthi', 'thing', 'result', 'translat', 'attitud', 'behavior', 'way', 'around', '6', 'undervalu', 'interview', 'annual', 'geniu', 'network', 'event', '2013', 'tim', 'ferriss', 'ask', '“with', 'variou', 'role', 'ever', 'get', 'stress', 'ever', 'feel', 'like', 'you’v', 'taken', 'much”', 'ferriss', 'respond', '“of', 'cours', 'get', 'stress', 'anyon', 'say', 'don’t', 'get', 'stress', 'they’r', 'lie', 'one', 'thing', 'mitig', 'take', 'time', 'morn', 'declar', 'focu', 'fact', '‘i', 'enough’', 'enough', 'don’t', 'need', 'worri', 'respond', 'everi', 'email', 'day', 'get', 'mad', 'that’', 'problem”', 'ferriss', 'later', 'ask', 'interview', '“after', 'read', '4hour', 'workweek', 'got', 'impress', 'tim', 'ferriss', 'doesn’t', 'care', 'money', 'talk', 'travel', 'world', 'without', 'spend', 'money', 'talk', 'balanc', 'abil', 'let', 'go', 'care', 'make', 'money”', 'ferriss', 'respond', '“it’', 'total', 'okay', 'lot', 'nice', 'thing', 'addict', 'wealth', 'like', 'fight', 'club', '“the', 'thing', 'end', 'own', 'you”', 'becom', 'surrog', 'thing', 'like', 'longterm', 'health', 'happi', 'connect', 'becom', 'diseas', 'state', 'nice', 'thing', 'fear', 'taken', 'away', 'it’', 'good', 'thing', 'money', 'realli', 'valuabl', 'tool”', 'appreci', 'alreadi', 'good', 'thing', 'life', 'feel', 'need', 'compens', 'someth', 'miss', 'life', 'you’ll', 'alway', 'left', 'want', 'matter', 'much', 'acquir', 'achiev', '7', 'downplay', 'current', 'posit', 'it’', 'easi', 'talk', 'hard', 'live', 'it’', 'easi', 'talk', 'unfair', 'life', 'got', 'shortend', 'stick', 'kind', 'talk', 'realli', 'help', 'anyon', 'judg', 'situat', 'wors', 'someon', 'else’', 'ignorantli', 'incorrectli', 'say', '“you’v', 'got', 'easi', 'you’r', 'like', 'success', 'come', 'easi', 'haven’t', 'deal', 'i’v', 'gone', 'through”', 'paradigm', 'formal', 'becom', 'known', 'victim', 'mental', 'gener', 'lead', 'feel', 'entitl', 'world', 'owe', 'noth', 'life', 'isn’t', 'meant', 'fair', 'howev', 'world', 'also', 'given', 'everyth', 'need', 'truth', 'everi', 'advantag', 'world', 'succeed', 'believ', 'bone', 'you’ll', 'feel', 'enorm', 'weight', 'respons', 'world', 'you’v', 'put', 'perfect', 'posit', 'succeed', 'everyth', 'univers', 'brought', 'point', 'shine', 'chang', 'worldth', 'world', 'oyster', 'natur', 'state', 'thrive', 'show', '8', 'compartment', 'life', 'human', 'be', 'holist', 'chang', 'part', 'system', 'simultan', 'chang', 'whole', 'can’t', 'chang', 'part', 'without', 'fundament', 'chang', 'everyth', 'everi', 'pebbl', 'thought', 'matter', 'inconsequenti', 'creat', 'endless', 'rippl', 'consequ', 'idea', 'coin', 'butterfli', 'effect', 'edward', 'lorenz', 'came', 'metaphor', 'exampl', 'hurrican', 'influenc', 'minor', 'signal', 'flap', 'wing', 'distant', 'butterfli', 'sever', 'week', 'earlier', 'littl', 'thing', 'becom', 'big', 'thing', 'one', 'area', 'life', 'align', 'everi', 'area', 'life', 'suffer', 'can’t', 'compartment', 'work', 'system', 'although', 'it’', 'easi', 'push', 'certain', 'area', 'like', 'health', 'relationship', 'side', 'unwittingli', 'infect', 'whole', 'life', 'eventu', 'alway', 'essenti', 'procrastin', 'avoid', 'catch', 'detriment', 'convers', 'improv', 'one', 'area', 'life', 'area', 'posit', 'influenc', 'jame', 'allen', 'wrote', 'man', 'thinketh', '“when', 'man', 'make', 'thought', 'pure', 'longer', 'desir', 'impur', 'food”', 'holist', 'system', 'human', 'whole', 'way', 'everyth', 'effect', 'whole', 'world', 'better', 'wors', 'invit', 'ask', '“am', 'part', 'cure', 'part', 'disease”', 'coldplay', '9', 'compet', 'peopl', '“all', 'fail', 'compani', 'fail', 'escap', 'competition”', 'peter', 'thiel', 'competit', 'extrem', 'costli', 'maximum', 'product', 'reach', 'wealth', 'creation', 'becom', 'battl', 'slightli', 'outdo', 'cheaper', 'cheaper', 'it’', 'race', 'bottom', 'parti', 'involv', 'instead', 'tri', 'compet', 'peopl', 'busi', 'it’', 'better', 'someth', 'complet', 'novel', 'focu', 'tightli', 'defin', 'nich', 'you’v', 'establish', 'author', 'someth', 'set', 'term', 'rather', 'reactiv', 'respond', 'competit', 'thu', 'want', 'monopol', 'space', 'creat', 'valu', 'compet', 'other', 'lead', 'peopl', 'spend', 'everi', 'day', 'live', 'pursu', 'goal', 'aren’t', 'realli', 'societi', 'deem', 'import', 'spend', 'whole', 'life', 'tri', 'keep', 'probabl', 'shallow', 'life', 'defin', 'success', 'base', 'valu', 'detach', 'nois', '10', 'tri', 'everi', 'decis', 'opportun', 'cost', 'choos', 'one', 'thing', 'simultan', 'don’t', 'choos', 'sever', 'other', 'someon', 'say', 'lie', 'almost', 'certainli', 'practic', 'preach', 'tri', 'sell', 'someth', 'truth', 'don’t', 'want', 'even', 'realiti', 'simpli', 'doesn’t', 'work', 'way', 'exampl', 'i’v', 'come', 'term', 'fact', 'want', 'famili', 'center', 'life', 'spend', 'time', 'wife', 'three', 'foster', 'kid', 'top', 'prioriti', 'result', 'can’t', 'spend', '12', '15', 'hour', 'day', 'work', 'like', 'peopl', 'that’', 'okay', 'i’v', 'made', 'choic', 'that’', 'point', 'need', 'choos', 'matter', 'us', 'attempt', 'everyth', 'we’ll', 'end', 'noth', 'intern', 'conflict', 'hell', 'although', 'tradit', 'view', 'creativ', 'unstructur', 'doesn’t', 'follow', 'rule', 'creativ', 'usual', 'occur', 'think', 'insid', 'proverbi', 'box', 'outsid', 'peopl', 'flex', 'creativ', 'muscl', 'constrain', 'option', 'rather', 'broaden', 'henc', 'clearli', 'defin', 'constrain', 'life’', 'object', 'better', 'allow', 'sever', 'everyth', 'outsid', 'object', '11', 'forget', 'came', 'it’', 'easi', 'achiev', 'level', 'success', 'believ', 'sole', 'respons', 'success', 'it’', 'easi', 'forget', 'came', 'it’', 'easi', 'forget', 'sacrific', 'peopl', 'made', 'get', 'it’', 'easi', 'see', 'superior', 'peopl', 'burn', 'bridg', 'you’ll', 'human', 'connect', 'left', 'intern', 'cave', 'isol', 'you’ll', 'lose', 'mind', 'ident', 'becom', 'person', 'never', 'intend', 'humil', 'gratitud', 'recognit', 'bless', 'keep', 'success', 'proper', 'perspect', 'couldn’t', 'you’v', 'without', 'help', 'countless', 'peopl', 'extrem', 'lucki', 'abl', 'contribut', 'way', '12', 'seek', 'permiss', 'other', 'fatherinlaw', 'highli', 'success', 'realest', 'investor', 'throughout', 'career', 'he’', 'hundr', 'peopl', 'ask', '“go', 'realestate”', 'tell', 'everi', 'one', 'thing', 'shouldn’t', 'fact', 'actual', 'tri', 'talk', 'case', 'succe', '“those', 'go', 'succeed', 'regardless', 'say”', 'fatherinlaw', 'told', 'know', 'mani', 'peopl', 'chase', 'whatev', 'work', 'peopl', 'never', 'truli', 'decid', 'want', 'end', 'jump', 'one', 'thing', 'next', 'tri', 'strike', 'quick', 'gold', 'repetit', 'stop', 'dig', 'feet', 'gold', 'resign', 'spot', 'barren', 'one', 'ever', 'give', 'permiss', 'live', 'dream', 'ryan', 'holiday', 'said', 'obstacl', 'way', '“stop', 'look', 'angel', 'start', 'look', 'angles”', 'rather', 'hope', 'someth', 'extern', 'chang', 'circumst', 'mental', 'refram', 'circumst', '“when', 'chang', 'way', 'see', 'thing', 'thing', 'see', 'change”', 'wayn', 'dyer', 'enough', 'whatev', 'decid', 'make', 'decis', 'forget', 'everyon', 'els', 'say', 'think', '13', 'let', 'other', 'determin', 'much', 'money', 'earn', 'peopl', '“say”', 'want', 'success', 'realli', 'want', 'they’d', 'success', 'use', 'tell', 'peopl', '“i', 'wish', 'play', 'piano”', 'someon', 'said', '“no', 'don’t', 'you’d', 'make', 'time', 'practice”', 'i’v', 'sinc', 'stop', 'say', 'right', 'life', 'matter', 'prioriti', 'decis', 'come', 'money', 'freemarket', 'economi', 'make', 'much', 'money', 'choos', 'question', 'much', 'money', 'realli', 'want', 'make', 'instead', 'veg', 'social', 'media', 'dayafterday', 'yearafteryear', 'spend', 'hour', 'two', 'day', 'build', 'someth', 'valu', 'like', 'book', 'think', 'grow', 'rich', 'napoleon', 'hill', 'invit', 'reader', 'write', 'piec', 'paper', 'amount', 'money', 'want', 'make', 'put', 'timelin', 'singl', 'act', 'challeng', 'think', 'act', 'new', 'way', 'creat', 'futur', 'want', 'exampl', 'despit', 'grow', 'poor', 'time', 'famili', 'live', 'volkswagen', 'van', 'relative’', 'lawn', 'jim', 'carrey', 'believ', 'futur', 'everi', 'night', 'late', '1980’', 'carrey', 'drive', 'atop', 'larg', 'hill', 'look', 'lo', 'angel', 'visual', 'director', 'valu', 'work', 'time', 'broke', 'struggl', 'young', 'comic', 'one', 'night', '1990', 'look', 'lo', 'angel', 'dream', 'futur', 'carrey', 'wrote', 'check', '10', 'million', 'put', 'notat', 'line', '“for', 'act', 'servic', 'rendered”', 'date', 'check', 'thanksgiv', '1995', 'stuck', 'wallet', 'gave', 'five', 'year', 'thanksgiv', '1995', 'got', 'paid', '10', 'million', 'dumb', 'dumber', '14', 'ignor', 'vision', 'futur', '“creat', 'highest', 'grandest', 'vision', 'possibl', 'life', 'becom', 'believe”', 'oprah', 'winfrey', 'matter', 'right', 'futur', 'want', 'one', 'thing', 'certain', 'plant', 'must', 'harvest', 'pleas', 'plant', 'intent', 'mental', 'creation', 'alway', 'preced', 'physic', 'creation', 'blueprint', 'design', 'head', 'becom', 'life', 'build', 'don’t', 'let', 'societi', 'tell', 'hous', 'look', 'artist', 'creator', 'life', 'exactli', 'want', 'whether', 'it’', 'consid', '“mansion”', 'other', 'home', 'heart', '15', 'there’', 'parabl', 'wealthi', 'parent', 'hesit', 'give', 'unwis', 'child', 'inherit', 'know', 'undoubtedli', 'squander', 'parent', 'said', 'child', '“all', 'desir', 'give', 'wealth', 'also', 'posit', 'stand', 'among', 'men', 'easili', 'give', 'must', 'obtain', 'qualifi', 'inherit', 'learn', 'learn', 'live', 'live', 'give', 'law', 'principl', 'acquir', 'wisdom', 'statur', 'follow', 'exampl', 'master', 'master', 'becom', 'yours”', 'go', 'motion', 'enough', 'isn’t', 'checklist', 'thing', 'must', 'success', 'fundament', 'chang', 'live', 'higher', 'level', 'must', 'shift', 'reflect', 'you’r', 'becom', 'you’v', 'experienc', 'chang', 'success', 'natur', 'see', 'point3', '“after', 'becom', 'millionair', 'give', 'money', 'away', 'what’', 'import', 'million', 'dollar', 'what’', 'import', 'person', 'becom', 'process', 'becom', 'millionaire”', 'jim', 'rohn', '16', 'believ', 'money', 'evil', '“for', 'better', 'wors', 'human', 'holist', 'even', 'human', 'bodi', 'best', 'spiritu', 'physic', 'side', 'synchronized…', 'people’', 'bodi', 'perform', 'best', 'brain', 'board', 'program…', 'help', 'mind', 'believ', 'good', 'nobl', 'worthwhil', 'help', 'fuel', 'energi', 'propel', 'efforts”', 'rabbi', 'daniel', 'lapin', 'know', 'mani', 'peopl', 'genuin', 'believ', 'make', 'money', 'immor', 'peopl', 'money', 'evil', 'believ', 'seek', 'profit', 'forc', 'weaker', 'buy', 'product', 'money', 'evil', 'neutral', 'symbol', 'perceiv', 'valu', 'i’m', 'sell', 'pair', 'shoe', '20', 'someon', 'decid', 'buy', 'perceiv', 'shoe', 'worth', '20', 'wouldn’t', 'buy', 'i’m', 'forc', 'buy', 'shoe', 'it’', 'choic', 'thu', 'valu', 'exchang', 'winwin', 'base', 'pure', 'percept', 'valu', 'subject', 'offer', 'person', '20', 'shoe', 'bought', 'probabl', 'wouldn’t', 'sell', 'see', 'worth', '20', 'offer', '30', 'still', 'might', 'sell', '“correct”', 'price', 'good', 'servic', 'correct', 'price', 'perceiv', 'worth', 'custom', 'price', 'high', 'custom', 'won’t', 'exchang', 'money', 'extrem', 'lucki', 'live', 'societi', 'system', 'money', 'allow', 'us', 'borrow', 'lend', 'leverag', 'abil', 'scale', 'work', 'enorm', 'limit', 'barter', 'trade', 'system', 'earn', 'money', 'complet', 'moral', 'pursuit', 'done', 'honesti', 'integr', 'fact', 'don’t', 'feel', 'moral', 'work', 'you’r', 'probabl', 'chang', 'job', 'believ', 'valu', 'provid', 'much', 'peopl', 'disservic', 'offer', 'servic', 'you’r', 'track', 'creat', 'coloss', 'valu', 'work', 'reflect', 'us', 'it’', 'alway', 'choic', 'whether', 'perceiv', 'valu', 'we’r', 'offer', '17', 'continu', 'distract', '“you', 'overestim', 'unimport', 'practic', 'everything”', 'greg', 'mckeown', 'almost', 'everyth', 'distract', 'realli', 'matter', 'realli', 'can’t', 'put', 'pricetag', 'certain', 'thing', 'beyond', 'particular', 'valu', 'you’d', 'give', 'everyth', 'even', 'life', 'thing', 'relationship', 'person', 'valu', 'don’t', 'pricetag', 'never', 'exchang', 'someth', 'priceless', 'price', 'keep', 'thing', 'proper', 'perspect', 'allow', 'remov', 'everyth', 'nonessenti', 'life', 'allow', 'live', 'simpli', 'laser', 'focus', 'toavoid', 'deadend', 'road', 'lead', 'nowher', '18', 'realiz', 'focu', 'today’', 'iq', 'live', 'distract', 'era', 'human', 'histori', 'internet', 'doubleedg', 'sword', 'like', 'money', 'internet', 'neutral', 'use', 'good', 'bad', 'base', 'use', 'sadli', 'us', 'simpli', 'respons', 'enough', 'internet', 'wast', 'hour', 'everi', 'day', 'stare', 'idli', 'screen', 'millenni', 'particularli', 'prone', 'distract', 'internet', 'nowaday', 'everyon', 'suscept', 'attent', 'span', 'shrunk', 'almost', 'noth', 'willpow', 'atrophi', 'we’v', 'develop', 'realli', 'bad', 'habit', 'often', 'requir', 'extrem', 'intervent', 'revers', 'grow', 'bodi', 'scientif', 'evid', 'suggest', 'internet', 'withit', 'constant', 'distract', 'interrupt', 'turn', 'us', 'scatter', 'superfici', 'thinker', 'one', 'biggest', 'challeng', 'constant', 'distract', 'lead', '“shallow”', 'rather', '“deep”', 'think', 'shallow', 'think', 'lead', 'shallow', 'live', 'roman', 'philosoph', 'seneca', 'may', 'put', 'best', '2000', 'year', 'ago', '“to', 'everywher', 'nowhere”', 'book', 'deep', 'work', 'rule', 'focus', 'success', 'distract', 'world', 'cal', 'newport', 'differenti', '“deep', 'work”', '“shallow', 'work”', 'deep', 'work', 'use', 'skill', 'creat', 'someth', 'valu', 'take', 'thought', 'energi', 'time', 'concentr', 'shallow', 'work', 'littl', 'administr', 'logist', 'stuff', 'email', 'meet', 'call', 'expens', 'report', 'etc', 'peopl', 'aren’t', 'move', 'toward', 'goal', 'priorit', 'shallow', 'work', '“the', 'abil', 'perform', 'deep', 'work', 'becom', 'increasingli', 'rare', 'exactli', 'time', 'becom', 'increasingli', 'valuabl', 'economi', 'consequ', 'cultiv', 'skill', 'make', 'core', 'work', 'life', 'thrive”', 'cal', 'newport', '19', 'pursu', '“logical”', 'goal', '“you', 'need', 'aim', 'beyond', 'capabl', 'need', 'develop', 'complet', 'disregard', 'abil', 'end', 'think', 'you’r', 'unabl', 'work', 'best', 'compani', 'sphere', 'make', 'aim', 'think', 'you’r', 'unabl', 'cover', 'time', 'magazin', 'make', 'busi', 'make', 'vision', 'want', 'realiti', 'noth', 'impossible”', 'paul', 'arden', 'people’', 'goal', 'complet', 'logic', 'don’t', 'requir', 'much', 'imagin', 'certainli', 'don’t', 'requir', 'faith', 'luck', 'magic', 'miracl', 'person', 'believ', 'it’', 'sad', 'skeptic', 'secular', 'mani', 'peopl', 'becom', 'find', 'great', 'pleasur', 'faith', 'spiritu', 'provid', 'context', 'life', 'mean', 'person', 'growth', 'faith', 'allow', 'pursu', 'other', 'call', 'absurd', 'like', 'walk', 'water', 'transcend', 'death', 'truli', 'god', 'thing', 'possibl', 'absolut', 'noth', 'fear', '20', 'seek', 'prais', 'rather', 'critic', 'cultur', 'we’v', 'becom', 'fragil', 'must', 'combin', 'honest', 'feedback', '20', 'compliment', 'get', 'feedback', 'best', 'disprov', 'psychologist', 'call', 'confirm', 'bia', 'tendenc', 'search', 'interpret', 'favor', 'recal', 'inform', 'confirm', 'belief', 'give', 'excess', 'less', 'consider', 'altern', 'possibl', 'it’', 'easi', 'get', 'prais', 'ask', 'famili', 'friend', 'tell', 'exactli', 'want', 'hear', 'instead', 'seek', 'prais', 'work', 'improv', 'seek', 'critic', 'better', 'know', 'work', 'merit', 'someon', 'care', 'enough', 'give', 'unsolicit', 'critiqu', 'someth', 'noteworthi', 'hater', 'robin', 'sharma', 'author', 'monk', 'sold', 'ferrari', 'said', '“hater', 'confirm', 'greatness”', 'realli', 'start', 'show', 'hater', 'intimid', 'rather', 'reflect', 'doyou', 'becom', 'reflect', '21', 'take', 'rather', 'give', 'scarciti', 'perspect', 'help', 'peopl', 'hurt', 'longer', 'advantag', 'perspect', 'see', 'world', 'giant', 'pie', 'everi', 'piec', 'pie', 'pie', 'don’t', 'order', 'win', 'must', 'lose', 'abund', 'perspect', 'one', 'pie', 'infinit', 'number', 'pie', 'want', 'make', 'thu', 'help', 'other', 'actual', 'help', 'make', 'system', 'whole', 'better', 'also', 'build', 'relationship', 'trust', 'confid', 'friend', 'nate', 'realli', 'innov', 'stuff', 'real', 'estat', 'invest', 'compani', 'work', 'he’', 'use', 'strategi', 'one', 'els', 'use', 'he’', 'kill', 'told', 'consid', 'keep', 'strategi', 'secret', 'peopl', 'knew', 'they’d', 'use', 'that’d', 'mean', 'less', 'lead', 'opposit', 'told', 'everyon', 'compani', 'even', 'give', 'ton', 'lead', 'away', 'never', 'seen', 'compani', 'nate', 'know', 'strategi', 'longer', 'work', 'come', 'anoth', 'one', 'that’', 'leadership', 'innov', 'peopl', 'come', 'trust', 'actual', 'they’v', 'come', 'reli', 'develop', 'best', 'strategi', 'nate', 'make', 'pie', 'sever', 'peopl', 'ye', 'also', 'topsel', 'highestearn', 'compani', 'it’', 'give', 'doesn’t', 'hord', 'idea', 'resourc', 'inform', '22', 'creat', 'think', '“should”', 'creat', 'mani', 'entrepreneur', 'design', 'product', '“scratch', 'itch”', 'actual', 'that’', 'load', 'problem', 'solv', 'experi', 'difficulti', 'creat', 'solut', 'musician', 'artist', 'approach', 'work', 'way', 'creat', 'music', 'they’d', 'want', 'listen', 'draw', 'paint', 'they’d', 'want', 'see', 'write', 'book', 'wish', 'written', 'that’', 'person', 'approach', 'work', 'write', 'articl', 'want', 'read', 'work', 'first', 'foremost', 'reson', 'don’t', 'enjoy', 'product', 'work', 'expect', 'peopl', '23', 'look', 'next', 'opportun', 'perfect', 'client', 'perfect', 'opportun', 'perfect', 'circumst', 'almost', 'never', 'happen', 'instead', 'wish', 'thing', 'differ', 'cultiv', 'what’', 'right', 'front', 'rather', 'wait', 'next', 'opportun', 'one', 'hand', 'opportun', 'said', 'anoth', 'way', 'grass', 'greener', 'water', 'see', 'mani', 'peopl', 'leav', 'marriag', 'believ', 'better', 'relationship', '“out”', 'case', 'peopl', 'start', 'new', 'relationship', 'end', 'way', 'previou', 'relationship', 'end', 'problem', 'isn’t', 'circumst', 'problem', 'don’t', 'find', 'soulmat', 'creat', 'soulmat', 'hard', 'work', 'jim', 'rohn', 'said', '“don’t', 'wish', 'easier', 'wish', 'better', 'don’t', 'wish', 'less', 'problem', 'wish', 'skill', 'don’t', 'wish', 'less', 'challeng', 'wish', 'wisdom”', '24', 'wait', 'start', 'don’t', 'purpos', 'carv', 'time', 'everi', 'day', 'progress', 'improv', 'without', 'question', 'time', 'get', 'lost', 'vacuum', 'increasingli', 'crowd', 'live', 'know', 'you’ll', 'old', 'wither', 'wonder', 'time', 'went', 'meredith', 'willson', 'said', '“you', 'pile', 'enough', 'tomorrow', 'you’ll', 'find', 'left', 'noth', 'lot', 'empti', 'yesterdays”', 'wait', 'year', 'long', 'activ', 'start', 'write', 'wait', 'right', 'moment', 'i’d', 'enough', 'time', 'money', 'whatev', 'els', 'thought', 'need', 'wait', 'somehow', 'qualifi', 'permiss', 'want', 'never', 'prequalifi', 'degre', '“live', 'dreams”you', 'qualifi', 'show', 'work', 'get', 'permiss', 'decid', 'life', 'short', 'don’t', 'wait', 'tomorrow', 'someth', 'today', 'futur', 'self', 'either', 'thank', 'shame', 'defend', '25', 'publish', 'earli', 'age', '22', 'toni', 'hsieh', 'ceo', 'zapposcom', 'graduat', 'harvard', 'toni', '23', 'year', 'old', 'six', 'month', 'start', 'linkexchang', 'offer', 'one', 'million', 'dollar', 'compani', 'amaz', 'toni', 'less', 'year', 'stoke', 'get', 'job', 'oracl', 'make', '40k', 'per', 'year', 'much', 'thought', 'discuss', 'partner', 'reject', 'offer', 'believ', 'continu', 'build', 'linkexchang', 'someth', 'bigger', 'true', 'love', 'build', 'creat', 'true', 'pro', 'get', 'paid', 'doesn’t', 'work', 'money', 'true', 'pro', 'work', 'love', 'five', 'month', 'later', 'hsieh', 'offer', '20', 'million', 'dollar', 'jerri', 'yang', 'cofound', 'yahoo', 'blew', 'toni', 'away', 'first', 'thought', '“i’m', 'glad', 'didn’t', 'sell', 'five', 'month', 'ago”', 'howev', 'held', 'cool', 'ask', 'day', 'consid', 'propos', 'make', 'decis', 'term', 'thought', 'thing', 'money', 'know', 'never', 'work', 'anoth', 'day', 'life', 'reflect', 'devis', 'small', 'list', 'thing', 'want', 'condo', 'tv', 'builtin', 'home', 'theater', 'abil', 'go', 'weekend', 'minivac', 'whenev', 'want', 'new', 'comput', 'start', 'anoth', 'compani', 'love', 'idea', 'build', 'grow', 'someth', 'passion', 'motiv', 'wasn’t', 'stuff', 'conclud', 'alreadi', 'afford', 'tv', 'new', 'comput', 'alreadi', 'go', 'weekend', 'minivac', 'whenev', 'want', '23', 'year', 'old', 'determin', 'condo', 'wait', 'sell', 'linkexchang', 'build', 'grow', 'anoth', 'compani', 'year', 'toni', 'reject', '20', 'million', 'dollar', 'offer', 'linkexchang', 'explod', '100', 'employe', 'busi', 'boom', 'yet', 'hsieh', 'longer', 'enjoy', 'cultur', 'polit', 'subtli', 'chang', 'process', 'rapid', 'growth', 'linkexchang', 'longer', 'hsieh', 'group', 'close', 'friend', 'build', 'someth', 'love', 'hire', 'bunch', 'peopl', 'hurri', 'didn’t', 'vision', 'motiv', 'mani', 'new', 'employe', 'didn’t', 'care', 'linkexchang', 'build', 'someth', 'love', 'rather', 'want', 'get', 'rich', 'quick', 'pure', 'selfinterest', 'decid', 'sell', 'compani', 'term', 'microsoft', 'purchas', 'linkexchang', '1998', '265', 'million', 'dollar', 'hsieh', '25', 'year', 'old', 'similar', 'concept', 'emerg', 'convers', 'one', 'year', 'ago', 'jeff', 'goin', 'bestsel', 'author', 'art', 'work', 'ask', 'advic', 'publish', 'book', 'want', 'write', 'said', '“wait', 'don’t', 'jump', 'gun', 'made', 'mistak', 'wait', 'year', 'two', 'you’ll', 'get', '10x', 'bigger', 'advanc', 'chang', 'trajectori', 'whole', 'career”', 'here’', 'work', '20k', 'email', 'subscrib', 'writer', 'get', 'around', '20–40k', 'book', 'advanc', '100–200k', 'email', 'subscrib', 'writer', 'get', 'around', '150–500k', 'book', 'advanc', 'wait', 'year', 'two', 'chang', 'trajectori', 'career', 'life', 'isn’t', 'procrastin', 'it’', 'strategi', 'time', 'even', 'second', 'chang', 'whole', 'life', '26', 'play', '“rules”', '“there', 'noth', 'certain', 'sign', 'insan', 'thing', 'expect', 'result', 'different”', 'albert', 'einstein', 'convent', 'we’r', 'break', 'convent', 'we’ll', 'evolv', 'requir', 'gargantuan', 'quantiti', 'failur', 'don’t', 'grit', 'fail', '10000', 'time', 'you’ll', 'never', 'invent', 'light', 'bulb', 'seth', 'godin', 'said', '“if', 'fail', 'win”', 'failur', 'someth', 'prize', 'prais', 'failur', 'feedback', 'failur', 'move', 'forward', 'it’', 'consciou', 'exert', 'effort', 'toward', 'someth', 'you’v', 'never', 'done', 'it’', 'incred', '“the', 'person', 'doesn’t', 'make', 'mistak', 'unlik', 'make', 'anything”', 'paul', 'arden', '27', 'set', 'win', 'start', '“peopl', 'may', 'spend', 'whole', 'live', 'climb', 'ladder', 'success', 'find', 'reach', 'top', 'ladder', 'lean', 'wrong', 'wall”', 'thoma', 'merton', 'mani', 'peopl', 'play', 'wrong', 'game', 'lose', 'game', 'onset', 'hurt', 'like', 'hell', 'it’', 'ruin', 'life', 'without', 'even', 'know', 'import', 'play', '“the', 'game”', 'game', 'set', 'set', 'game', 'determin', 'play', 'it’', 'better', 'win', 'first', 'play', 'work', 'start', 'end', 'work', 'backward', 'rather', 'think', 'what’', 'plausibl', 'what’', 'expect', 'make', 'sens', 'start', 'want', 'covey', 'put', '7', 'habit', '“begin', 'end', 'clearli', 'mind”', 'that’', 'nail', 'dictat', 'daili', 'weekli', 'monthli', 'yearli', 'behavior', 'facilit', 'jim', 'carrey', 'wrote', '10', 'million', 'check', 'set', 'earn', 'game', 'first', 'play', '28', 'leverag', 'everi', 'small', 'win', 'get', 'matter', 'small', 'win', 'along', 'way', 'leverag', 'posit', 'high', 'school', 'diploma', 'leverag', 'posit', 'know', 'guy', 'know', 'guy', 'know', 'guy', 'leverag', 'posit', 'get', 'articl', 'featur', 'unknown', 'blog', 'leverag', 'posit', '100', 'leverag', 'posit', 'sadli', 'peopl', 'can’t', 'stop', 'look', 'side', 'fenc', 'fail', 'realiz', 'brilliant', 'possibl', 'current', 'avail', 'bad', 'stewardship', 'peopl', 'alreadi', 'know', 'inform', 'need', 'peopl', 'alreadi', 'know', 'capit', 'use', 'peopl', 'alreadi', 'know', 'connect', 'peopl', 'know', 'instead', 'want', 'util', 'alreadi', 'won’t', 'help', 'actual', 'continu', 'hurt', 'learn', 'earn', 'someth', 'it’', 'easi', 'want', 'peopl', 'real', 'success', 'come', 'take', 'ownership', 'life', 'one', 'els', 'care', 'success', 'health', 'relationship', 'time', 'current', 'posit', 'ripe', 'abund', 'opportun', 'leverag', 'gain', 'anoth', 'inch', 'posit', 'leverag', 'it’', 'worth', 'don’t', 'wish', 'wish', 'better', 'soon', 'enough', 'you’ll', 'find', 'incred', 'posit', 'collabor', 'hero', 'success', 'base', 'choic', 'success', 'base', 'maintain', 'motiv', 'worth', 'fight', 'it’', 'base', 'believ', 'other', 'might', 'call', 'fantasi', 'it’', 'base', 'leverag', 'posit', 'maintain', 'momentum', 'everi', 'step', 'take', '29', 'view', 'work', '“work”', 'cool', 'part', 'poetri', 'poet', 'poem', 'perform', 'import', 'import', 'actual', 'said', 'similar', 'way', 'go', 'event', 'hear', 'speech', 'you’r', 'usual', 'go', 'see', 'speaker', 'hear', 'say', 'alreadi', 'know', 'say', 'matter', 'type', 'work', 'better', 'receiv', 'see', 'artform', 'perform', 'audienc', 'want', 'much', 'want', 'work', 'often', '30', 'let', 'other', 'decid', 'work', 'ryan', 'holiday', 'author', 'obstacl', 'way', 'explain', 'call', '“the', 'moment”', 'everi', 'skill', 'creativ', 'experienc', '“the', 'moment”', 'eye', 'open', 'mechan', 'behindthescen', 'craft', 'moment', 'seem', 'like', 'magic', 'idea', 'peopl', 'creat', 'creat', 'moment', 'realiz', 'everyth', 'done', 'person', 'intent', 'creat', 'particular', 'experi', 'recent', 'watch', 'lord', 'ring', 'dawn', 'movi', 'complet', 'differ', 'weren’t', 'direct', 'peter', 'jackson', 'complet', 'differ', 'everi', 'shot', 'everi', 'set', 'light', 'costum', 'charact', 'landscap', 'look', 'whole', 'film', 'feel', 'portray', 'look', 'felt', 'complet', 'differ', 'base', 'experi', 'differ', 'director', 'tri', 'creat', 'thu', 'right', 'wrong', 'way', 'rather', 'it’', 'thing', 'way', 'experi', '“moment”', 'you’ll', 'continu', 'attempt', 'correct', 'best', 'way', 'thing', 'you’ll', 'continu', 'copi', 'people’', 'work', 'persist', 'you’ll', 'becom', 'disillus', 'idol', 'peopl', 'like', 'they’v', 'made', 'decis', 'creat', 'way', 'idea', 'imit', 'becom', 'abhorr', 'free', 'creat', 'see', 'fit', 'you’ll', 'emerg', 'voic', 'origin', 'work', 'you’ll', 'less', 'troubl', 'work', 'receiv', 'focus', 'creat', 'someth', 'believ', '31', 'seek', 'retir', '“to', 'retir', 'die”', 'pablo', 'casal', 'power', 'way', 'punch', 'someon', 'face', 'aim', 'foot', 'behind', 'face', 'way', 'full', 'momentum', 'power', 'make', 'contact', 'aim', 'face', 'time', 'reach', 'you’ll', 'alreadi', 'begun', 'slow', 'thu', 'punch', 'power', 'intend', 'retir', 'way', 'peopl', 'plan', 'retir', 'begin', 'slow', '40’', '50’', 'sad', 'part', 'momentumbas', 'be', 'begin', 'slow', 'start', 'hardtorevers', 'decay', 'process', 'research', 'found', 'retir', 'often', 'increas', 'difficulti', 'mobil', 'daili', 'activ', 'increas', 'likelihood', 'becom', 'ill', 'decreas', 'mental', 'health', 'retir', '20th', 'centuri', 'phenomena', 'actual', 'foundat', 'gird', 'outdat', 'notion', 'make', 'littl', 'sens', 'modern', 'futur', 'societi', 'instanc', 'due', 'advanc', 'health', 'care', '65', 'consid', 'old', 'age', 'anymor', 'social', 'secur', 'system', 'design', 'planner', 'chose', 'age', '65', 'averag', 'lifespan', 'age', '63', 'time', 'thu', 'system', 'design', 'realli', 'need', 'creat', 'cultur', 'peopl', 'support', 'others’', 'labor', 'furthermor', 'percept', 'peopl', '65', 'can’t', 'provid', 'meaning', 'work', 'longer', 'make', 'sens', 'either', 'retir', 'becam', 'thing', 'work', 'manual', 'labor', 'today’', 'work', 'knowledgebas', 'there’', 'anyth', 'lack', 'today’', 'societi', 'wisdom', 'peopl', 'later', 'year', 'spent', 'lifetim', 'refin', 'retir', 'never', 'goal', 'fulli', 'capabl', 'work', 'capac', 'final', 'breath', '92', 'year', 'old', 'grandfath', 'rex', 'fighter', 'pilot', 'wwii', 'past', 'five', 'year', 'he’', 'written', 'three', 'book', 'goe', 'bed', 'everi', 'night', '8', 'pm', 'wake', 'everi', 'morn', '430', 'spend', 'first', '25', 'hour', 'day', 'watch', 'inspir', 'instruct', 'content', 'televis', 'eat', 'breakfast', '7', 'spend', 'day', 'read', 'write', 'connect', 'serv', 'peopl', 'even', 'physic', 'labor', 'around', 'son’', 'dad’', 'hous', 'even', 'walk', 'around', 'neighborhood', 'proselyt', 'faith', 'ask', 'random', 'stranger', 'help', 'intent', 'stop', 'slow', 'contrari', 'popular', 'belief', 'human', 'like', 'wine', 'get', 'better', 'age', '32', 'downplay', 'import', 'yesterday', '“the', 'best', 'time', 'plant', 'tree', '20', 'year', 'ago', 'second', 'best', 'time', 'now”', 'chines', 'proverb', 'present', 'circumst', 'reflect', 'past', 'decis', 'although', 'enorm', 'power', 'chang', 'trajectori', 'live', 'noww', 'past', 'it’', 'popular', 'say', 'past', 'doesn’t', 'matter', 'simpli', 'true', 'today', 'tomorrow’', 'yesterday', 'today', 'either', 'enhanc', 'diminish', 'futurepres', 'moment', 'peopl', 'put', 'thing', 'tomorrow', 'thoughtlessli', 'go', 'debt', 'forego', 'exercis', 'educ', 'justifi', 'neg', 'relationship', 'point', 'catch', 'like', 'airplan', 'offcours', 'longer', 'wait', 'correct', 'longer', 'harder', 'get', 'back', 'oncours', 'time', 'absolut', 'marvel', 'get', 'anticip', 'experi', 'want', 'often', 'enjoy', 'experi', 'get', 'experi', 'long', 'get', 'rememb', 'carri', 'experi', 'us', 'forev', 'past', 'present', 'futur', 'uniqu', 'import', 'enjoy', '33', 'believ', 'you’r', 'way', 'behind', 'sport', 'form', 'competit', 'peopl', 'perform', 'best', 'game', 'close', 'big', 'magic', 'happen', 'end', 'game', 'like', 'onsid', 'kick', 'retriev', 'follow', '30', 'second', 'touchdown', 'drive', 'contest', 'decidedli', 'one', 'opponent’', 'favor', 'neither', 'side', 'act', 'effort', 'you’r', 'win', 'big', 'it’', 'easi', 'get', 'lax', 'overconfid', 'you’r', 'lose', 'big', 'it’', 'easi', 'give', 'sadli', 'probabl', 'perceiv', 'top', 'field', '“in', 'differ', 'league”', 'altogeth', 'perform', 'less', 'intens', 'perceiv', '“game”', 'closer', 'elev', 'think', 'see', 'level', '“the', 'top”', 'quickli', 'becom', 'disillus', 'fallibl', 'perceiv', 'immort', 'peopl', 'importantli', 'begin', 'play', 'urgenc', 'often', 'surpass', 'even', 'game', 'close', 'game', 'close', '34', 'listen', 'music', 'destroy', 'joy', 'product', '“without', 'music', 'life', 'mistake”', 'friedrich', 'nietzsch', 'also', 'use', 'music', 'trigger', 'optim', 'perform', 'exampl', 'michael', 'phelp', 'routin', 'religi', 'swim', 'event', 'involv', 'music', 'he’', 'alon', 'mani', 'athlet', 'use', 'music', 'event', 'trigger', 'relax', 'pressur', 'even', 'psych', 'ask', 'time', 'magazin', 'use', 'music', 'prior', 'race', 'phelp', 'said', 'kept', 'focus', 'help', '“tune', 'everyth', 'take', 'one', 'step', 'time”', 'ask', 'kind', 'music', 'listen', 'answer', '“i', 'listen', 'hip', 'hop', 'rap”', 'interestingli', 'research', 'found', 'high', 'tempo', 'music', 'like', 'hip', 'hop', 'creat', 'strong', 'arous', 'perform', 'readi', 'evid', 'find', 'intens', 'emot', 'respons', 'linger', 'long', 'music', 'stop', 'phelp', 'water', 'swim', 'he’', 'still', 'hype', 'hip', 'hop', 'lastli', 'research', 'found', 'type', 'music', 'listen', 'impact', 'level', 'spiritu', 'last', 'point', 'particularli', 'import', 'spiritu', 'heavili', 'influenc', 'everyth', 'interact', 'famili', 'write', 'develop', 'pursu', 'goal', '35', 'call', 'action', 'want', 'focu', 'right', 'activ', 'get', 'result', '10x', 'faster', 'peopl', 'check', 'morn', 'checklist', 'click', 'get', 'checklist', 'right']

In [74]:

    

# #### testing removal of punctuation

# test = ' '.join(set_tr['text'][0])
# # print(test.lower())
# # tkns = word_tokenizer.tokenize(test.lower().translate(dict.fromkeys(string.punctuation)))
# # print(tkns)
# # test2 = ''.join(e for e in test if e.isalnum())
# # print(test2)
# # test3 = test.translate(None, string.punctuation)
# # print(test3)
# # tkns = nltk.word_tokenize(test.lower().translate(dict.fromkeys(string.punctuation)))
# # print(tkns)
# translator = str.maketrans('', '', string.punctuation)

# test2 = re.sub(u'\u2014','',test)

# print(test2.translate(translator))


    

    




httpsunsplashcomphotopFqrYbhIAXs 35 Things You Need to Give Up to Be Successful 1 Wasting Five Minutes When you have five minutes of downtime how do you spend that time Most people use it as an excuse to rest or laze By lazing for 5 five minute breaks each day we waste 25 minutes daily That’s 9125 minutes per year 25 X 365 Sadly my guess is we’re wasting far more time than that I was once told by my 9th grade English teacher that if I read every time I had a break  even if the break was just for a minute or two  that I’d get a lot more reading done than expected She was right Every time I finished my work early or had a spare moment I’d pick up a book and read How we spend our periodic five minute breaks is a determining factor to what we achieve in our lives Every little bit adds up Why can we justify wasting so much time 2 Not Valuing One Dollar I was recently in WalMart with my motherinlaw buying a few groceries While we were in the checkout line I pointed an item out to her I thought was interesting honestly can’t remember what it is anymore What stuck out to me is that she said “One dollar That’s a lot of money” Why this surprised me is that my inlaws are not short of money Actually this happened while we were on a family trip 30 people at Disney World  the whole thing being paid for by them Understanding the value of one dollar is the same as coming to appreciate the value of time To thoughtlessly spend one dollar may not seem like a big deal but it actually is That frivolous spending compounded over a long enough time could be millions It also reflects a lack of care about the details which is where the true art and value lies Additionally most millionaires are “selfmade” 80 percent being firstgeneration rich and 75 percent being selfemployed Not getting paid hourly challenges you to take more responsibility for every minute and every dollar Consequently a great majority of millionaires are extremely frugal  or at least highly mindful  with their money 3 Believing Success Will Make You Happy “One of the enemies of happiness is adaptation” says Dr Thomas Gilovich a Cornell psychologist who has studied the relationship between money and happiness for over two decades “We buy things to make us happy and we succeed But only for a while New things are exciting to us at first but then we adapt to them” Gilovich further states Actually savoring the anticipation or idea of a desired outcome is generally more satisfying than the outcome itself Once we get what we want  whether that’s wealth health or excellent relationships  we adapt and the excitement fades Often the experiences we’re seeking end up being underwhelming and even disappointing I love watching this phenomena in our foster kids They feel like they need a certain toy or the universe will explode Their whole world revolves around getting this one thing Yet once we buy the toy for them it’s not long before the joy fades and they want something else Until you appreciate what you currently have more won’t make your life better 4 Believing You’re Not Up For the Challenge Just as we deceive ourselves into believing something will make us happier than it will we also deceive ourselves into believing something will be harder than it will The longer you procrastinate or avoid doing something the more painful in your head it becomes However once you take action the discomfort is far less severe than you imagined Even to extremely difficult things humans adapt I recently sat on a plane with a lady who has 17 kids Yes you read that correctly After having eight of her own her and her husband felt inspired to foster four siblings whom they later adopted A few years later they took on another five foster siblings whom they also adopted Of course the initial shock to the system impacted her entire family But they’re handling it And believe it or not you could handle it too if you had to The problem with dread and fear is that it holds people back from taking on big challenges What you will find  no matter how big or small the challenge  is that you will adapt to it When you consciously adapt to enormous stress you evolve 5 Pursuing “Happiness” “There is no way to happiness  happiness is the way”  Thich Nhat Hanh Most people believe they must First have something eg money time or love Before they can do what they want to do eg travel the world write a book start a business or have a romantic relationship Which will ultimately allow them to be something eg happy peaceful content motivated or in love Paradoxically this have  do  be paradigm must actually be reversed to experience happiness success or anything else you desire First you be whatever it is you want to be eg happy compassionate peaceful wise or loving Then you start doing things from this space of being Almost immediately what you are doing will bring about the things you want to have You attract what you are If you want the things happy people have you must be happy to get those things If you want things wealthy people have you must be and live wealthy to have those things Results translate from attitudes and behaviors Not the other ways around 6 Undervaluing What You Have In an interview at the annual Genius Network Event in 2013 Tim Ferriss was asked “With all of your various roles do you ever get stressed out Do you ever feel like you’ve taken on too much” Ferriss responded “Of course I get stressed out If anyone says they don’t get stressed out they’re lying But one thing that mitigates that is taking time each morning to declare and focus on the fact that ‘I have enough’ I have enough I don’t need to worry about responding to every email each day If they get mad that’s their problem” Ferriss was later asked during the same interview “After having read The 4Hour Workweek I got the impression that Tim Ferriss doesn’t care about money You talked about how you travel the world without spending any money Talk about the balance and ability to let go of caring about making money” Ferriss responded “It’s totally okay to have lots of nice things If it is addiction to wealth like in Fight Club “The things you own end up owning you” and it becomes a surrogate for things like longterm health and happiness  connection  then it becomes a disease state But if you can have nice things and not fear having them taken away then it’s a good thing Because money is a really valuable tool” If you appreciate what you already have than more will be a good thing in your life If you feel the need to have more to compensate for something missing in your life you’ll always be left wanting  no matter how much you acquire or achieve 7 Downplaying Your Current Position It’s easy to talk about how hard our lives are It’s easy to talk about how unfair life is And that we got the shortend of the stick But does this kind of talking really help anyone When we judge our situation as worse than someone else’s we are ignorantly and incorrectly saying “You’ve got it easy You’re not like me Success should come easy to you because you haven’t had to deal with what I’ve gone through” This paradigm has formally become known as the victim mentality and it generally leads to feelings of entitlement The world owes you nothing Life isn’t meant to be fair However the world has also given you everything you need The truth is you have every advantage in the world to succeed And by believing this in your bones you’ll feel an enormous weight of responsibility to yourself and the world You’ve been put in a perfect position to succeed Everything in the universe has brought you to this point so you can now shine and change the worldThe world is your oyster Your natural state is to thrive All you have to do is show up 8 Compartmentalizing Your Life Human beings are holistic  when you change a part of any system you simultaneously change the whole You can’t change a part without fundamentally changing everything Every pebble of thought  no matter how inconsequential  creates endless ripples of consequence This idea coined the butterfly effect by Edward Lorenz came from the metaphorical example of a hurricane being influenced by minor signals  such as the flapping of the wings of a distant butterfly  several weeks earlier Little things become big things When one area of your life is out of alignment every area of your life suffers You can’t compartmentalize a working system Although it’s easy to push certain areas  like your health and relationships  to the side you unwittingly infect your whole life Eventually and always the essentials you procrastinate or avoid will catch up to your detriment Conversely when you improve one area of your life all other areas are positively influenced As James Allen wrote in As a Man Thinketh “When a man makes his thoughts pure he no longer desires impure food” We are holistic systems Humanity as a whole is the same way Everything you do effects the whole world for better or worse So I invite you to ask “Am I part of the cure Or am I part of the disease”  Coldplay 9 Competing With Other People “All failed companies are the same they failed to escape competition”  Peter Thiel Competition is extremely costly to maximum product reach and wealth creation It becomes a battle of who can slightly outdo the other for cheaper and cheaper It’s a race to the bottom for all parties involved Instead of trying to compete with other people or businesses it’s better to do something completely novel or to focus on a tightly defined niche Once you’ve established yourself as an authority over something you can set your own terms  rather than reactively responding to the competition Thus you want to monopolize the space in which you create value Competing with others leads people to spend every day of their lives pursuing goals that aren’t really their own  but what society has deemed important You could spend your whole life trying to keep up but will probably have a shallow life Or you can define success for yourself based on your own values and detach yourself from the noise 10 Trying To Have It All Every decision has opportunity cost When you choose one thing you simultaneously don’t choose several others When someone says you can have it all they are lying They are almost certainly not practicing what they preach and are trying to sell you on something The truth is you don’t want it all And even if you did reality simply doesn’t work that way For example I’ve come to terms with the fact that I want my family to be the center of my life Spending time with my wife and three foster kids is my top priority As a result I can’t spend 12 or 15 hours a day working like some people And that’s okay I’ve made my choice And that’s the point We all need to choose what matters most to us and own that If we attempt to be everything we’ll end up being nothing Internal conflict is hell Although the traditional view of creativity is that it is unstructured and doesn’t follow rules creativity usually occurs by thinking inside the proverbial box not outside of it People flex their creative muscles when they constrain their options rather than broaden them Hence the more clearly defined and constraining your life’s objectives the better because it allows you to sever everything outside those objectives 11 Forgetting Where You Came From It’s easy when you achieve any level of success to believe you are solely responsible for that success It’s easy to forget where you came from It’s easy to forget all the sacrifices other people have made to get you where you are It’s easy to see yourself as superior to other people Burn all your bridges and you’ll have no human connection left In that internal cave of isolation you’ll lose your mind and identity becoming a person you never intended to be Humility gratitude and recognition of your blessings keeps your success in proper perspective You couldn’t do what you’ve without the help of countless other people You are extremely lucky to be able to contribute in the way you have 12 Seeking Permission From Others My fatherinlaw is a highly successful realestate investor Throughout his career he’s had hundreds of people ask him if they should “go into realestate” He tells every one of them the same thing that they shouldn’t do it In fact he actually tries talking most of them out of it And in most cases he succeeds Why would he do that “Those who are going to succeed will do so regardless of what I say” my fatherinlaw told me I know so many people who chase whatever worked for other people They never truly decide what they want to do and end up jumping from one thing to the next  trying to strike quick gold And repetitively they stop digging just a few feet from the gold after resigning the spot is barren No one will ever give you permission to live your dreams As Ryan Holiday has said in The Obstacle is the Way “Stop looking for angels and start looking for angles” Rather than hoping for something external to change your circumstances mentally reframe yourself and your circumstances “When you change the way you see things the things you see change”  Wayne Dyer You are enough You can do whatever you decide to do Make the decision and forget what everyone else says or thinks about it 13 Letting Others Determine How Much Money You Can Earn Most people “say” they want to be successful But if they really wanted to they’d be successful I used to tell people “I wish I played the piano” Then someone said “No you don’t If you did you’d make the time to practice” I’ve since stopped saying that because he was right Life is a matter of priority and decision And when it comes to money  in a freemarket economy  you can make as much money as you choose The question is how much money do you really want to make Instead of vegging on social media dayafterday yearafteryear you could spend an hour or two each day building something of value  like yourself In the book Think and Grow Rich Napoleon Hill invites readers to write down on a piece of paper the amount of money they want to make and to put a timeline on it This single act will challenge you to think and act in new ways to create the future of your wanting For example despite growing up so poor that for a time his family lived in their Volkswagen van on a relative’s lawn Jim Carrey believed in his future Every night in the late 1980’s Carrey would drive atop a large hill that looked down over Los Angeles and visualize directors valuing his work At the time he was a broke and struggling young comic One night in 1990 while looking down on Los Angeles and dreaming of his future Carrey wrote himself a check for 10 million and put in the notation line “for acting services rendered” He dated the check for Thanksgiving 1995 and stuck it in his wallet He gave himself five years And just before Thanksgiving of 1995 he got paid 10 million for Dumb and Dumber 14 Ignoring the Vision You Have For Your Future “Create the highest grandest vision possible for your life because you become what you believe”  Oprah Winfrey No matter where you are right now you can have any future you want But one thing is for certain what you plant you must harvest So please plant with intention Mental creation always precedes physical creation The blueprint you design in your head becomes the life you build Don’t let society tell you how your house should look You are an artist and a creator Your life can be exactly how you want it whether or not it’s considered a “mansion” by others Home is where your heart is 15 Doing Not Being There’s a parable of a wealthy parent who hesitated giving their unwise child an inheritance knowing it would undoubtedly be squandered The parent said to the child “All that I have I desire to give you  not only my wealth but also my position and standing among men That which I have I can easily give you but that which I am you must obtain for yourself You will qualify for your inheritance by learning what I have learned and by living as I have lived I will give you the laws and principles by which I have acquired my wisdom and stature Follow my example mastering as I have mastered and you will become as I am and all that I have will be yours” Going through the motions is not enough There isn’t a checklist of things you must do to be successful You have to fundamentally change who you are to live at a higher level You must shift from doing to being  so that what you do is a reflection of who you are and who you’re becoming Once you’ve experienced this change success will be natural Be then do then have see point3 above “After you become a millionaire you can give all of your money away because what’s important is not the million dollars what’s important is the person you have become in the process of becoming a millionaire”  Jim Rohn 16 Believing Money Is Evil “For better or worse humans are holistic Even the human body does best when its spiritual and physical sides are synchronized… People’s bodies perform best when their brains are on board with the program… Helping your mind to believe what you do is good noble and worthwhile in itself helps to fuel your energies and propel your efforts”  Rabbi Daniel Lapin I know so many people who genuinely believe making money is immoral and that people with money are evil They believe those who seek profits force those weaker than them to buy their products Money is not evil but neutral It is a symbol of perceived value If I’m selling a pair of shoes for 20 and someone decides to buy them they perceive the shoes to be worth more than the 20 or they wouldn’t buy them I’m not forcing them to buy my shoes It’s their choice Thus value exchange is winwin and based purely on perception Value is subjective If you offered that same person 20 for the shoes they just bought they probably wouldn’t sell them They see them as worth more than 20 But what if you offered 30 They still might not sell them There is no “correct” price for goods and services The correct price is the perceived worth from the customer If the price is too high the customer won’t exchange their money for it We are extremely lucky to live in a society with a system of money It allows us to borrow lend and leverage Our ability to scale our work would be enormously limited in a bartering and trading system Earning money is a completely moral pursuit when it is done with honesty and integrity In fact if you don’t feel moral about the work you’re doing you should probably change your job When you believe in the value you provide so much that you are doing people a disservice by not offering them your services you’re on track to creating colossal value Our work should be a reflection of us It’s always their choice whether they perceive the value in what we’re offering or not 17 Continuing to be Distracted “You cannot overestimate the unimportance of practically everything”  Greg McKeown Almost everything is a distraction from what really matters You really can’t put a pricetag on certain things They are beyond a particular value to you You’d give up everything even your life for those things Your relationships and personal values don’t have a pricetag And you should never exchange something priceless for a price Keeping things in proper perspective allows you to remove everything nonessential from your life It allows you to live simply and laser focused and toavoid deadend roads leading nowhere 18 Not Realizing that Focus is Today’s IQ We live in the most distracted era of human history The internet is a doubleedged sword Like money the internet is neutral  and it can be used for good or bad based on who uses it Sadly most of us are simply not responsible enough for the internet We waste hours every day staring idly at a screen Millennials are particularly prone to distractions on the internet but nowadays everyone is susceptible Our attention spans have shrunk to almost nothing Our willpower has atrophied We’ve developed some really bad habits that often require extreme interventions to reverse There is a growing body of scientific evidence suggesting the internet  withits constant distractions and interruptions  is turning us into scattered and superficial thinkers One of the biggest challenges to constant distraction is that it leads to “shallow” rather than “deep” thinking and shallow thinking leads to shallow living The Roman philosopher Seneca may have put it best 2000 years ago “To be everywhere is to be nowhere” In his book Deep Work Rules for Focused Success in a Distracted World Cal Newport differentiates “deep work” from “shallow work” Deep work is using your skills to create something of value It takes thought energy time and concentration Shallow work is all the little administrative and logistical stuff email meetings calls expense reports etc Most people aren’t moving toward their goals because they prioritize shallow work “The ability to perform deep work is becoming increasingly rare at exactly the same time it is becoming increasingly valuable in our economy As a consequence the few who cultivate this skill and then make it the core of their working life will thrive”  Cal Newport 19 Pursuing “Logical” Goals “You need to aim beyond what you are capable of You need to develop a complete disregard for where your abilities end If you think you’re unable to work for the best company in its sphere make that your aim If you think you’re unable to be on the cover of Time magazine make it your business to be there Make your vision of where you want to be a reality Nothing is impossible”  Paul Arden Most people’s goals are completely logical They don’t require much imagination They certainly don’t require faith luck magic or miracles Personally I believe it’s sad how skeptical and secular many people are becoming I find great pleasure in having faith in the spiritual It provides context for life and meaning for personal growth Having faith allows me to pursue that which others would call absurd like walking on water and transcending death Truly with God all things are possible There is absolutely nothing to fear 20 Seeking Praise Rather than Criticism As a culture we’ve become so fragile that we must combine honest feedback with 20 compliments And when we get feedback we do our best to disprove it Psychologists call this confirmation bias  the tendency to search for interpret favor and recall information that confirms our own beliefs while giving excessively less consideration to alternative possibilities It’s easy to get praise when you ask family and friends who will tell you exactly what you want to hear Instead of seeking praise your work will improve if you seek criticism How could this be better You will know your work has merit when someone cares enough to give unsolicited critique If something is noteworthy there will be haters As Robin Sharma author of The Monk Who Sold His Ferrari has said “haters confirm greatness” When you really start showing up the haters will be intimidated by you Rather than being a reflection of what they could doyou become a reflection of what they are not doing 21 Taking Rather Than Giving From a scarcity perspective helping other people hurts you because you no longer have the advantage This perspective sees the world as a giant pie Every piece of the pie you have is pie I don’t have So in order for you to win I must lose From an abundance perspective there is not only one pie but an infinite number of pies If you want more you make more Thus helping others actually helps you because it makes the system as a whole better It also builds relationships and trust and confidence I have a friend Nate who is doing some really innovative stuff at the real estate investing company he works for He’s using strategies that no one else is using And he’s killing it He told me he considered keeping his strategies a secret Because if other people knew about them they’d use them and that’d mean less leads for him But then he did the opposite He told everyone in his company about what he was doing He has even been giving tons of his leads away This has never been seen before in his company But Nate knows that once this strategy no longer works he can come up with another one And that’s what leadership and innovation is all about And people have come to trust him Actually they’ve come to rely on him for developing the best strategies Nate makes pies  for himself and several other people And yes he is also the topselling and highestearning in his company It’s because he gives the most and doesn’t horde his ideas resources or information 22 Creating What You Think You “Should” Create Many entrepreneurs design products to “scratch their own itch” Actually that’s how loads of problems are solved You experience a difficulty and create a solution Musicians and artists approach their work the same way They create music they’d want to listen to draw painting they’d want to see and write books they wish were written That’s how I personally approach my work I write articles I myself would want to read Your work should first and foremost resonate with yourself If you don’t enjoy the product of your work how can you expect other people to 23 Looking For The Next Opportunity The perfect client perfect opportunity and perfect circumstances will almost never happen Instead of wishing things were different why not cultivate what’s right in front of you Rather than waiting for the next opportunity the one in your hands is the opportunity Said another way the grass is greener where you water it I see so many people leave marriages because they believe better relationships are “out” there In most cases these people start new relationships and end them the same way the previous relationship ended The problem isn’t your circumstances The problem is you You don’t find your soulmate you create your soulmate through hard work As Jim Rohn said “Don’t wish it was easier wish you were better Don’t wish for less problems wish for more skills Don’t wish for less challenge wish for more wisdom” 24 Waiting To Start If you don’t purposefully carve time out every day to progress and improve  without question your time will get lost in the vacuum of our increasingly crowded lives Before you know it you’ll be old and withered  wondering where all that time went As Meredith Willson has said  “You pile up enough tomorrows and you’ll find you are left with nothing but a lot of empty yesterdays” I waited a few years too long to actively start writing I was waiting for the right moment when I’d have enough time money and whatever else I thought I needed I was waiting until I was somehow qualified or had permission to do what I wanted to do But you are never prequalified There is no degree for “Live your dreams”You qualify yourself by showing up and working You get permission by deciding Life is short Don’t wait for tomorrow for something you could do today Your future self will either thank you or shamefully defend you 25 Publishing Too Early At age 22 Tony Hsieh now CEO of Zapposcom graduated from Harvard When Tony was 23 years old six months after starting Linkexchange he was offered one million dollars for the company This was amazing to Tony because less than a year before he was stoked to get a job at Oracle making 40K per year After much thought and discussion with his partner he rejected the offer believing he could continue to build Linkexchange into something bigger His true love is in building and creating A true pro gets paid but doesn’t work for money A true pro works for love Five months later Hsieh was offered 20 million dollars from Jerry Yang cofounder of Yahoo This blew Tony away His first thought was “I’m glad I didn’t sell five months ago” However he held his cool and asked for a few days to consider the proposal He would make this decisions on his terms He thought about all the things he would do if he had all that money knowing he would never have to work another day in his life After reflecting he could only devise a small list of things he wanted A condo A TV and builtin home theater The ability to go on weekend minivacations whenever he wanted A new computer To start another company because he loves the idea of building and growing something That was it His passion and motivation wasn’t in having stuff He concluded that he could already afford a TV a new computer and could already go on weekend minivacations whenever he wanted He was only 23 years old so he determined a condo could wait Why would he sell Linkexchange just to build and grow another company A year after Tony rejected the 20 million dollar offer Linkexchange exploded There were over 100 employees Business was booming Yet Hsieh no longer enjoyed being there The culture and politics had subtly changed in the process of rapid growth Linkexchange was no longer Hsieh and a group of close friends building something they loved They had hired a bunch of people in a hurry who didn’t have the same vision and motivations they had Many of the new employees didn’t care about Linkexchange or about building something they loved Rather they just wanted to get rich quick  purely selfinterested So he decided to sell the company on his terms Microsoft purchased Linkexchange in 1998 for 265 million dollars when Hsieh was 25 years old A similar concept emerged in a conversation I had about one year ago with Jeff Goins bestselling author of The Art of Work I asked his advice about publishing a book I want to write and he said “Wait Don’t jump the gun on this I made that mistake myself If you wait a year or two you’ll get a 10x bigger advance which will change the trajectory of your whole career” Here’s how it works With 20K email subscribers a writer can get around a 20–40K book advance But with 100–200K email subscribers a writer can get around a 150–500K book advance Wait a year or two and change the trajectory of your career and life This isn’t about procrastination It’s about strategy Timing  even a few seconds  could change your whole life 26 Playing By The “Rules” “There is nothing that is a more certain sign of insanity than to do the same thing over and over and expect the results to be different”  Albert Einstein Convention is where we’re at Breaking convention is how we’ll evolve which requires a gargantuan quantity of failure If you don’t have the grit to fail 10000 times you’ll never invent your light bulb As Seth Godin has said “If I fail more than you do I win” Failure is something to be prized and praised Failure is feedback Failure is moving forward It’s conscious and exerted effort toward something you’ve never done before It’s incredible “The person who doesn’t make mistakes is unlikely to make anything”  Paul Arden 27 Not Setting Yourself Up To Win Before You Start “People may spend their whole lives climbing the ladder of success only to find once they reach the top that the ladder is leaning against the wrong wall”  Thomas Merton Too many people are playing the wrong game  a losing game from the onset  and it hurts like hell It’s how you ruin your life without even knowing it More important than playing “the game” is how the game is set up How you set up the game determines how you play And it’s better to win first then play How does this work Start from the end and work backwards Rather than thinking about what’s plausible or what’s expected or what makes sense  start with what you want Or as Covey put it in 7 Habits “Begin with the end clearly in mind” Once that’s nailed down then dictate the daily weekly monthly and yearly behaviors that will facilitate that Jim Carrey wrote himself a 10 million check Then he set out to earn it He won the game first then played So can you 28 Not Leveraging Every Small Win You Get No matter how small your wins along the way are leverage your position You have a high school diploma Leverage your position You know a guy who knows a guy who knows a guy Leverage your position You get an article featured on some unknown blog Leverage your position You have 100 Leverage your position Sadly most people can’t stop looking at the other side of the fence They fail to realize the brilliant possibilities currently available to them This is bad stewardship There are people you already know who have information you need There are people you already know who have capital you can use There are people you already know who can connect you with people you should know Instead of wanting more how about you utilize what you already have Until you do more won’t help you Actually it will only continue hurting you until you learn to earn something for yourself It’s easy to want other people to do it for you But real success comes when you take ownership of your life No one else cares more about your success or health or relationships or time than you do Your current position is ripe with abundant opportunity Leverage it Once you gain another inch of position leverage it for all it’s worth Don’t wish for more Wish you were better And soon enough you’ll find yourself in incredible positions and collaborating with your heroes Success is based on choice Success is based on having and maintaining a motivation worth fighting for It’s based on believing what others might call a fantasy It’s based on leveraging your position and maintaining the momentum of every step you take 29 Viewing Your Work as “Work” The cool part about poetry is that to most poets how their poems are performed is just as important  if not more important  than what is actually said In a similar way when you go to an event or to hear a speech you’re usually going to see the speaker not hear what they have to say You already know what they have to say No matter what type of work you are in it will be better received if you see it as an artform You are performing for an audience They want you just as much as they want your work  often more 30 Letting Others Decide How Your Work Should Be Ryan Holiday author of The Obstacle is the Way explains what he calls “the moment” which every skilled creative has experienced “The moment” is when your eyes are opened to the mechanics and behindthescenes of your craft Until you have this moment it all seems like magic to you You have no idea how people create what they create After you have this moment you realize that everything is done by a person intentionally creating a particular experience I was recently watching Lord of the Rings and it dawned on me that those movies would be completely different if they weren’t directed by Peter Jackson Completely different Every shot every set the lighting the costumes how the characters and landscapes look and how the whole film feels and is portrayed It all would have looked and felt completely different based on the experience a different director was trying to create Thus there is no right or wrong way Rather it’s about doing things your way Until you experience this “moment” you’ll continue attempting the correct or best way to do things You’ll continue copying other people’s work But if you persist you’ll become disillusioned to those who were once your idols They are people just like you and me They’ve just made a decision to create in their own way The idea of imitation will become abhorrent freeing you to create as you see fit You’ll emerge with your own voice and original work You’ll be less troubled about how your work is received and more focused on creating something you believe in 31 Seeking Retirement “To retire is to die”  Pablo Casals The most powerful way to punch someone in the face is to aim a foot behind their face That way you have full momentum and power when you make contact If you aim only for the face itself by the time you reach it you’ll have already begun slowing down Thus your punch will not be as powerful as you intended it to be Retirement is the same way Most people planning for retirement begin slowing down in their 40’s and 50’s The sad part is as momentumbased beings when you begin to slow down you start a hardtoreverse decaying process Research has found that retirement often Increases the difficulty of mobility and daily activities Increases the likelihood of becoming ill And decreases mental health But retirement is a 20th century phenomena And actually the foundations under girding this outdated notion make little sense in modern and future society For instance due to advances in health care 65 is not considered old age anymore When the Social Security system was designed the planners chose age 65 because the average lifespan was age 63 at the time Thus the system was designed only for those who were really in need not to create a culture of people being supported by others’ labor Furthermore the perception that people over 65 can’t provide meaningful work no longer makes sense either Retirement became a thing when most work was manual labor  but today’s work is more knowledgebased And if there’s anything lacking in today’s society its wisdom which people in their later years have spent a lifetime refining Retirement should never be the goal We are fully capable to work  in some capacity  until our final breath My 92 year old grandfather Rex was a fighter pilot in WWII In the past five years he’s written three books He goes to bed every night at 8 PM and wakes up every morning at 430 AM He spends the first 25 hours of his day watching inspirational and instructional content on television He then eats breakfast at 7 AM and spends his day reading writing connecting and serving people and even doing physical labor around his son’s my dad’s house He even walks around his neighborhood proselyting his faith and asking random strangers how he can help them I have no intention of stopping or slowing down Contrary to popular belief humans are like wine and get better with age 32 Downplaying The Importance Of Yesterday “The best time to plant a tree was 20 years ago The second best time is now”  Chinese Proverb Our present circumstances are a reflection of our past decisions Although we have enormous power to change the trajectory of our lives here and nowwe are where we are because of our past While it’s popular to say the past doesn’t matter that simply is not true Today is tomorrow’s yesterday What we do today will either enhance or diminish our futurepresent moments But most people put things off until tomorrow We thoughtlessly go into debt forego exercise and education and justify negative relationships But at some point it all catches up Like an airplane offcourse the longer we wait to correct the longer and harder it is to get back oncourse Time is absolutely marvelous We get to anticipate the experiences we want to have  which is often more enjoyable than the experiences themselves We get to have the experiences we long for And then we get to remember and carry those experiences with us forever The past present and future are uniquely important and enjoyable 33 Believing You’re Way Behind In sports and all other forms of competition people perform best when the game is close Which is why big magic happens at the end of games like onsides kicks retrieved followed by 30 second touchdown drives But when the contest is decidedly in one opponent’s favor neither side acts with the same effort When you’re winning big it’s easy to get lax and overconfident When you’re losing big it’s easy to give up Sadly you probably perceive those at the top of your field “in a different league” altogether But when you do this you perform with less intensity than you would if you perceived the “game” to be closer When you elevate your thinking  and see yourself on the same level as those at “the top”  you quickly become disillusioned by the fallibility of those you once perceived as immortal They are just people Most importantly you will begin playing with an urgency that often surpasses even them The game is close The game is close 34 Listening To Music That Destroys Your Joy and Productivity “Without music life would be a mistake”  Friedrich Nietzsche You can also use music as a trigger for optimal performance For example Michael Phelps had a routine he did religiously before each swimming event involving music He’s not alone Many athletes use music before events to trigger relaxation from the pressure and even to psych themselves up When asked by Time Magazine about his use of music prior to races Phelps said it kept him focused and helped him “tune everything out and take one step at a time” When asked about the kind of music he listens to he answered “I listen to hip hop and rap” Interestingly research has found that high tempo music like hip hop can create strong arousal and performance readiness Other evidence finds the intensity of the emotional response can linger long after the music has stopped So while Phelps is in the water swimming he’s still hyped from his hip hop Lastly research has found that the types of music we listen to impact our level of spirituality This last point is particularly important to me Spirituality heavily influences everything I do from how I interact with my family to what and how I write to how I develop and pursue my goals 35 Call To Action If you want to focus on the right activities and get results 10x faster than most people check out my morning checklist Click here to get the checklist right now

In [103]:

    

flda_processedtexts = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_processedtexts','wb')
pickle.dump(all_texts_processed, flda_processedtexts)


    

In [104]:

    

# Make document-term matrix
dictionary = corpora.Dictionary(all_texts_processed)
# Convert to bag-of-words
corpus = [dictionary.doc2bow(text) for text in all_texts_processed]
print(corpus[0])


    

    




[(0, 1), (1, 2), (2, 40), (3, 11), (4, 14), (5, 19), (6, 1), (7, 5), (8, 9), (9, 8), (10, 1), (11, 13), (12, 29), (13, 50), (14, 12), (15, 1), (16, 1), (17, 2), (18, 2), (19, 5), (20, 11), (21, 5), (22, 3), (23, 10), (24, 1), (25, 2), (26, 18), (27, 1), (28, 1), (29, 4), (30, 1), (31, 4), (32, 2), (33, 4), (34, 1), (35, 1), (36, 1), (37, 1), (38, 7), (39, 23), (40, 13), (41, 5), (42, 3), (43, 30), (44, 4), (45, 4), (46, 4), (47, 8), (48, 1), (49, 44), (50, 2), (51, 1), (52, 5), (53, 1), (54, 9), (55, 1), (56, 4), (57, 1), (58, 3), (59, 16), (60, 4), (61, 1), (62, 1), (63, 2), (64, 9), (65, 1), (66, 17), (67, 23), (68, 10), (69, 3), (70, 1), (71, 1), (72, 7), (73, 1), (74, 1), (75, 2), (76, 5), (77, 1), (78, 8), (79, 1), (80, 1), (81, 7), (82, 2), (83, 2), (84, 2), (85, 12), (86, 2), (87, 2), (88, 1), (89, 1), (90, 2), (91, 24), (92, 11), (93, 3), (94, 6), (95, 1), (96, 4), (97, 1), (98, 12), (99, 12), (100, 4), (101, 1), (102, 8), (103, 3), (104, 2), (105, 3), (106, 2), (107, 18), (108, 7), (109, 2), (110, 1), (111, 1), (112, 5), (113, 9), (114, 9), (115, 8), (116, 7), (117, 2), (118, 7), (119, 1), (120, 5), (121, 2), (122, 3), (123, 1), (124, 3), (125, 1), (126, 1), (127, 2), (128, 1), (129, 3), (130, 1), (131, 1), (132, 1), (133, 7), (134, 9), (135, 5), (136, 3), (137, 2), (138, 1), (139, 6), (140, 1), (141, 1), (142, 2), (143, 3), (144, 1), (145, 20), (146, 27), (147, 12), (148, 1), (149, 1), (150, 9), (151, 1), (152, 2), (153, 2), (154, 1), (155, 2), (156, 1), (157, 11), (158, 1), (159, 1), (160, 9), (161, 4), (162, 6), (163, 2), (164, 8), (165, 5), (166, 1), (167, 3), (168, 1), (169, 2), (170, 6), (171, 4), (172, 2), (173, 2), (174, 1), (175, 38), (176, 3), (177, 4), (178, 6), (179, 1), (180, 2), (181, 6), (182, 10), (183, 7), (184, 10), (185, 1), (186, 1), (187, 9), (188, 3), (189, 2), (190, 4), (191, 3), (192, 7), (193, 5), (194, 2), (195, 2), (196, 2), (197, 1), (198, 6), (199, 2), (200, 30), (201, 2), (202, 24), (203, 6), (204, 4), (205, 3), (206, 28), (207, 13), (208, 1), (209, 11), (210, 2), (211, 1), (212, 2), (213, 9), (214, 3), (215, 2), (216, 1), (217, 2), (218, 23), (219, 3), (220, 2), (221, 1), (222, 8), (223, 5), (224, 2), (225, 1), (226, 8), (227, 1), (228, 1), (229, 1), (230, 2), (231, 2), (232, 1), (233, 1), (234, 1), (235, 2), (236, 2), (237, 1), (238, 2), (239, 5), (240, 2), (241, 1), (242, 7), (243, 2), (244, 1), (245, 1), (246, 9), (247, 2), (248, 1), (249, 2), (250, 2), (251, 5), (252, 1), (253, 3), (254, 1), (255, 2), (256, 7), (257, 10), (258, 4), (259, 1), (260, 4), (261, 4), (262, 2), (263, 5), (264, 1), (265, 2), (266, 22), (267, 1), (268, 1), (269, 1), (270, 1), (271, 10), (272, 4), (273, 2), (274, 8), (275, 13), (276, 4), (277, 1), (278, 1), (279, 6), (280, 2), (281, 2), (282, 4), (283, 1), (284, 2), (285, 2), (286, 2), (287, 4), (288, 1), (289, 1), (290, 2), (291, 5), (292, 1), (293, 1), (294, 1), (295, 3), (296, 4), (297, 1), (298, 1), (299, 2), (300, 1), (301, 1), (302, 2), (303, 1), (304, 1), (305, 1), (306, 4), (307, 1), (308, 2), (309, 5), (310, 10), (311, 1), (312, 1), (313, 1), (314, 3), (315, 6), (316, 2), (317, 1), (318, 4), (319, 1), (320, 2), (321, 20), (322, 1), (323, 3), (324, 1), (325, 4), (326, 4), (327, 1), (328, 1), (329, 1), (330, 4), (331, 1), (332, 1), (333, 2), (334, 1), (335, 1), (336, 4), (337, 1), (338, 7), (339, 6), (340, 5), (341, 1), (342, 5), (343, 4), (344, 8), (345, 1), (346, 1), (347, 2), (348, 2), (349, 1), (350, 2), (351, 1), (352, 8), (353, 1), (354, 1), (355, 1), (356, 1), (357, 4), (358, 1), (359, 4), (360, 5), (361, 11), (362, 2), (363, 1), (364, 9), (365, 1), (366, 1), (367, 15), (368, 4), (369, 3), (370, 2), (371, 3), (372, 2), (373, 13), (374, 13), (375, 2), (376, 1), (377, 1), (378, 1), (379, 2), (380, 10), (381, 1), (382, 1), (383, 3), (384, 6), (385, 1), (386, 1), (387, 1), (388, 1), (389, 1), (390, 4), (391, 1), (392, 1), (393, 1), (394, 1), (395, 1), (396, 4), (397, 7), (398, 1), (399, 1), (400, 7), (401, 4), (402, 1), (403, 1), (404, 1), (405, 12), (406, 2), (407, 2), (408, 1), (409, 1), (410, 7), (411, 4), (412, 1), (413, 1), (414, 15), (415, 1), (416, 1), (417, 2), (418, 1), (419, 3), (420, 2), (421, 2), (422, 2), (423, 3), (424, 6), (425, 2), (426, 2), (427, 1), (428, 1), (429, 20), (430, 1), (431, 1), (432, 1), (433, 2), (434, 2), (435, 1), (436, 1), (437, 3), (438, 1), (439, 5), (440, 1), (441, 3), (442, 1), (443, 1), (444, 1), (445, 1), (446, 1), (447, 1), (448, 1), (449, 5), (450, 1), (451, 1), (452, 3), (453, 1), (454, 4), (455, 1), (456, 1), (457, 1), (458, 1), (459, 2), (460, 1), (461, 2), (462, 3), (463, 1), (464, 1), (465, 3), (466, 2), (467, 1), (468, 2), (469, 3), (470, 1), (471, 1), (472, 2), (473, 1), (474, 1), (475, 1), (476, 1), (477, 1), (478, 3), (479, 2), (480, 5), (481, 10), (482, 1), (483, 1), (484, 2), (485, 1), (486, 3), (487, 1), (488, 1), (489, 5), (490, 3), (491, 3), (492, 1), (493, 1), (494, 1), (495, 2), (496, 2), (497, 1), (498, 1), (499, 2), (500, 5), (501, 7), (502, 7), (503, 1), (504, 1), (505, 3), (506, 1), (507, 1), (508, 4), (509, 6), (510, 4), (511, 11), (512, 1), (513, 6), (514, 1), (515, 9), (516, 6), (517, 2), (518, 5), (519, 1), (520, 9), (521, 4), (522, 4), (523, 5), (524, 8), (525, 1), (526, 1), (527, 4), (528, 6), (529, 6), (530, 1), (531, 4), (532, 2), (533, 2), (534, 1), (535, 7), (536, 2), (537, 4), (538, 1), (539, 1), (540, 2), (541, 3), (542, 2), (543, 2), (544, 2), (545, 4), (546, 4), (547, 4), (548, 2), (549, 2), (550, 2), (551, 1), (552, 2), (553, 1), (554, 2), (555, 4), (556, 1), (557, 3), (558, 2), (559, 2), (560, 1), (561, 11), (562, 1), (563, 1), (564, 1), (565, 2), (566, 1), (567, 1), (568, 2), (569, 1), (570, 1), (571, 1), (572, 2), (573, 1), (574, 2), (575, 1), (576, 4), (577, 4), (578, 1), (579, 1), (580, 12), (581, 1), (582, 1), (583, 1), (584, 1), (585, 1), (586, 4), (587, 1), (588, 9), (589, 10), (590, 2), (591, 1), (592, 1), (593, 1), (594, 1), (595, 2), (596, 5), (597, 1), (598, 1), (599, 2), (600, 1), (601, 1), (602, 4), (603, 2), (604, 1), (605, 1), (606, 1), (607, 2), (608, 6), (609, 1), (610, 1), (611, 1), (612, 4), (613, 1), (614, 2), (615, 1), (616, 1), (617, 1), (618, 1), (619, 16), (620, 8), (621, 1), (622, 2), (623, 6), (624, 2), (625, 3), (626, 1), (627, 2), (628, 2), (629, 1), (630, 5), (631, 1), (632, 1), (633, 1), (634, 1), (635, 1), (636, 2), (637, 2), (638, 2), (639, 2), (640, 1), (641, 9), (642, 3), (643, 1), (644, 1), (645, 1), (646, 5), (647, 1), (648, 1), (649, 1), (650, 1), (651, 3), (652, 1), (653, 4), (654, 1), (655, 4), (656, 2), (657, 11), (658, 8), (659, 1), (660, 1), (661, 2), (662, 1), (663, 1), (664, 1), (665, 2), (666, 2), (667, 1), (668, 2), (669, 1), (670, 1), (671, 1), (672, 9), (673, 5), (674, 1), (675, 2), (676, 1), (677, 2), (678, 1), (679, 1), (680, 1), (681, 1), (682, 4), (683, 8), (684, 1), (685, 1), (686, 1), (687, 1), (688, 1), (689, 1), (690, 4), (691, 4), (692, 3), (693, 1), (694, 1), (695, 2), (696, 1), (697, 1), (698, 2), (699, 1), (700, 2), (701, 1), (702, 1), (703, 1), (704, 1), (705, 1), (706, 4), (707, 1), (708, 2), (709, 3), (710, 1), (711, 1), (712, 2), (713, 2), (714, 1), (715, 1), (716, 1), (717, 1), (718, 1), (719, 1), (720, 4), (721, 1), (722, 1), (723, 1), (724, 4), (725, 1), (726, 1), (727, 1), (728, 3), (729, 1), (730, 1), (731, 3), (732, 1), (733, 3), (734, 1), (735, 4), (736, 2), (737, 2), (738, 1), (739, 3), (740, 4), (741, 1), (742, 2), (743, 1), (744, 2), (745, 1), (746, 2), (747, 1), (748, 1), (749, 2), (750, 2), (751, 1), (752, 1), (753, 1), (754, 1), (755, 1), (756, 1), (757, 1), (758, 3), (759, 3), (760, 1), (761, 1), (762, 2), (763, 1), (764, 2), (765, 1), (766, 1), (767, 3), (768, 1), (769, 1), (770, 2), (771, 1), (772, 5), (773, 3), (774, 1), (775, 2), (776, 1), (777, 3), (778, 3), (779, 10), (780, 4), (781, 1), (782, 3), (783, 8), (784, 1), (785, 1), (786, 1), (787, 1), (788, 1), (789, 1), (790, 2), (791, 1), (792, 1), (793, 1), (794, 1), (795, 1), (796, 1), (797, 1), (798, 1), (799, 2), (800, 1), (801, 2), (802, 1), (803, 7), (804, 2), (805, 1), (806, 4), (807, 9), (808, 5), (809, 2), (810, 3), (811, 1), (812, 2), (813, 1), (814, 8), (815, 1), (816, 2), (817, 2), (818, 1), (819, 4), (820, 3), (821, 2), (822, 3), (823, 1), (824, 1), (825, 10), (826, 1), (827, 1), (828, 1), (829, 1), (830, 2), (831, 1), (832, 1), (833, 1), (834, 2), (835, 3), (836, 1), (837, 1), (838, 1), (839, 5), (840, 7), (841, 3), (842, 1), (843, 1), (844, 1), (845, 1), (846, 1), (847, 2), (848, 2), (849, 2), (850, 1), (851, 1), (852, 1), (853, 1), (854, 4), (855, 1), (856, 1), (857, 1), (858, 1), (859, 1), (860, 3), (861, 3), (862, 1), (863, 1), (864, 1), (865, 5), (866, 1), (867, 1), (868, 3), (869, 1), (870, 1), (871, 1), (872, 1), (873, 2), (874, 1), (875, 1), (876, 1), (877, 1), (878, 1), (879, 1), (880, 1), (881, 1), (882, 2), (883, 4), (884, 2), (885, 4), (886, 1), (887, 1), (888, 2), (889, 1), (890, 1), (891, 2), (892, 1), (893, 1), (894, 1), (895, 1), (896, 1), (897, 1), (898, 1), (899, 1), (900, 1), (901, 1), (902, 1), (903, 1), (904, 3), (905, 2), (906, 1), (907, 1), (908, 3), (909, 2), (910, 2), (911, 1), (912, 1), (913, 2), (914, 1), (915, 4), (916, 1), (917, 1), (918, 1), (919, 3), (920, 1), (921, 6), (922, 1), (923, 1), (924, 1), (925, 2), (926, 2), (927, 1), (928, 3), (929, 1), (930, 2), (931, 1), (932, 1), (933, 1), (934, 1), (935, 4), (936, 2), (937, 1), (938, 2), (939, 1), (940, 1), (941, 2), (942, 1), (943, 2), (944, 2), (945, 1), (946, 4), (947, 1), (948, 3), (949, 1), (950, 2), (951, 1), (952, 1), (953, 1), (954, 1), (955, 2), (956, 2), (957, 1), (958, 2), (959, 3), (960, 1), (961, 1), (962, 1), (963, 2), (964, 4), (965, 2), (966, 3), (967, 1), (968, 1), (969, 1), (970, 3), (971, 1), (972, 1), (973, 3), (974, 1), (975, 1), (976, 1), (977, 1), (978, 2), (979, 1), (980, 3), (981, 2), (982, 1), (983, 1), (984, 1), (985, 3), (986, 3), (987, 1), (988, 1), (989, 1), (990, 1), (991, 2), (992, 1), (993, 1), (994, 1), (995, 1), (996, 1), (997, 1), (998, 1), (999, 1), (1000, 1), (1001, 1), (1002, 1), (1003, 3), (1004, 1), (1005, 6), (1006, 1), (1007, 6), (1008, 2), (1009, 1), (1010, 1), (1011, 2), (1012, 1), (1013, 3), (1014, 2), (1015, 2), (1016, 1), (1017, 1), (1018, 5), (1019, 1), (1020, 1), (1021, 1), (1022, 1), (1023, 1), (1024, 1), (1025, 1), (1026, 1), (1027, 2), (1028, 1), (1029, 1), (1030, 1), (1031, 1), (1032, 1), (1033, 2), (1034, 1), (1035, 1), (1036, 1), (1037, 1), (1038, 1), (1039, 1), (1040, 2), (1041, 1), (1042, 1), (1043, 2), (1044, 11), (1045, 5), (1046, 1), (1047, 1), (1048, 2), (1049, 2), (1050, 1), (1051, 1), (1052, 4), (1053, 3), (1054, 1), (1055, 6), (1056, 1), (1057, 10), (1058, 1), (1059, 1), (1060, 1), (1061, 1), (1062, 1), (1063, 1), (1064, 1), (1065, 2), (1066, 1), (1067, 1), (1068, 1), (1069, 1), (1070, 1), (1071, 1), (1072, 1), (1073, 1), (1074, 1), (1075, 1), (1076, 6), (1077, 1), (1078, 1), (1079, 1), (1080, 1), (1081, 1), (1082, 1), (1083, 3), (1084, 1), (1085, 1), (1086, 3), (1087, 1), (1088, 1), (1089, 1), (1090, 1), (1091, 1), (1092, 3), (1093, 1), (1094, 3), (1095, 1), (1096, 1), (1097, 1), (1098, 2), (1099, 5), (1100, 5), (1101, 5), (1102, 1), (1103, 1), (1104, 1), (1105, 1), (1106, 1), (1107, 3), (1108, 7), (1109, 1), (1110, 1), (1111, 1), (1112, 1), (1113, 1), (1114, 1), (1115, 2), (1116, 2), (1117, 2), (1118, 1), (1119, 1), (1120, 1), (1121, 1), (1122, 1), (1123, 1), (1124, 1), (1125, 3), (1126, 1), (1127, 1), (1128, 2), (1129, 1), (1130, 1), (1131, 1), (1132, 2), (1133, 2), (1134, 1), (1135, 1), (1136, 2), (1137, 2), (1138, 2), (1139, 2), (1140, 1), (1141, 1), (1142, 1), (1143, 1), (1144, 2), (1145, 2), (1146, 1), (1147, 1), (1148, 1), (1149, 1), (1150, 1), (1151, 4), (1152, 1), (1153, 1), (1154, 1), (1155, 1), (1156, 1), (1157, 1), (1158, 1), (1159, 1), (1160, 2), (1161, 1), (1162, 2), (1163, 1), (1164, 1), (1165, 1), (1166, 1), (1167, 1), (1168, 1), (1169, 2), (1170, 2), (1171, 4), (1172, 3), (1173, 1), (1174, 1), (1175, 1), (1176, 2), (1177, 2), (1178, 1), (1179, 1), (1180, 1), (1181, 3), (1182, 1), (1183, 1), (1184, 1), (1185, 1), (1186, 1), (1187, 1), (1188, 1), (1189, 2), (1190, 1), (1191, 1), (1192, 4), (1193, 1), (1194, 1), (1195, 1), (1196, 2), (1197, 1), (1198, 1), (1199, 1), (1200, 1), (1201, 1), (1202, 1), (1203, 1), (1204, 1), (1205, 1), (1206, 2), (1207, 2), (1208, 1), (1209, 1), (1210, 1), (1211, 1), (1212, 1), (1213, 2), (1214, 1), (1215, 3), (1216, 1), (1217, 1), (1218, 9), (1219, 1), (1220, 1), (1221, 1), (1222, 1), (1223, 1), (1224, 3), (1225, 1), (1226, 1), (1227, 1), (1228, 1), (1229, 1), (1230, 1), (1231, 1), (1232, 1), (1233, 1), (1234, 1), (1235, 1), (1236, 1), (1237, 1), (1238, 3), (1239, 1), (1240, 1), (1241, 1), (1242, 1), (1243, 1), (1244, 1), (1245, 1), (1246, 1), (1247, 1), (1248, 1), (1249, 1), (1250, 1), (1251, 1), (1252, 1), (1253, 1), (1254, 1), (1255, 2), (1256, 1), (1257, 2), (1258, 2), (1259, 1), (1260, 1), (1261, 1), (1262, 1), (1263, 1), (1264, 1), (1265, 1), (1266, 2), (1267, 2), (1268, 1), (1269, 1), (1270, 1), (1271, 2), (1272, 1), (1273, 1), (1274, 1), (1275, 1), (1276, 1), (1277, 1), (1278, 1), (1279, 1), (1280, 1), (1281, 1), (1282, 1), (1283, 1), (1284, 1), (1285, 1), (1286, 1), (1287, 1), (1288, 1), (1289, 1), (1290, 1), (1291, 1), (1292, 1), (1293, 2), (1294, 1), (1295, 1), (1296, 1), (1297, 1), (1298, 1), (1299, 1), (1300, 1), (1301, 1), (1302, 1), (1303, 8), (1304, 1), (1305, 1), (1306, 1), (1307, 4), (1308, 2), (1309, 3), (1310, 1), (1311, 2), (1312, 1), (1313, 1), (1314, 1), (1315, 4), (1316, 1), (1317, 3), (1318, 1), (1319, 1), (1320, 1), (1321, 1), (1322, 1), (1323, 3), (1324, 3), (1325, 2), (1326, 1), (1327, 1), (1328, 1), (1329, 1), (1330, 1), (1331, 1), (1332, 1), (1333, 1), (1334, 1), (1335, 1), (1336, 1), (1337, 1), (1338, 1), (1339, 3), (1340, 1), (1341, 1), (1342, 1), (1343, 1), (1344, 1), (1345, 1), (1346, 1), (1347, 1), (1348, 3), (1349, 1), (1350, 1), (1351, 1), (1352, 1), (1353, 1), (1354, 1), (1355, 1), (1356, 1), (1357, 1), (1358, 1), (1359, 1), (1360, 1), (1361, 1), (1362, 1), (1363, 1), (1364, 1), (1365, 1), (1366, 1), (1367, 5), (1368, 1), (1369, 1), (1370, 1), (1371, 1), (1372, 1), (1373, 1), (1374, 1), (1375, 1), (1376, 1), (1377, 1), (1378, 1), (1379, 1), (1380, 1), (1381, 1), (1382, 1), (1383, 1), (1384, 1), (1385, 2), (1386, 1), (1387, 1), (1388, 2), (1389, 1), (1390, 1), (1391, 1), (1392, 1), (1393, 2), (1394, 1), (1395, 1), (1396, 1), (1397, 1), (1398, 1), (1399, 1), (1400, 1), (1401, 1), (1402, 1), (1403, 1), (1404, 1), (1405, 1), (1406, 1), (1407, 1), (1408, 1), (1409, 1), (1410, 1), (1411, 1), (1412, 1), (1413, 1), (1414, 1), (1415, 1), (1416, 1), (1417, 1), (1418, 1), (1419, 1), (1420, 1), (1421, 1), (1422, 1), (1423, 1), (1424, 1), (1425, 1), (1426, 1), (1427, 1), (1428, 2), (1429, 1), (1430, 1), (1431, 1), (1432, 1), (1433, 1), (1434, 1), (1435, 1), (1436, 1), (1437, 1), (1438, 1), (1439, 1), (1440, 1), (1441, 1), (1442, 1), (1443, 1), (1444, 1), (1445, 2), (1446, 1), (1447, 1), (1448, 3), (1449, 1), (1450, 1), (1451, 2), (1452, 1), (1453, 1), (1454, 1), (1455, 1), (1456, 1), (1457, 1), (1458, 1), (1459, 1), (1460, 1), (1461, 1), (1462, 3), (1463, 3), (1464, 1), (1465, 1), (1466, 1), (1467, 1), (1468, 1), (1469, 1), (1470, 1), (1471, 1), (1472, 1), (1473, 1), (1474, 1), (1475, 1), (1476, 1), (1477, 1), (1478, 1)]

In [105]:

    

# save all_texts_processed
flda_dictionary = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_dictionary','wb')
flda_corpus = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_corpus','wb')
pickle.dump(dictionary, flda_dictionary)
pickle.dump(corpus, flda_corpus)


    

In [106]:

    

# Run LDA
# choose 10 topics, 1 pass for initial try and time it

tic = timeit.default_timer()
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=10, id2word = dictionary, passes=1)
toc = timeit.default_timer()
print(str(toc - tic) + ' seconds elapsed')
# current: with new dictionary/corpus excluding more stopwords


    

    




61.63819993999641 seconds elapsed

In [81]:

    

# # Run LDA
# # choose 20 topics, 1 pass to test how much time increases

# tic = timeit.default_timer()
# ldamodel2 = gensim.models.ldamodel.LdaModel(corpus, num_topics=20, id2word = dictionary, passes=1)
# toc = timeit.default_timer()
# print(str(toc - tic) + ' seconds elapsed')
# # current: with old dictionary/corpus without excluding nltk stopwords


    

    




52.94888425300451 seconds elapsed

In [82]:

    

# # Run LDA
# # choose 100 topics, 1 pass to test how much time increases

# tic = timeit.default_timer()
# ldamodel3 = gensim.models.ldamodel.LdaModel(corpus, num_topics=100, id2word = dictionary, passes=1)
# toc = timeit.default_timer()
# print(str(toc - tic) + ' seconds elapsed')
# # current: with old dictionary/corpus without excluding nltk stopwords


    

    




124.6579989890015 seconds elapsed

In [83]:

    

# flda_10topic1pass = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_10topic1pass','wb')
# pickle.dump(ldamodel,  flda_10topic1pass)
# # current: with new dictionary/corpus excluding more stopwords

# flda_20topic1pass = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_20topic1pass','wb')
# pickle.dump(ldamodel2, flda_20topic1pass)
# flda_100topic1pass = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_100topic1pass','wb')
# pickle.dump(ldamodel3, flda_100topic1pass)
# # current: with old dictionary/corpus without excluding nltk stopwords


    

In [109]:

    

print(ldamodel.print_topics(num_topics=3, num_words=3))


    

    




[(7, '0.006*"like" + 0.006*"peopl" + 0.006*"time"'), (3, '0.006*"one" + 0.006*"like" + 0.006*"work"'), (0, '0.007*"design" + 0.006*"use" + 0.006*"one"')]

In [85]:

    

# # Run LDA
# # choose 100 topics, 20 passes

# tic = timeit.default_timer()
# ldamodel4 = gensim.models.ldamodel.LdaModel(corpus, num_topics=100, id2word = dictionary, passes=20)
# toc = timeit.default_timer()
# print(str(toc - tic) + ' seconds elapsed')
# # current: with old dictionary/corpus without excluding nltk stopwords


    

    




1422.3470460919998 seconds elapsed

In [86]:

    

flda_100topic20pass = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_100topic20pass','wb')
pickle.dump(ldamodel4, flda_100topic20pass)


    

In [90]:

    

# Compare topic outputs of LDA models 1-4
print(ldamodel.print_topics( num_topics=10, num_words=10))
# print(ldamodel2.print_topics(num_topics=10, num_words=10))
# print(ldamodel3.print_topics(num_topics=10, num_words=10))
# print(ldamodel4.print_topics(num_topics=10, num_words=10))


    

    




[(0, '0.009*"design" + 0.008*"it’" + 0.007*"use" + 0.007*"can" + 0.006*"like" + 0.006*"get" + 0.006*"will" + 0.005*"make" + 0.005*"work" + 0.005*"time"'), (1, '0.006*"one" + 0.006*"work" + 0.006*"can" + 0.005*"will" + 0.005*"peopl" + 0.004*"it’" + 0.004*"make" + 0.004*"use" + 0.004*"just" + 0.004*"de"'), (2, '0.016*"que" + 0.014*"de" + 0.011*"fuck" + 0.011*"o" + 0.011*"e" + 0.006*"não" + 0.006*"é" + 0.005*"um" + 0.004*"para" + 0.004*"can"'), (3, '0.007*"product" + 0.007*"peopl" + 0.007*"get" + 0.006*"design" + 0.006*"will" + 0.005*"time" + 0.005*"one" + 0.005*"just" + 0.005*"can" + 0.005*"like"'), (4, '0.007*"peopl" + 0.006*"one" + 0.006*"can" + 0.005*"will" + 0.005*"make" + 0.005*"work" + 0.005*"time" + 0.004*"new" + 0.004*"thing" + 0.004*"like"'), (5, '0.008*"will" + 0.007*"thing" + 0.007*"one" + 0.007*"can" + 0.006*"it’" + 0.006*"peopl" + 0.005*"get" + 0.005*"time" + 0.005*"don’t" + 0.005*"make"'), (6, '0.009*"make" + 0.009*"work" + 0.008*"can" + 0.007*"one" + 0.007*"time" + 0.007*"want" + 0.006*"peopl" + 0.006*"it’" + 0.006*"will" + 0.006*"get"'), (7, '0.010*"time" + 0.007*"can" + 0.006*"one" + 0.005*"get" + 0.005*"like" + 0.005*"it’" + 0.004*"peopl" + 0.004*"make" + 0.004*"thing" + 0.004*"day"'), (8, '0.010*"like" + 0.007*"it’" + 0.006*"just" + 0.006*"time" + 0.005*"can" + 0.005*"will" + 0.005*"peopl" + 0.005*"get" + 0.005*"thing" + 0.005*"one"'), (9, '0.008*"peopl" + 0.007*"can" + 0.006*"like" + 0.006*"one" + 0.005*"time" + 0.005*"work" + 0.005*"will" + 0.005*"just" + 0.004*"get" + 0.004*"don’t"')]

In [100]:

    

# from nltk.corpus import stopwords
# stopw_en = stopwords.words('english')
# print(stopw_en)
# print(stop_en)
# print(len(stopw_en))
# print(len(stop_en))
# all_stopw = set(stopw_en) | set(stop_en)
# print(len(all_stopw))


    

    




['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won', 'wouldn']
['a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', "aren't", 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', "can't", 'cannot', 'could', "couldn't", 'did', "didn't", 'do', 'does', "doesn't", 'doing', "don't", 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', "hadn't", 'has', "hasn't", 'have', "haven't", 'having', 'he', "he'd", "he'll", "he's", 'her', 'here', "here's", 'hers', 'herself', 'him', 'himself', 'his', 'how', "how's", 'i', "i'd", "i'll", "i'm", "i've", 'if', 'in', 'into', 'is', "isn't", 'it', "it's", 'its', 'itself', "let's", 'me', 'more', 'most', "mustn't", 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same', "shan't", 'she', "she'd", "she'll", "she's", 'should', "shouldn't", 'so', 'some', 'such', 'than', 'that', "that's", 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', "there's", 'these', 'they', "they'd", "they'll", "they're", "they've", 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', "wasn't", 'we', "we'd", "we'll", "we're", "we've", 'were', "weren't", 'what', "what's", 'when', "when's", 'where', "where's", 'which', 'while', 'who', "who's", 'whom', 'why', "why's", 'with', "won't", 'would', "wouldn't", 'you', "you'd", "you'll", "you're", "you've", 'your', 'yours', 'yourself', 'yourselves']
153
174
207

In [110]:

    

# Run LDA
# choose 10 topics, 20 passes after removing more stopwords

tic = timeit.default_timer()
ldamodel5 = gensim.models.ldamodel.LdaModel(corpus, num_topics=10, id2word = dictionary, passes=20)
toc = timeit.default_timer()
print(str(toc - tic) + ' seconds elapsed')


    

    




765.3832683630026 seconds elapsed

In [114]:

    

print(ldamodel.print_topics( num_topics=10, num_words=5))


    

    




[(0, '0.007*"design" + 0.006*"use" + 0.006*"one" + 0.006*"like" + 0.005*"app"'), (1, '0.006*"it’" + 0.006*"work" + 0.006*"peopl" + 0.006*"time" + 0.005*"make"'), (2, '0.008*"like" + 0.007*"time" + 0.007*"it’" + 0.006*"use" + 0.006*"get"'), (3, '0.006*"one" + 0.006*"like" + 0.006*"work" + 0.005*"use" + 0.005*"make"'), (4, '0.010*"design" + 0.007*"like" + 0.005*"it’" + 0.005*"time" + 0.005*"peopl"'), (5, '0.009*"peopl" + 0.008*"time" + 0.007*"it’" + 0.007*"like" + 0.006*"get"'), (6, '0.007*"peopl" + 0.005*"like" + 0.005*"it’" + 0.005*"one" + 0.005*"make"'), (7, '0.006*"like" + 0.006*"peopl" + 0.006*"time" + 0.006*"one" + 0.005*"make"'), (8, '0.005*"one" + 0.004*"it’" + 0.004*"time" + 0.004*"que" + 0.003*"peopl"'), (9, '0.007*"one" + 0.007*"want" + 0.006*"work" + 0.006*"get" + 0.006*"make"')]

In [151]:

    

flatten = lambda all_texts_processed: [item for sublist in all_texts_processed for item in sublist]
# all_texts_combined = ' '.join(all_texts_processed)
all_texts_flattened = flatten(all_texts_processed)
all_texts_flattened[1000000]
print(len(all_texts_flattened))

flatten_uniq = set(all_texts_flattened)
print(len(flatten_uniq))

print(len(all_texts_processed))

commonwords = []
wordlist = []
i = 0
tic = timeit.default_timer()
for word in flatten_uniq:
    n = 0
    for text in all_texts_processed:
        if word in text:
#             print('yes!')
            n += 1
    frac = float(n / len(all_texts_processed))
    if frac >= 0.6:
        commonwords.append(word)
    elif frac < 0.6:
        wordlist.append(word)
    i += 1
#     print(word)
#     print(frac)
#     print(n)
#     if i == 20:
#         break
#     print(word)
#     print(frac)
#     if i >= 50:
#         break

toc = timeit.default_timer()
print(str(toc - tic) + ' seconds elapsed')


    

    




2527135
96378
3201

In [227]:

    

print(len(wordlist))
print(len(commonwords))
print(commonwords)
commonwords_2 = [i.strip('”“’‘') for i in commonwords]
print(commonwords_2)


    

    




96342
36
['like', 'tri', 'want', 'need', 'new', 'start', 'say', 'first', 'use', 'time', 'everi', 'see', 'one', 'come', 'look', 'day', 'get', 'it’', 'someth', 'much', 'good', 'also', 'don’t', 'go', 'think', 'even', 'work', 'make', 'take', 'know', 'way', 'right', 'mani', 'year', 'peopl', 'thing']
['like', 'tri', 'want', 'need', 'new', 'start', 'say', 'first', 'use', 'time', 'everi', 'see', 'one', 'come', 'look', 'day', 'get', 'it', 'someth', 'much', 'good', 'also', 'don’t', 'go', 'think', 'even', 'work', 'make', 'take', 'know', 'way', 'right', 'mani', 'year', 'peopl', 'thing']

In [164]:

    

# all_stopw2 = set(all_stopw_stem) | set(commonwords)
# print(len(all_stopw2))

# all_stopw_stem = [p_stemmer.stem(i) for i in all_stopw]
# print(all_stopw)
# print(all_stopw_stem)

# all_stopw2 = set(all_stopw_stem) | set(commonwords)
# print(len(all_stopw2))


    

    




235
{'your', 'didn', "i'd", "she'd", "it's", 'from', "wasn't", 'an', 'as', 'being', 'has', "they're", 'will', "where's", 'of', 'through', 'having', 'further', 'up', 'them', 'they', 'there', 'their', 'no', 'll', 'y', 'she', 'into', "i'm", "hadn't", "he'll", 'you', 'his', 'those', 'with', 't', 'me', "who's", 'should', 'when', 'shouldn', 'he', 'over', "there's", 'so', 'do', 'once', 'before', "weren't", 'what', 'ours', "here's", 'too', 'very', "we'll", "shan't", 's', "don't", 'why', 'yourselves', 'by', 'own', "what's", 'would', 'this', 'been', 'each', 'than', "when's", 'who', "didn't", 'the', 'just', 'mustn', 'my', 'hers', "haven't", 'are', 'himself', 'does', 'doing', 'after', 'weren', "we're", "mustn't", 'we', "he'd", 'if', "wouldn't", 'isn', "aren't", 'aren', 'for', 'between', 'on', "they'll", 'theirs', 'ourselves', "why's", 'have', "she's", "won't", 'themselves', "i've", 'here', 'any', 'such', 'haven', 'only', 'him', 'd', 've', 'its', "you'll", 'couldn', "let's", 'off', 'had', 'is', 'under', "she'll", "we'd", 'yours', 'wouldn', 'myself', "hasn't", 'now', "i'll", 'be', 'it', 'ain', 'doesn', 'to', 'ought', 'where', 'wasn', 'about', "they've", 'whom', 'at', 'hadn', 'that', 'against', 'most', 'was', 'in', "shouldn't", 'above', 'until', 'while', 'm', 'these', 'won', 'itself', 'how', 'mightn', "couldn't", 'but', "he's", 'down', 'same', "can't", 'then', 'few', 'during', "you've", 'hasn', 'below', 'her', "doesn't", "you'd", 'o', 'which', "we've", "isn't", 'or', 'a', "how's", "that's", 'again', 're', 'yourself', 'all', 'both', 'don', "you're", 'and', 'could', 'other', 'because', 'ma', 'shan', 'did', 'some', "they'd", 'needn', 'i', 'can', 'out', 'our', 'more', 'were', 'am', 'cannot', 'nor', 'herself', 'not'}
['your', 'didn', "i'd", "she'd", "it'", 'from', "wasn't", 'an', 'as', 'be', 'ha', "they'r", 'will', "where'", 'of', 'through', 'have', 'further', 'up', 'them', 'they', 'there', 'their', 'no', 'll', 'y', 'she', 'into', "i'm", "hadn't", "he'll", 'you', 'hi', 'those', 'with', 't', 'me', "who'", 'should', 'when', 'shouldn', 'he', 'over', "there'", 'so', 'do', 'onc', 'befor', "weren't", 'what', 'our', "here'", 'too', 'veri', "we'll", "shan't", 's', "don't", 'whi', 'yourselv', 'by', 'own', "what'", 'would', 'thi', 'been', 'each', 'than', "when'", 'who', "didn't", 'the', 'just', 'mustn', 'my', 'her', "haven't", 'are', 'himself', 'doe', 'do', 'after', 'weren', "we'r", "mustn't", 'we', "he'd", 'if', "wouldn't", 'isn', "aren't", 'aren', 'for', 'between', 'on', "they'll", 'their', 'ourselv', "why'", 'have', "she'", "won't", 'themselv', "i'v", 'here', 'ani', 'such', 'haven', 'onli', 'him', 'd', 've', 'it', "you'll", 'couldn', "let'", 'off', 'had', 'is', 'under', "she'll", "we'd", 'your', 'wouldn', 'myself', "hasn't", 'now', "i'll", 'be', 'it', 'ain', 'doesn', 'to', 'ought', 'where', 'wasn', 'about', "they'v", 'whom', 'at', 'hadn', 'that', 'against', 'most', 'wa', 'in', "shouldn't", 'abov', 'until', 'while', 'm', 'these', 'won', 'itself', 'how', 'mightn', "couldn't", 'but', "he'", 'down', 'same', "can't", 'then', 'few', 'dure', "you'v", 'hasn', 'below', 'her', "doesn't", "you'd", 'o', 'which', "we'v", "isn't", 'or', 'a', "how'", "that'", 'again', 're', 'yourself', 'all', 'both', 'don', "you'r", 'and', 'could', 'other', 'becaus', 'ma', 'shan', 'did', 'some', "they'd", 'needn', 'i', 'can', 'out', 'our', 'more', 'were', 'am', 'cannot', 'nor', 'herself', 'not']
235

In [245]:

    

fwordlist = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/wordlist','wb')
fcommonwords = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/commonwords','wb')
fcommonwords2 = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/commonwords2','wb')
pickle.dump(wordlist, fwordlist)
pickle.dump(commonwords, fcommonwords)
pickle.dump(commonwords_2, fcommonwords2)

tmp = pickle.load(open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/wordlist','rb'))
print(commonwords_2)


    

    




['like', 'tri', 'want', 'need', 'new', 'start', 'say', 'first', 'use', 'time', 'everi', 'see', 'one', 'come', 'look', 'day', 'get', 'it', 'someth', 'much', 'good', 'also', 'don’t', 'go', 'think', 'even', 'work', 'make', 'take', 'know', 'way', 'right', 'mani', 'year', 'peopl', 'thing']

In [160]:

    

# print(tmp)


    

In [230]:

    

# REDO LDA with common words removed

all_texts_processed_new = []


tic = timeit.default_timer()

n = 0
for text in set_tr['text']:
    txt = ' '.join(text)
    # remove punctuation
    translator = str.maketrans('', '', string.punctuation)
    txt2 = re.sub(u'\u2014','',txt) # remove em dashes
    txt3 = re.sub(r'\d+', '', txt2) # remove digits
    txt4 = txt3.translate(translator) # remove punctuation
    # split text into words
    tokens = word_tokenizer.tokenize(txt4.lower())
    # strip single and double quotes from ends of words
    tokens_strip = [i.strip('”“’‘') for i in tokens]
    # keep only english words
    tokens_en = [i for i in tokens_strip if i in en_words]
    # remove nltk/stop_word stop words
    nostop_tokens = [i for i in tokens_en if not i in all_stopw]
    # strip single and double quotes from ends of words
    nostop_strip = [i.strip('”“’‘') for i in nostop_tokens]
    # stem words
    stemmed = [p_stemmer.stem(i) for i in nostop_strip]
    # strip single and double quotes from ends of words
    stemmed_strip = [i.strip('”“’‘') for i in stemmed]
    # stem words
    stemmed2 = [p_stemmer.stem(i) for i in stemmed_strip]
    # strip single and double quotes from ends of words
    stemmed2_strip = [i.strip('”“’‘') for i in stemmed2]
    # remove common words post-stemming
    stemmed_nocommon = [i for i in stemmed2_strip if not i in commonwords_2]
    # append to processed texts
    all_texts_processed_new.append( stemmed_nocommon )
    if n == 0:
#         print(txt)
#         print(tokens)
#         print(nostop_tokens)
        print(stemmed_nocommon)
    n += 1
#     if n == 5:
#         break

toc = timeit.default_timer()
print(str(toc - tic) + ' seconds elapsed')



# #### TRY only using nouns? -- nah, since some of main words in e.g. topic 0 are 'design', 'use'


    

    




['give', 'success', 'wast', 'five', 'five', 'spend', 'excu', 'rest', 'laze', 'five', 'minut', 'wast', 'daili', 'per', 'x', 'sadli', 'guess', 'wast', 'far', 'told', 'th', 'grade', 'teacher', 'read', 'break', 'break', 'minut', 'two', 'lot', 'read', 'done', 'finish', 'earli', 'spare', 'moment', 'pick', 'book', 'read', 'spend', 'period', 'five', 'minut', 'factor', 'achiev', 'littl', 'bit', 'justifi', 'wast', 'dollar', 'recent', 'line', 'point', 'item', 'thought', 'interest', 'honestli', 'rememb', 'stuck', 'said', 'dollar', 'lot', 'money', 'short', 'money', 'actual', 'famili', 'trip', 'world', 'whole', 'understand', 'valu', 'dollar', 'appreci', 'valu', 'thoughtlessli', 'spend', 'dollar', 'may', 'seem', 'big', 'deal', 'actual', 'frivol', 'spend', 'long', 'enough', 'million', 'lack', 'care', 'true', 'art', 'valu', 'addit', 'percent', 'rich', 'percent', 'hourli', 'respon', 'minut', 'dollar', 'consequ', 'great', 'major', 'extrem', 'frugal', 'least', 'highli', 'mind', 'money', 'believ', 'success', 'happi', 'happi', 'adapt', 'psychologist', 'studi', 'relationship', 'money', 'happi', 'two', 'buy', 'us', 'happi', 'succeed', 'excit', 'us', 'adapt', 'actual', 'anticip', 'idea', 'desir', 'outcom', 'gener', 'satisfi', 'outcom', 'whether', 'wealth', 'health', 'excel', 'adapt', 'excit', 'often', 'seek', 'end', 'disappoint', 'love', 'watch', 'phenomena', 'foster', 'feel', 'certain', 'toy', 'univ', 'explod', 'whole', 'world', 'around', 'yet', 'buy', 'toy', 'long', 'joy', 'el', 'appreci', 'current', 'life', 'better', 'believ', 'challeng', 'deceiv', 'believ', 'us', 'happier', 'deceiv', 'believ', 'harder', 'longer', 'procrastin', 'avoid', 'pain', 'head', 'becom', 'howev', 'action', 'discomfort', 'far', 'less', 'sever', 'extrem', 'difficult', 'adapt', 'recent', 'sat', 'plane', 'ladi', 'ye', 'read', 'correctli', 'eight', 'husband', 'felt', 'inspir', 'foster', 'four', 'later', 'adopt', 'later', 'took', 'anoth', 'five', 'foster', 'adopt', 'cour', 'initi', 'shock', 'system', 'impact', 'entir', 'famili', 'handl', 'believ', 'handl', 'problem', 'dread', 'fear', 'back', 'big', 'find', 'matter', 'big', 'small', 'challeng', 'adapt', 'consciou', 'adapt', 'enorm', 'stress', 'evolv', 'happi', 'happi', 'happi', 'believ', 'must', 'money', 'love', 'travel', 'world', 'write', 'book', 'busi', 'romant', 'relationship', 'ultim', 'allow', 'happi', 'peac', 'content', 'love', 'paradox', 'paradigm', 'must', 'actual', 'rever', 'experi', 'happi', 'success', 'anyth', 'el', 'desir', 'whatev', 'happi', 'compass', 'peac', 'wise', 'love', 'space', 'almost', 'immedi', 'bring', 'attract', 'happi', 'must', 'happi', 'wealthi', 'must', 'live', 'wealthi', 'translat', 'around', 'undervalu', 'interview', 'annual', 'geniu', 'network', 'event', 'variou', 'ever', 'ever', 'feel', 'taken', 'cour', 'anyon', 'lie', 'morn', 'declar', 'focu', 'fact', 'enough', 'enough', 'worri', 'mad', 'problem', 'later', 'interview', 'read', 'hour', 'got', 'impress', 'care', 'money', 'travel', 'world', 'without', 'spend', 'money', 'talk', 'balanc', 'abil', 'let', 'money', 'total', 'lot', 'nice', 'addict', 'wealth', 'fight', 'club', 'end', 'becom', 'surrog', 'health', 'happi', 'connect', 'becom', 'disea', 'state', 'nice', 'fear', 'taken', 'away', 'money', 'realli', 'valuabl', 'tool', 'appreci', 'alreadi', 'life', 'feel', 'compen', 'miss', 'life', 'alway', 'left', 'matter', 'acquir', 'achiev', 'current', 'posit', 'easi', 'talk', 'hard', 'easi', 'talk', 'unfair', 'life', 'got', 'stick', 'kind', 'talk', 'realli', 'help', 'anyon', 'judg', 'situat', 'wor', 'someon', 'ignorantli', 'incorrectli', 'got', 'easi', 'success', 'easi', 'deal', 'gone', 'paradigm', 'formal', 'becom', 'known', 'victim', 'mental', 'gener', 'entitl', 'world', 'noth', 'life', 'meant', 'fair', 'howev', 'world', 'given', 'everyth', 'truth', 'advantag', 'world', 'succeed', 'believ', 'feel', 'enorm', 'weight', 'respon', 'world', 'put', 'perfect', 'posit', 'succeed', 'everyth', 'univ', 'brought', 'point', 'shine', 'chang', 'world', 'oyster', 'natur', 'state', 'thrive', 'show', 'life', 'human', 'holist', 'chang', 'part', 'system', 'simultan', 'chang', 'whole', 'chang', 'part', 'without', 'fundament', 'everyth', 'pebbl', 'thought', 'matter', 'inconsequenti', 'endless', 'consequ', 'idea', 'butterfli', 'effect', 'came', 'metaphor', 'exampl', 'hurrican', 'minor', 'distant', 'butterfli', 'sever', 'littl', 'becom', 'big', 'area', 'life', 'align', 'area', 'life', 'compart', 'system', 'although', 'easi', 'push', 'certain', 'health', 'side', 'unwittingli', 'infect', 'whole', 'life', 'eventu', 'alway', 'procrastin', 'avoid', 'catch', 'detriment', 'conver', 'improv', 'area', 'life', 'posit', 'wrote', 'man', 'man', 'pure', 'longer', 'impur', 'food', 'holist', 'human', 'whole', 'everyth', 'effect', 'whole', 'world', 'better', 'wor', 'invit', 'ask', 'part', 'cure', 'part', 'disea', 'escap', 'competit', 'peter', 'competit', 'extrem', 'costli', 'maximum', 'product', 'reach', 'wealth', 'creation', 'becom', 'battl', 'slightli', 'outdo', 'race', 'bottom', 'involv', 'instead', 'compet', 'better', 'complet', 'novel', 'focu', 'tightli', 'defin', 'nich', 'establish', 'author', 'set', 'rather', 'reactiv', 'competit', 'thu', 'monopol', 'space', 'creat', 'valu', 'spend', 'realli', 'societi', 'import', 'spend', 'whole', 'life', 'keep', 'probabl', 'shallow', 'life', 'defin', 'success', 'base', 'detach', 'noi', 'deci', 'opportun', 'cost', 'choo', 'simultan', 'choo', 'sever', 'someon', 'lie', 'almost', 'certainli', 'preach', 'sell', 'truth', 'realiti', 'simpli', 'exampl', 'fact', 'famili', 'center', 'life', 'spend', 'wife', 'three', 'foster', 'top', 'prioriti', 'result', 'spend', 'made', 'choic', 'point', 'choo', 'us', 'attempt', 'everyth', 'end', 'noth', 'intern', 'conflict', 'hell', 'although', 'tradit', 'view', 'creativ', 'follow', 'creativ', 'usual', 'insid', 'proverbi', 'box', 'outsid', 'flex', 'creativ', 'constrain', 'rather', 'broaden', 'henc', 'clearli', 'defin', 'constrain', 'better', 'sever', 'everyth', 'outsid', 'forget', 'came', 'easi', 'achiev', 'level', 'success', 'believ', 'sole', 'respon', 'success', 'easi', 'forget', 'came', 'easi', 'forget', 'made', 'easi', 'superior', 'burn', 'human', 'connect', 'left', 'intern', 'cave', 'isol', 'lose', 'mind', 'ident', 'becom', 'person', 'never', 'intend', 'humil', 'gratitud', 'recognit', 'success', 'proper', 'perspect', 'without', 'help', 'countless', 'extrem', 'lucki', 'abl', 'contribut', 'seek', 'permiss', 'highli', 'success', 'investor', 'throughout', 'career', 'ask', 'fact', 'actual', 'talk', 'succeed', 'regardless', 'told', 'chase', 'whatev', 'never', 'truli', 'decid', 'end', 'next', 'strike', 'quick', 'gold', 'repetit', 'stop', 'dig', 'gold', 'spot', 'barren', 'ever', 'give', 'permiss', 'live', 'holiday', 'said', 'obstacl', 'stop', 'rather', 'extern', 'chang', 'mental', 'refram', 'chang', 'chang', 'dyer', 'enough', 'whatev', 'decid', 'deci', 'forget', 'everyon', 'el', 'determin', 'money', 'earn', 'success', 'realli', 'success', 'tell', 'wish', 'piano', 'someon', 'said', 'practic', 'sinc', 'stop', 'life', 'matter', 'prioriti', 'deci', 'money', 'economi', 'money', 'choo', 'question', 'money', 'realli', 'instead', 'social', 'media', 'spend', 'hour', 'two', 'build', 'valu', 'book', 'grow', 'rich', 'napoleon', 'hill', 'write', 'piec', 'paper', 'amount', 'money', 'put', 'singl', 'act', 'challeng', 'act', 'creat', 'futur', 'exampl', 'despit', 'grow', 'poor', 'famili', 'live', 'van', 'lawn', 'futur', 'night', 'late', 'drive', 'atop', 'larg', 'hill', 'visual', 'broke', 'struggl', 'young', 'comic', 'night', 'futur', 'wrote', 'check', 'million', 'put', 'notat', 'line', 'act', 'check', 'thanksgiv', 'stuck', 'wallet', 'gave', 'five', 'thanksgiv', 'got', 'million', 'dumb', 'vision', 'futur', 'creat', 'highest', 'vision', 'possibl', 'life', 'becom', 'believ', 'matter', 'futur', 'certain', 'plant', 'must', 'harvest', 'plea', 'plant', 'intent', 'mental', 'creation', 'alway', 'physic', 'creation', 'blueprint', 'design', 'head', 'becom', 'life', 'build', 'let', 'societi', 'tell', 'hou', 'artist', 'creator', 'life', 'exactli', 'whether', 'consid', 'mansion', 'home', 'heart', 'parabl', 'wealthi', 'parent', 'give', 'unwi', 'child', 'inherit', 'undoubtedli', 'parent', 'said', 'child', 'desir', 'give', 'wealth', 'posit', 'stand', 'among', 'men', 'easili', 'give', 'must', 'obtain', 'qualifi', 'inherit', 'learn', 'learn', 'live', 'live', 'give', 'acquir', 'wisdom', 'statur', 'follow', 'exampl', 'becom', 'enough', 'must', 'success', 'fundament', 'chang', 'live', 'higher', 'level', 'must', 'shift', 'reflect', 'becom', 'experienc', 'chang', 'success', 'natur', 'point', 'becom', 'millionair', 'give', 'money', 'away', 'import', 'million', 'import', 'person', 'becom', 'process', 'becom', 'millionair', 'believ', 'money', 'evil', 'better', 'wor', 'holist', 'human', 'bodi', 'best', 'spiritu', 'physic', 'side', 'perform', 'best', 'brain', 'board', 'help', 'mind', 'believ', 'nobl', 'fuel', 'rabbi', 'genuin', 'believ', 'money', 'immor', 'money', 'evil', 'believ', 'seek', 'forc', 'buy', 'money', 'evil', 'neutral', 'symbol', 'valu', 'sell', 'pair', 'someon', 'buy', 'perceiv', 'worth', 'buy', 'forc', 'buy', 'choic', 'thu', 'valu', 'exchang', 'base', 'pure', 'percept', 'valu', 'subject', 'person', 'bought', 'probabl', 'sell', 'worth', 'still', 'might', 'sell', 'correct', 'price', 'correct', 'price', 'worth', 'custom', 'price', 'high', 'custom', 'exchang', 'money', 'extrem', 'lucki', 'live', 'societi', 'system', 'money', 'us', 'borrow', 'lend', 'leverag', 'abil', 'scale', 'enorm', 'limit', 'trade', 'system', 'earn', 'money', 'complet', 'moral', 'pursuit', 'done', 'honesti', 'integr', 'fact', 'feel', 'moral', 'probabl', 'chang', 'job', 'believ', 'valu', 'provid', 'disserv', 'offer', 'track', 'coloss', 'valu', 'reflect', 'us', 'alway', 'choic', 'whether', 'perceiv', 'valu', 'offer', 'distract', 'overestim', 'unimport', 'practic', 'everyth', 'almost', 'everyth', 'distract', 'realli', 'realli', 'put', 'certain', 'beyond', 'particular', 'valu', 'give', 'everyth', 'life', 'person', 'never', 'exchang', 'priceless', 'price', 'keep', 'proper', 'perspect', 'remov', 'everyth', 'nonessenti', 'life', 'live', 'simpli', 'laser', 'lead', 'nowher', 'realiz', 'focu', 'live', 'distract', 'era', 'human', 'histori', 'sword', 'money', 'neutral', 'bad', 'base', 'sadli', 'us', 'simpli', 'respon', 'enough', 'wast', 'stare', 'idli', 'screen', 'particularli', 'prone', 'nowaday', 'everyon', 'suscept', 'attent', 'shrunk', 'almost', 'noth', 'atrophi', 'realli', 'bad', 'often', 'requir', 'extrem', 'rever', 'grow', 'bodi', 'scientif', 'evid', 'suggest', 'constant', 'turn', 'us', 'scatter', 'superfici', 'biggest', 'constant', 'distract', 'shallow', 'rather', 'deep', 'shallow', 'shallow', 'live', 'philosoph', 'may', 'put', 'best', 'ago', 'everywh', 'nowher', 'book', 'deep', 'success', 'distract', 'world', 'cal', 'deep', 'shallow', 'deep', 'creat', 'valu', 'thought', 'energi', 'concentr', 'shallow', 'littl', 'administr', 'logist', 'stuff', 'expen', 'move', 'toward', 'shallow', 'abil', 'perform', 'deep', 'becom', 'increasingli', 'rare', 'exactli', 'becom', 'increasingli', 'valuabl', 'economi', 'consequ', 'cultiv', 'skill', 'core', 'life', 'thrive', 'cal', 'logic', 'aim', 'beyond', 'capabl', 'develop', 'complet', 'disregard', 'end', 'unabl', 'best', 'compani', 'sphere', 'aim', 'unabl', 'cover', 'magazin', 'busi', 'vision', 'realiti', 'noth', 'imposs', 'complet', 'logic', 'requir', 'imagin', 'certainli', 'requir', 'faith', 'luck', 'magic', 'person', 'believ', 'sad', 'skeptic', 'secular', 'becom', 'find', 'great', 'pleasur', 'faith', 'spiritu', 'context', 'life', 'mean', 'person', 'growth', 'faith', 'pursu', 'call', 'absurd', 'walk', 'water', 'transcend', 'death', 'truli', 'god', 'possibl', 'absolut', 'noth', 'fear', 'seek', 'prai', 'rather', 'critic', 'cultur', 'becom', 'fragil', 'must', 'combin', 'honest', 'feedback', 'feedback', 'best', 'disprov', 'call', 'confirm', 'bia', 'tendenc', 'search', 'interpret', 'favor', 'recal', 'inform', 'give', 'excess', 'less', 'consid', 'altern', 'easi', 'prai', 'ask', 'famili', 'tell', 'exactli', 'hear', 'instead', 'seek', 'prai', 'improv', 'seek', 'critic', 'better', 'merit', 'someon', 'enough', 'give', 'unsolicit', 'critiqu', 'noteworthi', 'robin', 'author', 'monk', 'sold', 'said', 'confirm', 'great', 'realli', 'show', 'rather', 'reflect', 'becom', 'reflect', 'rather', 'give', 'scarciti', 'perspect', 'help', 'longer', 'advantag', 'perspect', 'world', 'giant', 'pie', 'piec', 'pie', 'pie', 'order', 'win', 'must', 'lose', 'abund', 'perspect', 'pie', 'infinit', 'number', 'thu', 'help', 'actual', 'system', 'whole', 'better', 'trust', 'confid', 'friend', 'realli', 'innov', 'stuff', 'real', 'estat', 'compani', 'el', 'kill', 'told', 'consid', 'keep', 'secret', 'knew', 'mean', 'less', 'opposit', 'told', 'everyon', 'compani', 'give', 'away', 'never', 'seen', 'compani', 'strategi', 'longer', 'anoth', 'leadership', 'innov', 'trust', 'actual', 'reli', 'best', 'sever', 'ye', 'compani', 'hord', 'inform', 'creat', 'design', 'scratch', 'itch', 'actual', 'experi', 'difficulti', 'creat', 'solut', 'approach', 'creat', 'music', 'listen', 'draw', 'paint', 'write', 'wish', 'written', 'person', 'approach', 'write', 'read', 'foremost', 'reson', 'enjoy', 'product', 'expect', 'next', 'opportun', 'perfect', 'client', 'perfect', 'opportun', 'perfect', 'almost', 'never', 'happen', 'instead', 'wish', 'differ', 'cultiv', 'front', 'rather', 'wait', 'next', 'opportun', 'opportun', 'said', 'anoth', 'grass', 'greener', 'water', 'leav', 'believ', 'better', 'end', 'previou', 'relationship', 'end', 'problem', 'problem', 'find', 'creat', 'hard', 'said', 'wish', 'easier', 'wish', 'better', 'wish', 'less', 'wish', 'wish', 'less', 'challeng', 'wish', 'wisdom', 'wait', 'purpo', 'carv', 'progress', 'improv', 'without', 'question', 'lost', 'vacuum', 'increasingli', 'crowd', 'old', 'wither', 'wonder', 'went', 'said', 'pile', 'enough', 'find', 'left', 'noth', 'lot', 'empti', 'long', 'activ', 'write', 'wait', 'moment', 'enough', 'money', 'whatev', 'el', 'thought', 'wait', 'somehow', 'qualifi', 'permiss', 'never', 'degr', 'live', 'qualifi', 'show', 'permiss', 'life', 'short', 'wait', 'tomorrow', 'today', 'futur', 'self', 'either', 'thank', 'shame', 'defend', 'earli', 'age', 'toni', 'graduat', 'toni', 'old', 'six', 'million', 'compani', 'amaz', 'toni', 'less', 'job', 'oracl', 'k', 'per', 'thought', 'discuss', 'partner', 'offer', 'believ', 'continu', 'build', 'bigger', 'true', 'love', 'build', 'true', 'pro', 'money', 'true', 'pro', 'love', 'five', 'later', 'million', 'jerri', 'yang', 'cofound', 'yahoo', 'toni', 'away', 'thought', 'glad', 'sell', 'five', 'ago', 'howev', 'cool', 'consid', 'propo', 'thought', 'money', 'never', 'anoth', 'life', 'reflect', 'devi', 'small', 'list', 'home', 'theater', 'abil', 'weekend', 'whenev', 'comput', 'anoth', 'compani', 'idea', 'build', 'grow', 'passion', 'motiv', 'stuff', 'alreadi', 'afford', 'comput', 'alreadi', 'weekend', 'whenev', 'old', 'determin', 'wait', 'sell', 'build', 'grow', 'anoth', 'compani', 'toni', 'million', 'dollar', 'offer', 'explod', 'busi', 'boom', 'yet', 'longer', 'cultur', 'polit', 'subtli', 'process', 'rapid', 'growth', 'longer', 'group', 'close', 'build', 'hire', 'bunch', 'hurri', 'vision', 'care', 'build', 'rather', 'rich', 'quick', 'pure', 'decid', 'sell', 'compani', 'million', 'old', 'similar', 'concept', 'conver', 'ago', 'jeff', 'author', 'art', 'advic', 'book', 'write', 'said', 'wait', 'jump', 'gun', 'made', 'mistak', 'wait', 'two', 'x', 'bigger', 'advanc', 'chang', 'trajectori', 'whole', 'career', 'k', 'writer', 'around', 'book', 'advanc', 'writer', 'around', 'book', 'advanc', 'wait', 'two', 'chang', 'trajectori', 'career', 'life', 'procrastin', 'strategi', 'chang', 'whole', 'life', 'noth', 'certain', 'sign', 'insan', 'expect', 'differ', 'convent', 'break', 'convent', 'evolv', 'quantiti', 'failur', 'grit', 'fail', 'never', 'invent', 'light', 'bulb', 'seth', 'said', 'fail', 'win', 'failur', 'failur', 'feedback', 'failur', 'move', 'forward', 'consciou', 'effort', 'toward', 'never', 'done', 'incr', 'person', 'unlik', 'anyth', 'set', 'win', 'may', 'spend', 'whole', 'climb', 'ladder', 'success', 'find', 'reach', 'top', 'ladder', 'lean', 'wrong', 'wall', 'wrong', 'game', 'lose', 'game', 'onset', 'hell', 'ruin', 'life', 'without', 'import', 'game', 'game', 'set', 'set', 'game', 'play', 'better', 'win', 'play', 'end', 'backward', 'rather', 'plausibl', 'sen', 'covey', 'put', 'begin', 'end', 'clearli', 'mind', 'dictat', 'daili', 'weekli', 'monthli', 'yearli', 'facilit', 'wrote', 'million', 'check', 'set', 'earn', 'game', 'small', 'win', 'matter', 'small', 'along', 'leverag', 'posit', 'high', 'school', 'diploma', 'leverag', 'posit', 'guy', 'guy', 'guy', 'leverag', 'posit', 'articl', 'featur', 'unknown', 'leverag', 'posit', 'leverag', 'posit', 'sadli', 'stop', 'side', 'fenc', 'fail', 'realiz', 'brilliant', 'current', 'avail', 'bad', 'stewardship', 'alreadi', 'inform', 'alreadi', 'capit', 'alreadi', 'connect', 'instead', 'util', 'alreadi', 'help', 'actual', 'continu', 'hurt', 'learn', 'earn', 'easi', 'real', 'success', 'ownership', 'life', 'el', 'success', 'health', 'current', 'posit', 'ripe', 'abund', 'opportun', 'leverag', 'gain', 'anoth', 'inch', 'posit', 'leverag', 'worth', 'wish', 'wish', 'better', 'soon', 'enough', 'find', 'incr', 'success', 'base', 'choic', 'success', 'base', 'motiv', 'worth', 'fight', 'base', 'believ', 'might', 'call', 'fantasi', 'base', 'posit', 'momentum', 'step', 'cool', 'part', 'poetri', 'import', 'import', 'actual', 'said', 'similar', 'event', 'hear', 'speech', 'usual', 'speaker', 'hear', 'alreadi', 'matter', 'type', 'better', 'receiv', 'audienc', 'often', 'decid', 'holiday', 'author', 'obstacl', 'moment', 'skill', 'creativ', 'experienc', 'moment', 'mechan', 'craft', 'moment', 'magic', 'idea', 'creat', 'creat', 'moment', 'realiz', 'everyth', 'done', 'person', 'intent', 'particular', 'experi', 'recent', 'watch', 'lord', 'complet', 'differ', 'direct', 'peter', 'complet', 'differ', 'shot', 'set', 'light', 'whole', 'film', 'felt', 'complet', 'differ', 'base', 'experi', 'differ', 'director', 'creat', 'thu', 'wrong', 'rather', 'experi', 'moment', 'continu', 'correct', 'best', 'continu', 'persist', 'becom', 'made', 'deci', 'creat', 'idea', 'imit', 'becom', 'abhorr', 'free', 'creat', 'fit', 'emerg', 'voic', 'origin', 'less', 'receiv', 'believ', 'seek', 'retir', 'retir', 'die', 'pablo', 'power', 'punch', 'someon', 'face', 'aim', 'foot', 'behind', 'face', 'full', 'momentum', 'power', 'contact', 'aim', 'face', 'reach', 'alreadi', 'begun', 'thu', 'punch', 'power', 'intend', 'retir', 'retir', 'begin', 'sad', 'part', 'begin', 'slow', 'process', 'research', 'found', 'retir', 'often', 'difficulti', 'mobil', 'daili', 'likelihood', 'becom', 'ill', 'mental', 'health', 'retir', 'th', 'centuri', 'phenomena', 'actual', 'gird', 'outdat', 'notion', 'littl', 'sen', 'modern', 'futur', 'societi', 'instanc', 'due', 'health', 'care', 'consid', 'old', 'age', 'social', 'secur', 'system', 'design', 'chose', 'age', 'averag', 'age', 'thu', 'system', 'design', 'realli', 'creat', 'cultur', 'labor', 'furthermor', 'percept', 'provid', 'mean', 'longer', 'sen', 'either', 'retir', 'manual', 'labor', 'anyth', 'societi', 'wisdom', 'later', 'spent', 'lifetim', 'refin', 'retir', 'never', 'goal', 'fulli', 'capabl', 'capac', 'final', 'breath', 'old', 'grandfath', 'rex', 'fighter', 'pilot', 'past', 'five', 'written', 'three', 'goe', 'bed', 'night', 'wake', 'morn', 'watch', 'inspir', 'instruct', 'content', 'televi', 'eat', 'breakfast', 'read', 'write', 'serv', 'physic', 'labor', 'around', 'hou', 'around', 'neighborhood', 'faith', 'random', 'help', 'intent', 'stop', 'contrari', 'popular', 'belief', 'wine', 'better', 'age', 'import', 'yesterday', 'best', 'plant', 'tree', 'ago', 'second', 'best', 'proverb', 'present', 'reflect', 'past', 'although', 'enorm', 'power', 'chang', 'trajectori', 'past', 'popular', 'past', 'matter', 'simpli', 'true', 'today', 'yesterday', 'today', 'either', 'enhanc', 'diminish', 'put', 'tomorrow', 'thoughtlessli', 'debt', 'forego', 'exerci', 'educ', 'justifi', 'neg', 'point', 'airplan', 'longer', 'wait', 'correct', 'longer', 'harder', 'back', 'absolut', 'marvel', 'anticip', 'often', 'enjoy', 'long', 'rememb', 'carri', 'us', 'forev', 'past', 'present', 'futur', 'uniqu', 'import', 'enjoy', 'believ', 'behind', 'sport', 'competit', 'perform', 'best', 'game', 'close', 'big', 'magic', 'end', 'second', 'touchdown', 'contest', 'decidedli', 'favor', 'neither', 'side', 'effort', 'win', 'big', 'easi', 'lax', 'overconfid', 'lose', 'big', 'easi', 'give', 'sadli', 'probabl', 'perceiv', 'top', 'field', 'differ', 'leagu', 'altogeth', 'perform', 'less', 'inten', 'game', 'closer', 'elev', 'level', 'top', 'quickli', 'becom', 'fallibl', 'immort', 'importantli', 'begin', 'urgenc', 'often', 'game', 'close', 'game', 'close', 'listen', 'music', 'joy', 'product', 'without', 'music', 'life', 'mistak', 'music', 'trigger', 'optim', 'perform', 'exampl', 'routin', 'religi', 'swim', 'event', 'music', 'alon', 'music', 'trigger', 'relax', 'pressur', 'psych', 'magazin', 'music', 'prior', 'said', 'kept', 'tune', 'everyth', 'step', 'kind', 'music', 'listen', 'hip', 'hop', 'rap', 'interestingli', 'research', 'found', 'high', 'tempo', 'music', 'hip', 'hop', 'creat', 'strong', 'arou', 'perform', 'readi', 'evid', 'inten', 'emot', 'respon', 'linger', 'long', 'music', 'stop', 'water', 'swim', 'still', 'hip', 'hop', 'lastli', 'research', 'found', 'music', 'listen', 'impact', 'level', 'spiritu', 'last', 'point', 'particularli', 'import', 'spiritu', 'heavili', 'everyth', 'interact', 'famili', 'write', 'develop', 'pursu', 'call', 'action', 'focu', 'x', 'faster', 'check', 'morn', 'click']
126.21320975600975 seconds elapsed

In [231]:

    

# flda_processedtexts_new = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_processedtexts_new','wb')
# pickle.dump(all_texts_processed_new, flda_processedtexts_new)
# # above: without filtering for english words

flda_processedtexts_new2 = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_processedtexts_new2','wb')
pickle.dump(all_texts_processed_new, flda_processedtexts_new2)
# above: with filtering for english words


    

In [232]:

    

# dictionary_new = corpora.Dictionary(all_texts_processed_new)
# # Convert to bag-of-words
# corpus_new = [dictionary_new.doc2bow(text) for text in all_texts_processed_new]
# print(corpus_new[0])
# # above: without filtering for english words


# Make document-term matrix
dictionary_new2 = corpora.Dictionary(all_texts_processed_new)
# Convert to bag-of-words
corpus_new2 = [dictionary_new2.doc2bow(text) for text in all_texts_processed_new]
print(corpus_new2[0])
# above: with filtering for english words


    

    




[(0, 13), (1, 19), (2, 5), (3, 9), (4, 11), (5, 1), (6, 1), (7, 1), (8, 4), (9, 3), (10, 2), (11, 3), (12, 4), (13, 1), (14, 2), (15, 4), (16, 2), (17, 1), (18, 1), (19, 7), (20, 3), (21, 5), (22, 4), (23, 4), (24, 1), (25, 2), (26, 1), (27, 7), (28, 1), (29, 7), (30, 1), (31, 1), (32, 3), (33, 4), (34, 1), (35, 2), (36, 6), (37, 3), (38, 2), (39, 6), (40, 1), (41, 7), (42, 1), (43, 1), (44, 2), (45, 2), (46, 12), (47, 26), (48, 2), (49, 11), (50, 6), (51, 1), (52, 12), (53, 12), (54, 1), (55, 13), (56, 3), (57, 2), (58, 3), (59, 1), (60, 7), (61, 2), (62, 1), (63, 5), (64, 10), (65, 9), (66, 1), (67, 4), (68, 5), (69, 2), (70, 1), (71, 2), (72, 3), (73, 1), (74, 5), (75, 3), (76, 3), (77, 1), (78, 6), (79, 1), (80, 1), (81, 2), (82, 4), (83, 20), (84, 13), (85, 6), (86, 1), (87, 1), (88, 3), (89, 6), (90, 9), (91, 4), (92, 2), (93, 2), (94, 5), (95, 3), (96, 2), (97, 2), (98, 1), (99, 3), (100, 4), (101, 6), (102, 1), (103, 6), (104, 7), (105, 10), (106, 1), (107, 6), (108, 3), (109, 2), (110, 4), (111, 5), (112, 5), (113, 2), (114, 2), (115, 2), (116, 6), (117, 2), (118, 2), (119, 6), (120, 4), (121, 28), (122, 13), (123, 4), (124, 2), (125, 1), (126, 2), (127, 9), (128, 3), (129, 2), (130, 1), (131, 2), (132, 23), (133, 3), (134, 2), (135, 1), (136, 8), (137, 5), (138, 1), (139, 1), (140, 1), (141, 1), (142, 2), (143, 1), (144, 1), (145, 1), (146, 2), (147, 2), (148, 1), (149, 5), (150, 2), (151, 1), (152, 7), (153, 2), (154, 1), (155, 1), (156, 8), (157, 2), (158, 1), (159, 2), (160, 4), (161, 1), (162, 3), (163, 2), (164, 6), (165, 8), (166, 4), (167, 2), (168, 4), (169, 1), (170, 2), (171, 10), (172, 2), (173, 8), (174, 3), (175, 1), (176, 1), (177, 1), (178, 2), (179, 2), (180, 1), (181, 2), (182, 2), (183, 5), (184, 3), (185, 4), (186, 1), (187, 1), (188, 2), (189, 5), (190, 1), (191, 1), (192, 1), (193, 3), (194, 11), (195, 1), (196, 1), (197, 2), (198, 1), (199, 1), (200, 1), (201, 3), (202, 1), (203, 3), (204, 2), (205, 2), (206, 2), (207, 3), (208, 1), (209, 4), (210, 4), (211, 1), (212, 1), (213, 2), (214, 4), (215, 1), (216, 6), (217, 5), (218, 1), (219, 4), (220, 2), (221, 1), (222, 2), (223, 1), (224, 2), (225, 1), (226, 1), (227, 3), (228, 2), (229, 2), (230, 4), (231, 11), (232, 2), (233, 1), (234, 9), (235, 1), (236, 1), (237, 4), (238, 3), (239, 2), (240, 12), (241, 13), (242, 2), (243, 1), (244, 1), (245, 2), (246, 7), (247, 1), (248, 1), (249, 3), (250, 6), (251, 1), (252, 1), (253, 1), (254, 1), (255, 1), (256, 1), (257, 4), (258, 1), (259, 7), (260, 1), (261, 1), (262, 1), (263, 13), (264, 2), (265, 2), (266, 1), (267, 7), (268, 4), (269, 1), (270, 1), (271, 14), (272, 1), (273, 2), (274, 2), (275, 3), (276, 5), (277, 3), (278, 6), (279, 2), (280, 2), (281, 1), (282, 1), (283, 1), (284, 2), (285, 2), (286, 3), (287, 1), (288, 5), (289, 1), (290, 1), (291, 1), (292, 3), (293, 1), (294, 1), (295, 3), (296, 1), (297, 4), (298, 1), (299, 1), (300, 1), (301, 1), (302, 1), (303, 2), (304, 3), (305, 3), (306, 2), (307, 3), (308, 1), (309, 1), (310, 1), (311, 3), (312, 1), (313, 1), (314, 4), (315, 2), (316, 1), (317, 1), (318, 3), (319, 3), (320, 3), (321, 1), (322, 1), (323, 1), (324, 1), (325, 1), (326, 1), (327, 5), (328, 1), (329, 7), (330, 1), (331, 1), (332, 3), (333, 1), (334, 1), (335, 4), (336, 6), (337, 11), (338, 1), (339, 6), (340, 1), (341, 15), (342, 5), (343, 9), (344, 3), (345, 4), (346, 7), (347, 8), (348, 1), (349, 1), (350, 4), (351, 6), (352, 1), (353, 4), (354, 2), (355, 1), (356, 7), (357, 2), (358, 4), (359, 1), (360, 1), (361, 2), (362, 4), (363, 2), (364, 1), (365, 4), (366, 4), (367, 1), (368, 2), (369, 1), (370, 2), (371, 1), (372, 1), (373, 4), (374, 2), (375, 2), (376, 1), (377, 1), (378, 1), (379, 2), (380, 1), (381, 2), (382, 1), (383, 1), (384, 2), (385, 4), (386, 4), (387, 1), (388, 1), (389, 1), (390, 1), (391, 1), (392, 4), (393, 1), (394, 9), (395, 10), (396, 2), (397, 1), (398, 1), (399, 1), (400, 2), (401, 5), (402, 1), (403, 2), (404, 1), (405, 1), (406, 4), (407, 1), (408, 1), (409, 3), (410, 1), (411, 1), (412, 2), (413, 4), (414, 3), (415, 1), (416, 2), (417, 2), (418, 1), (419, 6), (420, 1), (421, 1), (422, 1), (423, 2), (424, 2), (425, 1), (426, 1), (427, 1), (428, 3), (429, 2), (430, 4), (431, 3), (432, 11), (433, 1), (434, 2), (435, 1), (436, 2), (437, 2), (438, 2), (439, 1), (440, 8), (441, 5), (442, 1), (443, 2), (444, 2), (445, 1), (446, 1), (447, 1), (448, 3), (449, 8), (450, 1), (451, 1), (452, 1), (453, 1), (454, 3), (455, 1), (456, 1), (457, 1), (458, 1), (459, 1), (460, 1), (461, 1), (462, 1), (463, 1), (464, 4), (465, 1), (466, 2), (467, 1), (468, 1), (469, 1), (470, 4), (471, 1), (472, 2), (473, 3), (474, 1), (475, 1), (476, 3), (477, 3), (478, 1), (479, 4), (480, 2), (481, 1), (482, 1), (483, 3), (484, 5), (485, 1), (486, 2), (487, 1), (488, 1), (489, 2), (490, 1), (491, 2), (492, 2), (493, 1), (494, 1), (495, 1), (496, 1), (497, 1), (498, 1), (499, 3), (500, 3), (501, 3), (502, 1), (503, 1), (504, 1), (505, 6), (506, 2), (507, 2), (508, 3), (509, 3), (510, 2), (511, 10), (512, 4), (513, 6), (514, 1), (515, 1), (516, 1), (517, 1), (518, 1), (519, 1), (520, 1), (521, 2), (522, 2), (523, 1), (524, 1), (525, 3), (526, 5), (527, 3), (528, 2), (529, 1), (530, 1), (531, 2), (532, 2), (533, 4), (534, 4), (535, 2), (536, 3), (537, 1), (538, 1), (539, 8), (540, 1), (541, 1), (542, 1), (543, 2), (544, 1), (545, 1), (546, 1), (547, 2), (548, 2), (549, 1), (550, 4), (551, 1), (552, 1), (553, 5), (554, 1), (555, 1), (556, 2), (557, 2), (558, 1), (559, 1), (560, 1), (561, 1), (562, 1), (563, 2), (564, 3), (565, 1), (566, 1), (567, 1), (568, 3), (569, 1), (570, 1), (571, 1), (572, 2), (573, 1), (574, 1), (575, 1), (576, 1), (577, 1), (578, 1), (579, 3), (580, 1), (581, 2), (582, 1), (583, 2), (584, 1), (585, 1), (586, 1), (587, 1), (588, 5), (589, 1), (590, 4), (591, 1), (592, 2), (593, 1), (594, 1), (595, 1), (596, 1), (597, 3), (598, 1), (599, 2), (600, 2), (601, 3), (602, 1), (603, 2), (604, 2), (605, 1), (606, 2), (607, 4), (608, 2), (609, 2), (610, 1), (611, 2), (612, 9), (613, 1), (614, 1), (615, 2), (616, 1), (617, 1), (618, 4), (619, 1), (620, 3), (621, 2), (622, 1), (623, 1), (624, 1), (625, 1), (626, 3), (627, 2), (628, 2), (629, 4), (630, 1), (631, 1), (632, 3), (633, 1), (634, 1), (635, 1), (636, 2), (637, 3), (638, 2), (639, 3), (640, 1), (641, 1), (642, 1), (643, 3), (644, 1), (645, 2), (646, 1), (647, 1), (648, 1), (649, 1), (650, 2), (651, 1), (652, 3), (653, 1), (654, 1), (655, 3), (656, 1), (657, 1), (658, 1), (659, 1), (660, 1), (661, 1), (662, 1), (663, 1), (664, 1), (665, 4), (666, 1), (667, 6), (668, 2), (669, 1), (670, 1), (671, 2), (672, 1), (673, 1), (674, 2), (675, 2), (676, 1), (677, 1), (678, 1), (679, 1), (680, 1), (681, 1), (682, 2), (683, 1), (684, 1), (685, 1), (686, 1), (687, 1), (688, 2), (689, 1), (690, 2), (691, 11), (692, 4), (693, 1), (694, 1), (695, 2), (696, 1), (697, 1), (698, 3), (699, 2), (700, 1), (701, 1), (702, 7), (703, 1), (704, 10), (705, 1), (706, 1), (707, 1), (708, 1), (709, 1), (710, 1), (711, 1), (712, 1), (713, 1), (714, 1), (715, 1), (716, 6), (717, 1), (718, 1), (719, 1), (720, 1), (721, 1), (722, 1), (723, 1), (724, 1), (725, 2), (726, 3), (727, 1), (728, 3), (729, 1), (730, 1), (731, 1), (732, 5), (733, 5), (734, 1), (735, 1), (736, 1), (737, 1), (738, 2), (739, 1), (740, 1), (741, 4), (742, 2), (743, 2), (744, 1), (745, 1), (746, 1), (747, 1), (748, 1), (749, 2), (750, 1), (751, 1), (752, 1), (753, 1), (754, 2), (755, 2), (756, 2), (757, 1), (758, 2), (759, 1), (760, 1), (761, 1), (762, 1), (763, 1), (764, 1), (765, 4), (766, 1), (767, 1), (768, 1), (769, 2), (770, 1), (771, 1), (772, 1), (773, 1), (774, 1), (775, 2), (776, 3), (777, 3), (778, 2), (779, 1), (780, 1), (781, 2), (782, 1), (783, 4), (784, 1), (785, 3), (786, 1), (787, 2), (788, 1), (789, 1), (790, 1), (791, 2), (792, 2), (793, 1), (794, 1), (795, 2), (796, 1), (797, 3), (798, 1), (799, 10), (800, 1), (801, 1), (802, 2), (803, 1), (804, 1), (805, 3), (806, 1), (807, 4), (808, 1), (809, 1), (810, 1), (811, 1), (812, 1), (813, 1), (814, 1), (815, 1), (816, 3), (817, 1), (818, 1), (819, 1), (820, 1), (821, 1), (822, 1), (823, 1), (824, 1), (825, 1), (826, 1), (827, 1), (828, 1), (829, 1), (830, 1), (831, 1), (832, 1), (833, 2), (834, 2), (835, 1), (836, 1), (837, 1), (838, 1), (839, 2), (840, 1), (841, 1), (842, 1), (843, 1), (844, 1), (845, 1), (846, 1), (847, 1), (848, 1), (849, 1), (850, 1), (851, 1), (852, 1), (853, 1), (854, 1), (855, 1), (856, 8), (857, 1), (858, 1), (859, 4), (860, 2), (861, 3), (862, 1), (863, 2), (864, 1), (865, 1), (866, 1), (867, 1), (868, 3), (869, 3), (870, 1), (871, 1), (872, 1), (873, 1), (874, 1), (875, 1), (876, 1), (877, 1), (878, 1), (879, 1), (880, 1), (881, 1), (882, 1), (883, 3), (884, 1), (885, 1), (886, 1), (887, 1), (888, 1), (889, 1), (890, 1), (891, 1), (892, 1), (893, 1), (894, 1), (895, 1), (896, 1), (897, 1), (898, 5), (899, 1), (900, 1), (901, 1), (902, 1), (903, 1), (904, 1), (905, 1), (906, 1), (907, 1), (908, 1), (909, 1), (910, 2), (911, 1), (912, 1), (913, 2), (914, 1), (915, 2), (916, 1), (917, 2), (918, 1), (919, 1), (920, 1), (921, 1), (922, 1), (923, 1), (924, 1), (925, 1), (926, 1), (927, 1), (928, 1), (929, 1), (930, 1), (931, 1), (932, 1), (933, 1), (934, 1), (935, 1), (936, 1), (937, 1), (938, 1), (939, 1), (940, 2), (941, 1), (942, 1), (943, 1), (944, 1), (945, 1), (946, 1), (947, 1), (948, 2), (949, 1), (950, 1), (951, 1), (952, 2), (953, 1), (954, 1), (955, 1), (956, 1), (957, 1), (958, 1), (959, 1), (960, 3), (961, 3), (962, 1), (963, 1), (964, 1), (965, 1), (966, 1), (967, 1), (968, 1), (969, 1), (970, 1), (971, 1), (972, 1), (973, 1), (974, 1), (975, 1)]

In [233]:

    

# flda_dictionary_new = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_dictionary_new','wb')
# flda_corpus_new = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_corpus_new','wb')
# pickle.dump(dictionary_new, flda_dictionary_new)
# pickle.dump(corpus_new, flda_corpus_new)
# # above: without filtering for english words

flda_dictionary_new2 = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_dictionary_new2','wb')
flda_corpus_new2 = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/lda_corpus_new2','wb')
pickle.dump(dictionary_new2, flda_dictionary_new2)
pickle.dump(corpus_new2, flda_corpus_new2)
# above: with filtering for english words


    

In [234]:

    

# # choose 10 topics, 20 passes
# tic = timeit.default_timer()
# ldamodel_new = gensim.models.ldamodel.LdaModel(corpus_new, num_topics=10, id2word = dictionary_new, passes=20)
# toc = timeit.default_timer()
# print(str(toc - tic) + ' seconds elapsed')
# # current: with new dictionary/corpus excluding more stopwords and common words
# # # above: without filtering for english words


# choose 10 topics, 20 passes
tic = timeit.default_timer()
ldamodel_new = gensim.models.ldamodel.LdaModel(corpus_new2, num_topics=10, id2word = dictionary_new2, passes=20)
toc = timeit.default_timer()
print(str(toc - tic) + ' seconds elapsed')
# current: with new dictionary/corpus excluding more stopwords and common words
# above: with filtering for english words


    

    




535.3515410920081 seconds elapsed

In [235]:

    

# Save LDA model
# # flda_10topic20pass_new = open('/Users/clarencecheng/Dropbox/~Insight/skimr/lda_10topic20pass_new','wb')
# # pickle.dump(ldamodel_new, flda_10topic20pass_new)

# flda_10topic20pass_new2 = open('/Users/clarencecheng/Dropbox/~Insight/skimr/lda_10topic20pass_new2','wb')
# pickle.dump(ldamodel_new, flda_10topic20pass_new2)
# # # above: without filtering for english words


flda_10topic20pass_new2b = open('/Users/clarencecheng/Dropbox/~Insight/skimr/lda_10topic20pass_new2b','wb')
pickle.dump(ldamodel_new, flda_10topic20pass_new2b)
# above: with filtering for english words


    

In [240]:

    

print(ldamodel_new.print_topics( num_topics=10, num_words=5))


    

    




[(0, '0.011*"compani" + 0.010*"product" + 0.008*"busi" + 0.007*"build" + 0.007*"team"'), (1, '0.030*"design" + 0.010*"user" + 0.007*"code" + 0.006*"web" + 0.006*"color"'), (2, '0.013*"write" + 0.013*"read" + 0.008*"learn" + 0.008*"love" + 0.007*"life"'), (3, '0.034*"via" + 0.028*"music" + 0.025*"game" + 0.020*"univ" + 0.019*"data"'), (4, '0.007*"us" + 0.006*"learn" + 0.006*"world" + 0.005*"system" + 0.005*"human"'), (5, '0.153*"de" + 0.103*"e" + 0.044*"um" + 0.040*"da" + 0.039*"para"'), (6, '0.055*"white" + 0.033*"black" + 0.016*"photo" + 0.016*"hou" + 0.016*"presid"'), (7, '0.016*"life" + 0.008*"success" + 0.007*"feel" + 0.007*"becom" + 0.006*"learn"'), (8, '0.009*"food" + 0.007*"home" + 0.006*"eat" + 0.006*"hou" + 0.006*"live"'), (9, '0.007*"trump" + 0.007*"us" + 0.007*"said" + 0.005*"men" + 0.005*"never"')]

In [238]:

    

def lda_to_vec(lda_input):
    num_topics = 10
    vec = [0]*num_topics
    for i in lda_input:
        col = i[0]
        val = i[1]
        vec[col] = val
    return vec


    

In [239]:

    

all_lda_vecs = []

n = 0
for i in corpus_new2:
    doc_lda = ldamodel_new[i]
    vec_lda = lda_to_vec(doc_lda)
    all_lda_vecs.append(vec_lda)
    n += 1
    if n <= 20:
#         print(doc_lda)
        print(vec_lda)


    

    




[0.10107509328751951, 0, 0.10852728610649451, 0, 0.075517989451679202, 0, 0, 0.67870291423836493, 0, 0.022212525354884893]
[0.12651750305493689, 0, 0.078822498619816675, 0, 0, 0, 0, 0, 0, 0.78631764775959023]
[0.25624254286537357, 0.055555672560972158, 0.52666362787494203, 0, 0, 0, 0, 0.080845266191335349, 0.030650578584385711, 0.047868018732012435]
[0, 0, 0.089334073178282222, 0, 0.28513255654787562, 0, 0.12729748109316336, 0.013654861923539681, 0.13482279146305834, 0.34849217621184131]
[0, 0, 0.07048149573898782, 0, 0.049042266612691489, 0, 0, 0.74698620673638572, 0, 0.12742346224893911]
[0.12483905231383763, 0, 0.67938636218160664, 0.049743677044155082, 0, 0, 0, 0.090519253639670699, 0.053174661362144673, 0]
[0.43435859745966204, 0, 0, 0, 0.5633153219581547, 0, 0, 0, 0, 0]
[0.14330170401379469, 0.011304899531451831, 0, 0.50968739188797629, 0, 0, 0, 0.14212458616559379, 0, 0.18878576465933591]
[0.2582394907418743, 0.67150372166000294, 0, 0, 0, 0, 0.015085966464631617, 0.044863513737936962, 0, 0]
[0.5974397502058838, 0.04668449381995067, 0.17697859186129475, 0.13312071435631403, 0, 0, 0, 0.043937889332972858, 0, 0]
[0, 0, 0.087285469161189913, 0, 0.42772750705620227, 0, 0, 0.26682984227391804, 0.023295928581208946, 0.19331777381790038]
[0.01942651987924118, 0.66275946529507235, 0.19618261460818687, 0, 0, 0, 0, 0.12069813710614713, 0, 0]
[0.16223983488488722, 0.15157519425821056, 0.68194176904106529, 0, 0, 0, 0, 0, 0, 0]
[0.10907974189752828, 0.803938928938013, 0, 0, 0, 0, 0, 0.038881002894151327, 0.046941738087943677, 0]
[0.73545423731064519, 0.14329267203510232, 0, 0, 0.12026559235724067, 0, 0, 0, 0, 0]
[0.065044537164511543, 0.022479349303608384, 0.075485742692583069, 0.40373894363497015, 0, 0, 0, 0.069066524381932656, 0.033815399421313619, 0.32993527405999856]
[0.39868476651787693, 0.4017233575946888, 0, 0, 0.1709475527166936, 0.020475693896635415, 0, 0, 0, 0]
[0.70113910864713669, 0.023987201794184315, 0.09279268553086012, 0, 0.020209653675172225, 0, 0, 0, 0.01165150919638026, 0.14877548221498543]
[0, 0, 0, 0, 0, 0, 0.018648884967718157, 0.10157486903844661, 0, 0.87774921091356095]
[0.047508028191598467, 0.67910023799983799, 0.20075033734825551, 0, 0, 0, 0, 0, 0.06284930883038807, 0]

In [241]:

    

print(len(all_lda_vecs))
print(sum(all_lda_vecs[1000]))


    

    




3201
0.993042167848

In [243]:

    

# Save all_lda_vecs
fall_lda_vecs = open('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/all_lda_vecs','wb')
pickle.dump(all_lda_vecs, fall_lda_vecs)