In [2]:

    

import pandas as pd
import numpy as np
import sklearn.cluster
import sklearn.preprocessing
import pylab as pl


    

In [3]:

    

stock_prices = pd.read_csv('full_set/sp500hst.txt', header=None, names = ['Date', 'Ticker', 'Open Price', 'High Price', 'Low Price',
                                                                          'Closing Price', 'Volume'])


    

In [6]:

    

price_freq = pd.DataFrame(stock_prices.Ticker.value_counts(), columns = ['Freq'])


    

In [8]:

    

merged_stocks = pd.merge(stock_prices, price_freq, left_on='Ticker', right_index=True)


    

In [10]:

    

filtered_stocks = merged_stocks[merged_stocks.Freq == 245]


    

In [13]:

    

stocks_pivot = filtered_stocks.pivot_table('Closing Price', rows = 'Ticker', cols = 'Date')


    

In [15]:

    

P = stocks_pivot.values


    

In [17]:

    

price_difference = P[:,:-1]-P[:,1:]


    

In [19]:

    

corr_matrix = stocks_pivot.T.corr()


    

In [22]:

    

standard_P = sklearn.preprocessing.scale(P, axis=1)


    

In [24]:

    

ward = sklearn.cluster.Ward(n_clusters=100)


    

In [25]:

    

ward.fit(standard_P)


    

    
Out[25]:





Ward(compute_full_tree='auto', connectivity=None, copy=True,
   memory=Memory(cachedir=None), n_clusters=100, n_components=None)

In [27]:

    

clusters = pd.DataFrame(np.array([stocks_pivot.index.values, ward.labels_])).T


    

In [29]:

    

data = np.array([stocks_pivot.index.values, ward.labels_]).T


    

In [30]:

    

clusters = pd.DataFrame(data, columns=['ticker', 'cluster'])


    

In [32]:

    

idx = np.argsort(ward.labels_)


    

In [34]:

    

sorted_corr = corr_matrix.values


    

In [36]:

    

company_names = pd.read_table('ussyms/ussyms.txt', sep=';', header=None, names = ['ticker', 'company name'])


    

In [48]:

    

for i in idx:
    if sum(company_names.ticker == clusters.ix[i,0])!=0:
        print clusters.ix[i,1], company_names[company_names.ticker==clusters.ix[i,0]]['company name'].values[0]


    

    




 0  Meredith Corporation
0  Gannett Co., Inc.
0  Genworth Financial Inc
0  ConAgra Foods, Inc.
0  Teradyne, Inc.
1  Noble Energy Inc.
1  Apache Corporation
1  National-Oilwell, Inc.
1  Hess Corporation
1  Schlumberger N.V.
1  Cooper Cameron Corporation
2  Altria Group
2  DTE Energy Company
2  Sysco Corporation
2  Dominion Resources, Inc.
2  McDonald's Corporation
2  Deere & Company
2  McCormick & Company, Incorporated
2  Sara Lee Corporation
2  Ventas, Inc.
2  Clorox Company (The)
2  Northeast Utilities
2  Hormel Foods Corporation
2  Wisconsin Energy Corporation
3  Public Service Enterprise Group Incorporated
3  PROGRESS ENERGY INC
4  Ryder System, Inc.
4  Legg Mason, Inc.
4  Newell Rubbermaid Inc.
5  Kimberly-Clark Corporation
5  Eli Lilly and Company
5  CenturyTel, Inc.
6  First Tennessee National Corporation
6  Weyerhaeuser Company
6  Office Depot, Inc.
6  LSI Logic Corporation
6  Masco Corporation
7  ProLogis
7  Meadwestvaco Corporation
7  Mastercard Incorporated
7  International Paper Company
7  Hewlett-Packard Company
8  AutoZone, Inc.
8  EQUITY RESIDENTIAL
8  El Paso Corporation
8  Hasbro, Inc.
8  EMC Corporation
8  Apartment Investment and Management Company
8  Assurant, Inc.
8  Salesforce.com Inc
8  Family Dollar Stores, Inc.
8  TIME WARNER CABLE INC
8  Cummins Inc.
8  Hershey Foods Corporation
8  M&T Bank Corporation
8  Titanium Metals Corporation
8  AvalonBay Communities, Inc.
8  Qwest Communications International Inc.
9  Eastman Chemical Company
9  Rockwell Collins, Inc.
9  Northrop Grumman Corporation
9  FedEx Corporation
9  Polo Ralph Lauren Corporation
9  Precision Castparts Corporation
9  Textron Inc.
9  AMERIPRISE FINANCIAL SERVICES, INC.
9  Tiffany & Co.
10  Constellation Brands, Inc.
10  CONOCOPHILLIPS
10  Time Warner Inc.
11  Pactiv Corporation
11  Sunoco, Inc.
11  AutoNation, Inc.
12  McKesson Corporation
12  Intercontinental Exchange, Inc.
13  Illinois Tool Works Inc.
13  ChevronTexaco Corporation
13  Avery Dennison Corporation
14  HOSPIRA INC
14  DaVita Inc.
14  Laboratory Corporation of America Holdings
14  Cardinal Health, Inc.
14  Radioshack Corporation
15  Rowan Companies, Inc.
15  XL CAPITAL LIMITED
15  FMC Technologies, Inc.
15  Abercrombie & Fitch Company
16  H.J. Heinz Company
16  Varian Medical Systems, Inc.
16  Emerson Electric Company
16  Goodrich Corporation (The)
16  Wyndham Worldwide Corp
16  Ford Motor Credit Company
16  Estee Lauder Companies, Inc. (The)
16  Dover Corporation
16  Kraft Foods Inc.
16  Big Lots, Inc.
16  Snap-On Incorporated
16  Stanley Works (The)
16  Boeing Company (The)
16  ROCKWELL AUTOMATION INC
16  WPS Resources Corporation
16  Allegheny Technologies Incorporated
16  Southwest Airlines Company
17  Merck & Company, Inc.
17  J.M. Smucker Company (The)
17  CenterPoint Energy, Inc (Holding Co)
17  American Electric Power Company, Inc.
17  Potomac Electric Power Company
18  Parker-Hannifin Corporation
18  Prudential Financial Inc
18  Regions Financing Tr I
18  Xerox Corporation
18  Kimco Realty Corporation
18  United Parcel Service, Inc.
18  Honeywell International Inc.
18  SunTrust Banks, Inc.
18  Torchmark Corporation
18  Target Corporation
18  V.F. Corporation
19  Williams Companies, Inc. (The)
19  Jabil Circuit, Inc.
19  New York Times Company (The)
19  Zimmer Holdings, Inc.
19  Dow Chemical Company (The)
20  Praxair, Inc.
21  American Express Company
21  Harris Corporation
21  Corning Incorporated
21  United Technologies Corporation
21  Advanced Micro Devices, Inc.
21  Washington Mutual, Inc.
21  Plum Creek Timber Company, Inc.
21  SLM Corporation
22  Forest Laboratories, Inc.
22  Monsanto Company
22  PPL Corporation
23  Jacobs Engineering Group Inc.
23  Principal Financial Group Inc
23  Washington Post Company (The)
24  Goodyear Tire & Rubber Company (The)
24  FirstEnergy Corporation
24  Boston Scientific Corporation
24  Aes Trust
24  Exelon Corporation
25  Sprint  Nextel Corporation
25  Fidelity National Information Services, Inc.
25  MetroPCS Communications, Inc.
26  Aetna Inc.
26  CIGNA Corporation
26  WellPoint Health Networks Inc.
26  Coventry Health Care, Inc.
27  Stilwell Financial Inc.
27  ITT Industries, Inc.
27  CA Inc.
27  SuperValu Inc.
27  Anadarko Petroleum Corporation
28  Equifax, Inc.
28  Pall Corporation
28  Spectra Energy Corp
28  AFLAC Incorporated
28  Johnson Controls, Inc.
29  Entergy Corporation
29  Ameren Corporation
29  Edison International
30  PerkinElmer, Inc.
30  Lowe's Companies, Inc.
30  Nordstrom, Inc.
30  Constellation Energy Group, Inc.
30  Agilent Technologies, Inc.
30  Home Depot, Inc. (The)
31  Chesapeake Energy Corporation
31  Range Resources Corporation
32  Coca-Cola Company (The)
32  Freeport-McMoRan Copper & Gold, Inc.
32  Abbott Laboratories
33  Omnicom Group Inc.
33  Harley-Davidson, Inc.
34  Progressive Corporation (The)
34  Marriot International
34  Starwood Hotels & Resorts Worldwide, Inc.
34  Yum! Brands, Inc.
34  CB Richard Ellis Group Inc
34  Host Marriott Financial Trust
34  TJX Companies, Inc. (The)
34  Sherwin-Williams Company (The)
35  C.R. Bard, Inc.
35  St. Jude Medical, Inc.
36  Genuine Parts Company
36  Public Storage, Inc.
36  Coca-Cola Enterprises, Inc.
36  Boston Properties, Inc.
36  Vornado Realty Trust
36  Simon Property Group, Inc.
36  E.I. du Pont de Nemours and Company
36  Interpublic Group of Companies, Inc. (The)
36  PPG Industries, Inc.
37  Motorola, Inc.
37  Molson Coors Brewing  Company
38  Avon Products, Inc.
38  Networks Associates, Inc.
38  Goldman Sachs Group, Inc. (The)
38  Morgan Stanley
38  Exxon Mobil Corporation
39  Wal-Mart Stores, Inc.
39  JOHNSON & JOHNSON
39  Medtronic, Inc.
39  Massey Energy Company
39  Becton, Dickinson and Company
40  Dun & Bradstreet Corporation (The)
40  Invesco Plc
40  Total System Services, Inc.
40  Western Union Company (The)
40  Air Products and Chemicals, Inc.
41  Texas Instruments Incorporated
41  Analog Devices, Inc.
42  Bank of America Corporation
42  CVS Corporation
42  Iron Mountain Incorporated
43  Kellogg Company
43  Procter & Gamble Company (The)
43  Colgate-Palmolive Company
44  King Pharmaceuticals, Inc.
44  CF Industries Holdings, Inc.
44  AK Steel Holding Corporation
44  Baxter International Inc.
45  CBS Corporation
45  PNC Bank Corp.
45  Internationa Flavors & Fragrances, Inc.
45  Norfolk Souther Corporation
45  Walt Disney Company (The)
45  Nike, Inc.
45  CSX Corporation
45  Waters Corporation
45  Coach, Inc.
45  Capital One Financial Corporation
45  Union Pacific Corporation
46  BB&T Corporation
46  Fortune Brands, Inc.
46  General Electric Company
46  Lockheed Martin Corporation
46  Carnival Corporation
47  Federated Investors, Inc.
47  Dean Foods Company
47  Walgreen Company
47  Diamond Offshore Drilling, Inc.
48  Citigroup Inc.
48  American International Group, Inc.
49  Archer-Daniels-Midland Company
49  SEMPRA ENERGY
50  PITNEY BOWES INC
51  St. Paul Companies, Inc. (The)
52  Caterpillar, Inc.
52  Republic Services, Inc.
52  Health Care Property Investors, Inc.
52  Southern Company (The)
53  Murphy Oil Corporation
53  Gamestop Corporation
54  Tesoro Petroleum Corporation
54  Allegheny Energy, Inc.
55  International Game Technology
55  Vulcan Materials Company
56  UnumProvident Corporation
56  Marshall & Ilsley Corporation
56  NYSE Group, Inc.
56  MetLife, Inc.
56  Denbury Resources, Inc. (Holding Company)
57  Halliburton Company
57  Peabody Energy Corporation
58  Quanta Services, Inc.
59  Kroger Company (The)
60  Best Buy Co., Inc.
60  Gap, Inc. (The)
61  International Business Machines Corporation
61  Amphenol Corporation
61  Bemis Company, Inc.
61  Sealed Air Corporation
62  General Dynamics Corporation
62  Raytheon Company
62  Stryker Corporation
62  L-3 Communications Holdings, Inc.
62  DeVry Inc.
63  Comerica Incorporated
63  KeyCorp
63  Tyson Foods, Inc.
63  Lexmark International, Inc.
63  Darden Restaurants, Inc.
63  Limited Brands, Inc.
63  Smith International, Inc.
64  National Semiconductor Corporation
65  Ball Corporation
65  Chubb Corporation (The)
66  H&R Block, Inc.
66  Pfizer, Inc.
67  Bristol-Myers Squibb Company
67  Allergan, Inc.
68  Leggett & Platt, Incorporated
68  Whirlpool Corporation
68  Thermo Fisher Scientific Inc
69  Humana Inc.
69  UnitedHealth Group Incorporated
70  Marathon Oil Corporation
71  Moody's Corporation
71  McGraw-Hill Companies, Inc. (The)
72  Pacific Gas & Electric Co.
72  Duke Energy Corporation
73  Lennar Corporation
73  Eastman Kodak Company
74  Robert Half International Inc.
74  USX Corporation
75  Ecolab Inc.
76  Watson Pharmaceuticals, Inc.
76  EOG Resources, Inc.
76  AmerisourceBergen Corporation (Holding Co)
76  Pioneer Natural Resources Company
77  Fluor Corporation
78  3M Company
78  Xcel Energy Inc.
78  CMS Energy Corporation
78  Northern Indiana Public Service Company
78  Nicor Inc.
78  REYNOLDS AMERICAN INC
78  Consolidated Edison, Inc.
78  SCANA Corporation
78  Pinnacle West Capital Corporation
78  TECO Energy, Inc.
79  Danaher Corporation
79  General Mills, Inc.
80  Occidental Petroleum Corporation
81  Flowserve Corporation
81  Franklin Resources, Inc.
82  J P Morgan Chase & Co
82  Owens-Illinois, Inc.
82  J.C. Penney Company, Inc. Holding Company
82  Kohl's Corporation
83  Health Care REIT, Inc.
84  Hartford Financial Services Group, Inc. (The)
84  Tenet Healthcare Corporation
85  Nucor Corporation
85  Leucadia National Corporation
85  Bank of New York Company, Inc. (The)
86  CONSOL Energy Inc.
86  Equitable Resources, Inc.
86  Cabot Oil & Gas Corporation
86  Alcoa Inc.
87  Campbell Soup Company
87  Newmont Mining Corporation (Holding Company)
88  AT&T Inc.
88  Verizon Communications Inc.
89  Lincoln National Corporation
89  Wells Fargo & Company
89  Allstate Corporation (The)
90  Marsh & McLennan Companies, Inc.
91  Baker Hughes Incorporated
92  Nabors Industries, Inc.
92  Southwestern Energy Company
92  Devon Energy Corporation
93  U.S. Bancorp
93  Safeway Inc.
93  Harman International Industries, Incorporated
94  MEMC Electronic Materials, Inc.
94  State Street Corp
95  D.R. Horton, Inc.
95  Pulte Homes, Inc.
96  Red Hat, Inc.
96  American Tower Corporation
97  Valero Energy Corporation
98  Pepsico, Inc.
98  W.W. Grainger, Inc.
98  Eaton Corporation
99  Computer Sciences Corporation
99  MedcoHealth Solutions Inc
99  Quest Diagnostics Incorporated
99  Western Digital Corporation

In [ ]: