In [2]:
import json
import numpy as np
import pandas as pd
import datetime
import os
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("white")
sns.set_context("poster")

colours = sns.color_palette()

%matplotlib inline


/Users/nickhoernle/anaconda2/lib/python2.7/site-packages/pandas/computation/__init__.py:19: UserWarning: The installed version of numexpr 2.4.4 is not supported in pandas and will be not be used

  UserWarning)

In [3]:
question_types = {
    1: 'ranking',
    2: 'select5',
    3: 'pointwise_score'
}

json_data=open('../web-selection-form/data/questions.json').read()
question_mapping = json.loads(json_data)

In [4]:
# get the actual questions and answers
print(question_mapping['question1']['answers']['1'])
print(question_mapping['question1']['answers']['10'])


Current admission guidelines for US colleges admissions offices result in the colleges failing to admit many top quality students.
Beyond admissions, colleges should use Data Science as an early warning tool to detect students who need possible assistance.

In [5]:
columns = ([('question1', i) for i in range(1,11)] 
            +[('question1', 'qustion_type'),('question1', 'end_time')]
            +[('question2', i) for i in range(1,11)]
            +[('question2', 'qustion_type'),('question2', 'end_time')]
            +[('question3', i) for i in range(1,11)]
            +[('question3', 'qustion_type'),('question3', 'end_time')]
            +[('feedback', '')]
            +[('start_time', '')]
            +[('end_time', '')])

columns = pd.MultiIndex.from_tuples(columns)

data_df = pd.DataFrame(columns=columns)
data_df.index.name = 'respondent'
data_df


Out[5]:
question1 ... question3 feedback start_time end_time
1 2 3 4 5 6 7 8 9 10 ... 6 7 8 9 10 qustion_type end_time
respondent

0 rows × 39 columns


In [6]:
data_path = './../responses/'
# Deal with question 1
    
for index, file in enumerate(os.listdir(data_path)):
    
    if 'second_stage' in file: continue
        
    json_data=open(os.path.join(data_path, file)).read()
    data = json.loads(json_data)

    data_df.set_value(index=index, col=('feedback', ''), value=data['qualitative_feedback'])

    start_time = datetime.datetime.fromtimestamp(data['time_start']/1000.0)
    end_time = datetime.datetime.fromtimestamp(data['time_end']/1000.0)
    
    data_df.set_value(index=index, col=('start_time', ''), value=start_time)
    data_df.set_value(index=index, col=('end_time', ''), value=end_time)

    question_times = [datetime.datetime.fromtimestamp(data['question%i_time'%i]/1000.0) for i in range(1,4)]
    question_times = list(np.sort(question_times + [start_time]))
    
    for question_number in range(1,4):

        question_number_string = 'question%i'%question_number
        question_type = 'question%i_type'%question_number
        question_time = datetime.datetime.fromtimestamp(data['question%i_time'%question_number]/1000.0)
        question_selections = 'question%i_selections'%question_number

        this_q = question_times.index(question_time)
        time_delta = question_times[this_q] - question_times[this_q-1]
        
        data_df.set_value(index=index, col=(question_number_string, 'qustion_type'), value=data[question_type])
        data_df.set_value(index=index, col=(question_number_string, 'end_time'), value=time_delta)

        if data[question_type] == 1: # 'ranking question'

            for i in range(1,11):

                answer = data[question_selections][i-1]
                data_df.set_value(index=index, col=(question_number_string, answer), value=i)

        elif data[question_type] == 2: # 'selecting question'

            for i in range(1,11):

                answer = 10 if i in data[question_selections] else 0
                data_df.set_value(index=index, col=(question_number_string, i), value=answer)

        else: # 'number question'

            answers = []
            for i in range(1,11):

                answers.append(int(data[question_selections][i]))
                
            for i in range(1,11):
                
                data_df.set_value(index=index, col=(question_number_string, i), value=(answers[i-1]/float(np.sum(answers))))

        
data_df = data_df.drop_duplicates()
data_df = data_df.ix[~data_df.index.isin([7,17,28,29])]

In [7]:
data_df


Out[7]:
question1 ... question3 feedback start_time end_time
1 2 3 4 5 6 7 8 9 10 ... 6 7 8 9 10 qustion_type end_time
respondent
0 0.0555556 0.0925926 0.0925926 0.166667 0.037037 0.0740741 0.166667 0.12963 0.148148 0.037037 ... 10 0 0 10 10 2 0:02:17.718000 Ranking was quite challenging, especially give... 2017-04-27 14:17:23.001000 2017-04-27 14:27:09.070000
1 0.15625 0.15625 0.15625 0.015625 0.015625 0.015625 0.15625 0.15625 0.15625 0.015625 ... 6 3 2 5 7 1 0:11:04.422000 2017-04-27 14:40:32.115000 2017-04-27 14:59:16.515000
2 10 10 10 0 0 0 0 0 10 10 ... 0.160714 0.0714286 0.0357143 0.0178571 0.107143 3 0:04:24.975000 ranking is difficult, scoring and/or selecting... 2017-04-27 14:44:51.384000 2017-04-27 15:00:08.110000
3 4 6 10 7 9 5 8 3 1 2 ... 0.176471 0.0196078 0.196078 0.0196078 0.156863 3 0:01:21.415000 Ticking the 5 most relevant points to support ... 2017-04-27 14:46:53.279000 2017-04-27 15:00:30.966000
4 3 2 1 8 7 10 5 4 9 6 ... 0.0526316 0.0526316 0.526316 0.0526316 0.0526316 3 0:05:27.783000 2017-04-27 15:07:22.730000 2017-04-27 15:20:44.715000
5 6 4 8 7 5 9 10 3 1 2 ... 0 10 10 0 10 2 0:01:50.762000 2017-04-27 15:47:02.165000 2017-04-27 15:54:57.614000
6 5 7 4 2 10 9 1 8 6 3 ... 10 10 0 10 0 2 0:01:21.468000 I liked the first metric (separate scores) mos... 2017-04-27 17:49:19.700000 2017-04-27 17:58:54.911000
8 0.241379 0.103448 0.0344828 0.172414 0.103448 0.0344828 0.0344828 0.172414 0.0344828 0.0689655 ... 0 0 0 0 10 2 0:01:39.260000 2017-04-27 17:40:01.636000 2017-04-27 18:44:42.758000
9 0.132075 0.150943 0.0377358 0.0188679 0.113208 0.132075 0.188679 0.0754717 0.132075 0.0188679 ... 6 5 9 10 8 1 0:00:51.064000 Selection was easiest to complete, but I think... 2017-04-27 19:09:53.142000 2017-04-27 19:36:42.114000
10 0.109091 0.0909091 0.0909091 0.0909091 0.109091 0.0909091 0.0909091 0.145455 0.0909091 0.0909091 ... 10 0 10 0 10 2 0:01:24.218000 2017-04-27 20:20:16.054000 2017-04-27 20:23:20.520000
11 0.0882353 0.205882 0.147059 0.0294118 0.0294118 0.0588235 0.147059 0.0294118 0.205882 0.0588235 ... 1 6 9 2 4 1 0:03:12.313000 2017-04-27 20:13:55.582000 2017-04-27 20:24:19.487000
12 6 3 9 5 4 10 1 2 8 7 ... 10 0 0 10 10 2 0:00:54.006000 2017-04-27 20:45:19.272000 2017-04-27 20:48:55.049000
13 0.078125 0.109375 0.09375 0.09375 0.125 0.0625 0.078125 0.125 0.09375 0.140625 ... 2 3 10 6 8 1 0:03:32.674000 Giving a score of 1 to 10 was the most difficu... 2017-04-27 20:51:08.794000 2017-04-27 21:06:33.623000
14 6 1 3 7 8 5 4 10 2 9 ... 0.157895 0.0175439 0.0175439 0.157895 0.0877193 3 0:03:20.558000 2017-04-27 23:37:29.434000 2017-04-27 23:49:04.291000
15 10 10 0 0 10 10 0 0 10 0 ... 0.196078 0.0392157 0.0980392 0.176471 0.0980392 3 0:07:33.466000 Ranking is the most difficult. I prefer the f... 2017-04-27 23:30:32.928000 2017-04-27 23:53:12.786000
16 10 0 0 0 10 0 0 10 10 10 ... 0.0958904 0.136986 0.0684932 0.0821918 0.0821918 3 0:03:26.556000 Assigning values from 1-10 was the most diffic... 2017-04-28 01:21:43.367000 2017-04-28 01:28:29.475000
18 0 0 0 0 10 10 10 10 10 0 ... 0.163636 0.0909091 0.0181818 0.109091 0.0545455 3 0:01:26.213000 2017-04-28 08:23:26.424000 2017-04-28 08:39:33.224000
19 0.117647 0.0235294 0.0823529 0.117647 0.0941176 0.117647 0.105882 0.117647 0.105882 0.117647 ... 10 9 5 1 8 1 0:01:29.336000 2017-04-28 08:34:39.694000 2017-04-28 08:44:19.413000
20 0.115385 0.0769231 0.0897436 0.102564 0.102564 0.0641026 0.0641026 0.128205 0.128205 0.128205 ... 7 6 4 5 1 1 0:02:02.332000 I think ranking was the easiest format in this... 2017-04-28 08:54:13.865000 2017-04-28 09:00:56.242000
21 0.125 0.111111 0.0972222 0.111111 0.0694444 0.0972222 0.111111 0.0694444 0.0833333 0.125 ... 3 7 9 8 6 1 0:02:17.449000 The checkbox question was the easiest to compl... 2017-04-28 08:55:47.607000 2017-04-28 09:04:06.216000
22 7 4 9 1 3 5 6 8 10 2 ... 10 10 10 0 10 2 0:02:10.011000 2/3rds of everything is irrelevant. What matte... 2017-04-28 09:26:14.238000 2017-04-28 09:36:20.315000
23 3 1 2 10 6 5 7 4 8 9 ... 10 10 0 0 0 2 0:02:23.703000 Question 1's ranking scheme was the easiest as... 2017-04-28 09:44:08.985000 2017-04-28 09:54:05.696000
24 10 10 10 0 0 0 10 0 10 0 ... 1 5 9 7 4 1 0:02:33.403000 Ranking was more difficult than selection.\nTh... 2017-04-28 10:16:32.505000 2017-04-28 10:34:30.335000
25 0.0357143 0.160714 0.125 0.125 0.0892857 0.0357143 0.125 0.142857 0.107143 0.0535714 ... 10 0 0 0 10 2 0:03:16.609000 Ranking was more difficult. Particularly when ... 2017-04-28 10:31:23.779000 2017-04-28 10:47:10.506000
26 10 10 10 0 0 0 10 0 10 0 ... 0.188679 0.113208 0.0188679 0.0566038 0.0754717 3 0:01:20.461000 Was confusing that the first two questions wen... 2017-04-28 10:48:33.597000 2017-04-28 10:54:47.054000
27 1 9 5 8 4 3 6 2 7 10 ... 0.0714286 0.0952381 0.214286 0.0238095 0.119048 3 0:02:58.553000 2017-04-28 11:54:20.225000 2017-04-28 12:00:05.339000
30 0.0862069 0.0862069 0.137931 0.0172414 0.137931 0.0689655 0.0862069 0.137931 0.137931 0.103448 ... 10 0 0 10 10 2 0:05:52.408000 I thought ranking the options (Question 3) was... 2017-04-28 12:01:02.944000 2017-04-28 12:28:23.361000
31 7 3 5 6 8 4 2 10 1 9 ... 0.14 0.08 0.04 0.12 0.1 3 0:03:42.542000 the drag-and-drop method facilitates focus on ... 2017-04-28 16:25:39.019000 2017-04-28 16:40:21.713000
32 0.106383 0.212766 0.212766 0.0851064 0.0425532 0.0638298 0.0851064 0.0425532 0.106383 0.0425532 ... 10 0 0 10 10 2 0:04:37.951000 Several theorems in game theory and social sci... 2017-04-28 16:28:03.486000 2017-04-28 17:00:47.559000
33 2 4 5 6 7 8 3 9 1 10 ... 0.147059 0.132353 0.0735294 0.147059 0.0441176 3 0:01:18.584000 2017-04-28 17:46:38.305000 2017-04-28 17:52:54.337000
34 2 5 4 7 6 9 3 8 1 10 ... 0.0545455 0.145455 0.127273 0.0181818 0.163636 3 0:06:40.669000 I liked the voting format that automatically r... 2017-04-28 20:35:57.810000 2017-04-28 20:57:54.747000
35 0.0555556 0.166667 0.222222 0.0555556 0.0555556 0.111111 0.0555556 0.0555556 0.0555556 0.166667 ... 1 5 8 4 6 1 0:04:59.009000 Drag drop provides the most feedback -- you ca... 2017-04-28 22:57:09.282000 2017-04-28 23:10:47.643000
36 0.120482 0.0843373 0.060241 0.060241 0.120482 0.0963855 0.120482 0.120482 0.120482 0.0963855 ... 2 4 7 1 5 1 0:04:41.069000 I found that question 1 required a fair amount... 2017-04-29 01:24:46.422000 2017-04-29 01:43:55.095000
37 0.142857 0.128571 0.0714286 0.0571429 0.0857143 0.114286 0.114286 0.114286 0.128571 0.0428571 ... 10 10 0 10 0 2 0:07:41.038000 I thought tanking was a bit more difficult. Tr... 2017-04-29 01:50:05.323000 2017-04-29 02:12:47.818000
38 0 0 0 0 10 10 10 0 10 10 ... 8 9 1 7 4 1 0:02:38.187000 2017-04-29 03:41:16.234000 2017-04-29 03:51:01.789000
39 2 4 1 9 8 7 3 6 5 10 ... 10 0 0 10 10 2 0:03:30.085000 2017-04-29 23:00:43.212000 2017-04-29 23:52:37.473000
40 3 4 5 8 7 2 6 10 1 9 ... 10 10 0 10 0 2 0:02:56.280000 2017-05-01 05:37:32.448000 2017-05-01 05:48:46.538000

37 rows × 39 columns


In [8]:
fig, axes = plt.subplots(1,3, figsize=(15,3))
print('Question 1')
question='question1'
print(question_mapping[question]['question_description'])
for i, ax in enumerate(axes):
    
    agg_num = data_df[question][data_df[question]['qustion_type'] == i + 1][[1,2,3,4,5,6,7,8,9,10]].sum()
    
    q_num = agg_num.index
    data = agg_num.values
    ax.bar(q_num, data)
    ax.set_title(question_types[i+1])


Question 1
<i>Many people feel that USA college admissions are highly skewed toward outdated metrics. Recent research has uncovered metrics that are better able to predict students' future performance in college. However, colleges are reluctant to adopt these new metrics.</i><br/><br/>Imagine that you are in a debate and you are asked to support the argument that USA college admissions should change their admission metrics.

In [9]:
fig, axes = plt.subplots(1,3, figsize=(15,3))
print('Question 2')
question = 'question2'
print(question_mapping[question]['question_description'])
for i, ax in enumerate(axes):
    
    if i == 0:
        q_type = 1
    elif i == 2:
        q_type = 2
    else: 
        q_type = 3
    
    agg_num = data_df[question][data_df[question]['qustion_type'] == i + 1][[1,2,3,4,5,6,7,8,9,10]].sum()
    
    q_num = agg_num.index
    data = agg_num.values
    ax.bar(q_num, data)
    
    if q_type == 1:
        ax.set_title('Ranking')
    elif q_type == 2:
        ax.set_title('Plurality')
    else:
        ax.set_title('Cardinal Score')
        
    if i == 0:
        ax.set_ylabel('cumulative score')
    ax.set_xlabel('point id')
    ax.set_xticks(np.arange(1,11)+0.4)
    ax.set_xticklabels(np.arange(1,11), fontdict={'size':14})


Question 2
<i>The 'get to know you' styled interview is becoming a popular trend. However, research shows that humans are surprisingly bad at inferring essential qualities about an applicant's character and thus may make poor judgments about the future success of an applicant.</i><br/><br/> Imagine that you are in a debate and that you are asked to argue in support of the motion above, that the current structure of interviews is not helpful in successful hiring practices.

In [10]:
fig, axes = plt.subplots(1,3, figsize=(15,3))
print('Question 3')
question = 'question3'
print(question_mapping[question]['question_description'])
for i, ax in enumerate(axes):
    
    if i == 0:
        q_type = 1
    elif i == 2:
        q_type = 2
    else: 
        q_type = 3
        
    agg_num = data_df[question][data_df[question]['qustion_type'] == q_type][[1,2,3,4,5,6,7,8,9,10]].sum()
    
    agg_num.sort_index(inplace=True)
    q_num = agg_num.index
    data = agg_num.values
    ax.bar(q_num, data)
    
    if q_type == 1:
        ax.set_title('Ranking')
    elif q_type == 2:
        ax.set_title('Plurality')
    else:
        ax.set_title('Cardinal Score')
        
    if i == 0:
        ax.set_ylabel('cumulative score')
    ax.set_xlabel('point id')
    ax.set_xticks(np.arange(1,11)+0.4)
    ax.set_xticklabels(np.arange(1,11), fontdict={'size':14})
fig.tight_layout()


Question 3
<i>The World Health Organization (WHO) has proposed to have video game addiction included in its catalog of mental diseases, but rhetoric stating that video games are addictive and comparing them to drugs is misguided.</i><br/><br/> Imagine that you are in a debate and that you are asked to support the statement that video games should not be classified as addictive.

In [11]:
print(question_mapping['question1']['answers']['6'])
print(question_mapping['question1']['answers']['9'])
print(question_mapping['question1']['answers']['8'])
print
print(question_mapping['question2']['answers']['3'])
print(question_mapping['question2']['answers']['5'])
print(question_mapping['question2']['answers']['1'])
print
print(question_mapping['question3']['answers']['6'])
print(question_mapping['question3']['answers']['6'])
print(question_mapping['question3']['answers']['10'])


Improved admissions, resulting in improved graduation rates, will show in world rankings four years after initial implementation.
Currently colleges are not taking advantage of advanced statistical methods that can be used to predict student success.
Admissions officers should be more engaged in the long term goals of the university rather than optimizing for world rankings.

Unstructured, 'get-to-know' interviews are becoming popular in the workspace and in college admissions, yet these form a poor metric for predicting the future job performance of the interviewee.
Students are better at predicting other students' GPA scores when no interview is conducted.
Circumstance may be highly misleading in an interview scenario as an interviewee's demeanor may depend highly on external, hidden circumstances.

The American Journal of Psychiatry has published a study showing that the mental and social heath of the purported video game addicts is no different from individuals who are not addicted to video games.
The American Journal of Psychiatry has published a study showing that the mental and social heath of the purported video game addicts is no different from individuals who are not addicted to video games.
Using video gaming to relax does not constitute an addiction in much the same way as watching sports is not addictive.

View the qualitative feedback:


In [12]:
for i, f in data_df['feedback'].iteritems():
    
    print(f)
    print('------------------------------------------------------------------------')


Ranking was quite challenging, especially given that several statements were highly similar or conceptually related.
I think it would be helpful to have some very stupid arguments thrown in so that you can have greater variability in your measurement.
Glad to see you taking an interest in social psychology, little sister!
------------------------------------------------------------------------

------------------------------------------------------------------------
ranking is difficult, scoring and/or selecting is easier
------------------------------------------------------------------------
Ticking the 5 most relevant points to support my argument is the easiest form of feedback. Placing the different points in order is the most difficult and requires the most time as you have to read through each point multiple times in order to make comparisons and form the list in the order you want. Ranking the points on a scale of 1-10 is also relatively easy, but may not give the most accurate results as I often just end up choosing a random number in the region of important (6-10) or unimportant (which in this case I just left as 1 as I had 5 points already).
------------------------------------------------------------------------

------------------------------------------------------------------------

------------------------------------------------------------------------
I liked the first metric (separate scores) most, because I didn't have to do any artificial ranking or make difficult decisions between N things at once. I could. This is interesting though. One factor that will complicate your analysis is that more of the arguments for the first position were compelling than for the second two (though perhaps I'm biased by the format). So direct quantitative comparison between the formats, which doesn't account for the inherent differences in the distribution of argument persuasivenesses, might be misleading.


------------------------------------------------------------------------

------------------------------------------------------------------------
Selection was easiest to complete, but I think the rank-ordering will be the most highly informative.
------------------------------------------------------------------------

------------------------------------------------------------------------

------------------------------------------------------------------------

------------------------------------------------------------------------
Giving a score of 1 to 10 was the most difficult as it was hard to be fair and determine what I thought deserved a certain number. Choosing the 5 best arguments was fairly easy as I didn't really need to rank the statements. Dragging to rank all the choices was somewhat difficult but visually it was simple and easy because I was able to see my choices in order, rather than just attributing them a number 1 through 10.
------------------------------------------------------------------------

------------------------------------------------------------------------
Ranking is the most difficult.  I prefer the format that lets me choose on a scale from 1-10 how strong I think the argument is.  
------------------------------------------------------------------------
Assigning values from 1-10 was the most difficult voting format.  Simple selection was the easiest, and ranking fell in the middle.  I believe simple selection will produce the most coherent points, but it may be a very small difference between ranking and selection.  Assigning values, while probably being better to analyze statistically, will probably produce the worst bias.
------------------------------------------------------------------------

------------------------------------------------------------------------

------------------------------------------------------------------------
I think ranking was the easiest format in this survey
------------------------------------------------------------------------
The checkbox question was the easiest to complete. The ranking question took the longest - I put greater emphasis on the first 5 ranks and cared less about the remaining 5. In the coring question from 1-10 I felt  that my numbers were fairly arbitrary and I did not need to use the full 1-10 scale (only used scores 1,5,6,7,8,9). 
------------------------------------------------------------------------
2/3rds of everything is irrelevant. What matters I suppose is being able to justifiably argue your point in a way which will support the evidence that preceded. Furthermore question the obvious and simply do what you can with what you have in the time you have left (desperado)
------------------------------------------------------------------------
Question 1's ranking scheme was the easiest as it's easier to visually arrange points. However, Question 2's ranking scheme might produce a better sub-selection of 5 because the focus was on finding only the most relevant points (minimizing cognitive load a bit). 
------------------------------------------------------------------------
Ranking was more difficult than selection.
The selection will result in the most coherent sub-selection of 5 points. The constraint of only choosing the 5 most relevant points necessitates the person to do some sort of ranking.
I like the simplicity of selecting the 5 points (the third format presented), but my preferred format of scoring each point from 1 to 10. This allowed me to assign 1 to the points I would never use and also put together a list of the points I would use in my debate. Furthermore, the usable points are now ranked in order, which may be handy when selecting points in a debate.
------------------------------------------------------------------------
Ranking was more difficult. Particularly when needing to evaluate several equally poor statements, needing to determine an ordering among those was not as easy as simply assigning a score to each statement. I believe that the scoring mechanism will allow you to see definitive separation between stronger and weaker statements as the gap between their position/score can be demonstrably widened.
------------------------------------------------------------------------
Was confusing that the first two questions went in opposite directions - easy to miss
------------------------------------------------------------------------

------------------------------------------------------------------------
I thought ranking the options (Question 3) was the best approach for comparing arguments. It's easier and faster to compare a given argument with two or one adjacent arguments than it is to judge them in absolute terms. 

I think the formats in Questions 2 and 3 are probably equivalent for selecting the best subset of five arguments. I disliked the format in Question 1 because it requires more work to compare different arguments. 
------------------------------------------------------------------------
the drag-and-drop method facilitates focus on comparing two points when I repeatedly ask myself "is this option better than the one above it?" whereas the "select top 5" makes me compare the option in question to up to 5 others (if I've already selected 5) to decide if it deserves to be selected above one of the others; and the "rate each option" voting mechanism forces me to weigh up the relative strength of each option against all the other options in order to balance my scoring. 
------------------------------------------------------------------------
Several theorems in game theory and social science [Arrow's, Gibbard–Satterthwaite] state that there's no excellent method of taking preferences from the members of a group and building a set of preferences (or single top choice) that the group would agree with. There might be a loophole around "separate into a top half and a bottom half", but it seems unlikely. Whatever mechanism you decide on will be vulnerable to some particular set of participant votes. That said, it may be possible to find a mechanism that works well for common voting patterns.

It's also not clear why collecting LESS data (ordinal or top-5) would ever be more useful than collecting the full scores, unless perhaps the task of scoring 1-10 is more noisy for reasons of cognitive load and greater breaking of IIA. But certainly whatever math is run on the top-5 data could instead be run on the top 5 of the ranking data. Unless the aim of this research is to show that less taxing question formats are less noisy, I don't see the point.

Overall, I suspect that I'm about to click through and get told I've been lied to and the real purpose of this research is something else altogether. 
------------------------------------------------------------------------

------------------------------------------------------------------------
I liked the voting format that automatically resorted the choices as they were selected. This made it visually easy to follow the order of 10 items and reorder as needed. I also liked the "choose the best 5" voting format, because I didn't have to select options that I felt were irrelevant, and also because I didn't have to put them in an order and found many choices to be equally as good as another. It was also quick and easy. I disliked the "insert order" voting option because it was cumbersome to go back and re enter numbers and ensure I didn't rank multiple options with the same number. I liked it better when it resorted for me..
------------------------------------------------------------------------
Drag drop provides the most feedback -- you can immediately see the other items reordered. 
------------------------------------------------------------------------
I found that question 1 required a fair amount of previous understanding of the US college application process and of acronyms used (such as ACT). I wasn't clear on what a few of the statements meant, not being familiar with the US system myself. I found selecting 5 relevant points the easiest selection, probably because I didn't need to have organized my thoughts as much as the first two that required specific ranking or scoring. I therefore believe that the relevant selection will probably result in the most whereby subsection of 5 points :)
------------------------------------------------------------------------
I thought tanking was a bit more difficult. Trying to assess the strength of a single sentence set of claims based on a value of 1-10 is arbitrary.
I thought it was a fascinating way to ask questions and fruitful to uncover interesting dynamics in the way that questions are answered. 
------------------------------------------------------------------------

------------------------------------------------------------------------

------------------------------------------------------------------------

------------------------------------------------------------------------

In [13]:
data_df['question1'][data_df['question1']['qustion_type'] == 1]


Out[13]:
1 2 3 4 5 6 7 8 9 10 qustion_type end_time
respondent
3 4 6 10 7 9 5 8 3 1 2 1 0:03:20.055000
4 3 2 1 8 7 10 5 4 9 6 1 0:02:46.078000
5 6 4 8 7 5 9 10 3 1 2 1 0:01:35.948000
6 5 7 4 2 10 9 1 8 6 3 1 0:01:24.361000
12 6 3 9 5 4 10 1 2 8 7 1 0:00:06.229000
14 6 1 3 7 8 5 4 10 2 9 1 0:00:42.892000
22 7 4 9 1 3 5 6 8 10 2 1 0:03:09.368000
23 3 1 2 10 6 5 7 4 8 9 1 0:03:22.487000
27 1 9 5 8 4 3 6 2 7 10 1 0:00:58.677000
31 7 3 5 6 8 4 2 10 1 9 1 0:04:19.443000
33 2 4 5 6 7 8 3 9 1 10 1 0:02:57.543000
34 2 5 4 7 6 9 3 8 1 10 1 0:09:01.224000
39 2 4 1 9 8 7 3 6 5 10 1 0:43:13.912000
40 3 4 5 8 7 2 6 10 1 9 1 0:05:24.098000

In [16]:
ans0 = []
ans1 = []
# j = 0
import scipy.stats as stats
for question in ['question1','question2','question3']:
    for q_type in [1,3]:
        scores = [[] for i in range(10)]
#         q_type = 1
#         question = 'question3'
        for i,row in data_df[question][data_df[question]['qustion_type'] == q_type].iterrows():

            row = row.ix[0:10]
            row = (row/row.max()) * 10 if q_type == 3 else row
            j = 0
            for _,k in row.iteritems():

                scores[j].append(k)

                j += 1
        
        res = stats.mode(scores, axis=1)
        sorted(np.array([np.mean(scores, axis=1), np.std(scores, axis=1)]).T, key=lambda x: x[1])
        # np.array([np.mean(scores, axis=1), np.std(scores, axis=1)]).T
        ans0.append(np.array([np.mean(scores, axis=1), np.std(scores, axis=1)])[0,:])
        ans1.append(np.array([np.mean(scores, axis=1), np.std(scores, axis=1)])[1,:])
plt.scatter(ans0, ans1)
plt.xlabel('score mean')
plt.ylabel('score std')


Out[16]:
<matplotlib.text.Text at 0x10ce76510>

In [17]:
np.array([np.mean(scores, axis=1), np.std(scores, axis=1)])


Out[17]:
array([[ 3.36111111,  4.9537037 ,  6.36111111,  6.51851852,  5.32407407,
         7.27777778,  4.62037037,  4.91666667,  4.50925926,  5.0462963 ],
       [ 2.35489377,  2.5865505 ,  2.77708325,  3.25378328,  3.36480768,
         2.99330529,  3.00803394,  3.47510991,  3.41849795,  2.08732129]])

In [18]:
def set_regret(profile, S, T):
    '''
    Minimax Regret Set (naive method)
    '''

    if len(T) == 0:
        regret = 0;
        return regret

In [19]:
T = set([2,3,7,11])
S = set([1,5])

In [20]:
len(T) == 0


Out[20]:
False

In [21]:
profile = [list(np.random.permutation([1,2,3,4,5,6,7,8,9,10])) for i in range(10)]

# first_indices_T = np.array([[i for i,elem in enumerate(x) if x[i] in T][0] + 1 for x in profile])
# first_indices_S = np.array([[i for i,elem in enumerate(x) if x[i] in S][0] + 1 for x in profile])

# regret = np.sum((first_indices_T < first_indices_S)*(1./first_indices_T));
# regret

In [19]:
import itertools
k = 3
permutations1 = itertools.permutations([1,2,3,4,5,6,7,8,9,10], k)

In [20]:
output_regret = np.inf
permutations = None
for S in permutations1:
    
    max_reg = 0
    permutation = None
    
    permutations2 = itertools.permutations([1,2,3,4,5,6,7,8,9,10], k)
    for T in permutations2:
        
        first_indices_T = np.array([[i for i,elem in enumerate(x) if x[i] in T][0] + 1 for x in profile])
        first_indices_S = np.array([[i for i,elem in enumerate(x) if x[i] in S][0] + 1 for x in profile])

        regret = np.sum((first_indices_T < first_indices_S)*(1./first_indices_T));
        
        if regret > max_reg:
            
            max_reg = regret
            
    if max_reg < output_regret:

        output_regret = max_reg
        permutations = S


---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-20-5c5a971f4cb7> in <module>()
     10 
     11         first_indices_T = np.array([[i for i,elem in enumerate(x) if x[i] in T][0] + 1 for x in profile])
---> 12         first_indices_S = np.array([[i for i,elem in enumerate(x) if x[i] in S][0] + 1 for x in profile])
     13 
     14         regret = np.sum((first_indices_T < first_indices_S)*(1./first_indices_T));

KeyboardInterrupt: 

In [20]:
output_regret


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-20-14accf435c37> in <module>()
----> 1 output_regret

NameError: name 'output_regret' is not defined

In [21]:
permutations


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-21-e0d3aa1f8a99> in <module>()
----> 1 permutations

NameError: name 'permutations' is not defined

In [ ]:


In [22]:
# question_rank = data_df['question1'][data_df['question1']['qustion_type'] == 1]
# data_df[['question1', 'question2' , 'question3']][data_df[['question1', 'question2' , 'question3'], 'qustion_type'] == 1]
q_type = 1
question_rank = [data_df['question1'][data_df['question1']['qustion_type'] == q_type],
                data_df['question2'][data_df['question2']['qustion_type'] == q_type],
                data_df['question3'][data_df['question3']['qustion_type'] == q_type]]

q_type = 2
question_select = [data_df['question1'][data_df['question1']['qustion_type'] == q_type],
                data_df['question2'][data_df['question2']['qustion_type'] == q_type],
                data_df['question3'][data_df['question3']['qustion_type'] == q_type]]

q_type = 3
question_number = [data_df['question1'][data_df['question1']['qustion_type'] == q_type],
                data_df['question2'][data_df['question2']['qustion_type'] == q_type],
                data_df['question3'][data_df['question3']['qustion_type'] == q_type]]

In [23]:
rank = []
select = []
number = []
rank_std = []
select_std = []
number_std = []

for df in question_rank:
    
    acceptable_times = df[((df.end_time < datetime.timedelta(minutes=20)) & (df.end_time > datetime.timedelta(minutes=0)))]
    
    times = []
    for i, t in acceptable_times.end_time.iteritems():
        
        times.append(t.seconds)
    
    rank.append(times)
    
for df in question_select:
    
    acceptable_times = df[((df.end_time < datetime.timedelta(minutes=20)) & (df.end_time > datetime.timedelta(minutes=0)))]
    
    times = []
    for i, t in acceptable_times.end_time.iteritems():
        
        times.append(t.seconds)
    
    select.append(times)
    
for df in question_number:
    
    acceptable_times = df[((df.end_time < datetime.timedelta(minutes=20)) & (df.end_time > datetime.timedelta(minutes=0)))]
    
    times = []
    for i, t in acceptable_times.end_time.iteritems():
        
        times.append(t.seconds)
    
    number.append(times)

In [24]:
# indexes = np.array([1,1.2,1.4])
plt.figure(figsize=(5,4))

indexes = np.array([1,2,3])
answers = np.array([[np.mean(r) for r in rank], [np.mean(r) for r in number], [np.mean(r) for r in select]])
std = np.array([[np.std(r) for r in rank], [np.std(r) for r in select], [np.std(r) for r in number]])

plt.bar(indexes, answers[0,:], yerr=std[0,:], width=0.08, linewidth=0, color=[colours[0], colours[0], colours[0]], label='question1', ecolor='black', error_kw={'linewidth': 1.5},  alpha=0.8)
plt.bar(indexes+0.08, answers[1,:], yerr=std[1,:], width=0.08, color=[colours[1], colours[1], colours[1]], label='question2', ecolor='black', error_kw={'linewidth': 1.5}, alpha=0.8)
plt.bar(indexes+0.16, answers[2,:], yerr=std[2,:], width=0.08, color=[colours[2], colours[2], colours[2]], label='question3', ecolor='black', error_kw={'linewidth': 1.5}, alpha=0.8)
plt.ylabel('time to complete (s)')

plt.legend(loc='best')
plt.xticks([1.2,2.2,3.2], ['Ranking', 'Cardinal Score', 'Plurality'])

plt.show()



In [25]:
plt.boxplot(rank + select + number)

plt.xticks([2,5,8], ['rank', 'select', 'number'])
plt.show()



In [26]:
question_times = []
type_times = []
types = []
times = []

for index, row in data_df.iterrows():
    
    q1_time = row.question1.end_time.seconds
    q2_time = row.question2.end_time.seconds
    q3_time = row.question3.end_time.seconds
    
#     if q1_time < 60*20 and q2_time < 60*20 and q2_time < 60*20:
        
    if q1_time > 0 and q2_time > 0 and q2_time > 0:
        types_ = [0,0,0]    
        types_[row.question1.qustion_type - 1] = row.question1.end_time.seconds
        types_[row.question2.qustion_type - 1] = row.question2.end_time.seconds
        types_[row.question3.qustion_type - 1] = row.question3.end_time.seconds

        question_times.append(np.argsort([q1_time, q2_time, q3_time]))
        type_times.append(np.argsort(types_))

        times.append([q1_time, q2_time, q3_time])
        types.append([row.question1.qustion_type, row.question2.qustion_type, row.question3.qustion_type])

question_times = np.array(question_times)
type_times = np.array(type_times)

In [27]:
fig, (ax1,ax2) = plt.subplots(1,2, figsize=(12,4))

ax1.hist(question_times, label=['rank', 'select' , 'number'])
ax1.set_xticks([0,1,2])
ax1.set_xticklabels(['Topic1', 'Topic2', 'Topic3'])
ax1.legend(loc='best')

ax2.hist(type_times, label=['admissions', 'interview' , 'gaming'], alpha=0.8)
ax2.set_xticks([0,1,2])
ax2.set_xticklabels(['Ranking', 'select', 'number'])
ax2.legend(loc='right')

plt.show()



In [28]:
for q in range(1,4):
    
    df = data_df['question{}'.format(q)][data_df['question{}'.format(q)].qustion_type == 2][[1,2,3,4,5,6,7,8,9,10]].sum()
    df.sort_values(inplace=True)
    print(df)
    selections = list(df.index[6:])
    print('Question{}, Type: Plurality'.format(q))
    print(selections)
    
    for i in selections:
        print('"' +question_mapping['question{}'.format(q)]['answers']['{}'.format(i)] + '",')
        
    print


4      0.0
8     20.0
3     30.0
6     30.0
10    30.0
2     40.0
5     40.0
7     40.0
1     50.0
9     70.0
dtype: float64
Question1, Type: Plurality
[5, 7, 1, 9]
"Schools should be unconcerned about brief fluctuations in college ranking scores (caused by different admissions criteria), if it results in a stronger student body.",
"There is currently an inherent disconnect between admissions offices and graduation rates (e.g. there is no direct feedback from graduation to the admissions decisions that were made four years prior).",
"Current admission guidelines for US colleges admissions offices result in the colleges failing to admit many top quality students.",
"Currently colleges are not taking advantage of advanced statistical methods that can be used to predict student success.",

5      20.0
6      40.0
9      60.0
10     60.0
7      80.0
4      90.0
3     100.0
1     110.0
2     110.0
8     130.0
dtype: float64
Question2, Type: Plurality
[3, 1, 2, 8]
"Unstructured, 'get-to-know' interviews are becoming popular in the workspace and in college admissions, yet these form a poor metric for predicting the future job performance of the interviewee.",
"Circumstance may be highly misleading in an interview scenario as an interviewee's demeanor may depend highly on external, hidden circumstances.",
"An interviewee's conduct may be interpreted differently by different interviewers.",
"Interviewers naturally turn irrelevant information into a coherent narrative, biasing their conclusions.",

8      30.0
2      40.0
5      40.0
1      50.0
7      60.0
9      80.0
3      90.0
4      90.0
10    100.0
6     120.0
dtype: float64
Question3, Type: Plurality
[3, 4, 10, 6]
"The fact that a pleasurable activity released dopamine is uninformative, as dopamine is released while playing video games, taking drugs and while partaking in any form of pleasurable activity.",
"The American Journal of Psychiatry has published a study showing that at most 1 percent of video game players might exhibit characteristics of an addiction.",
"Using video gaming to relax does not constitute an addiction in much the same way as watching sports is not addictive.",
"The American Journal of Psychiatry has published a study showing that the mental and social heath of the purported video game addicts is no different from individuals who are not addicted to video games.",


In [29]:
qualitative_feedback = np.array([
    [-1, 0, 0],
    [-1, 1, 1],
    [-1, 1, 0],
    [ 0, 0, 1],
    [ 0, 1, 0],
    [ 0, 1,-1],
    [-1, 0, 1],
    [ 0, 1,-1],
    [ 1, 0, 0],
    [-1, 1, 0],
    [ 1, 1, 0],
    [-1, 1, 1],
    [-1, 0, 1],
    [ 1, 1,-1],
    [ 1, 0, 0],
    [ 0, 1, 0],
    [-1, 0, 0]
])

dislike = []
neutral = []
like    = []

for doc in qualitative_feedback:
    
    for i, rating in enumerate(doc):
        
        if rating == -1:
            dislike.append(i+1)
        elif rating == 0:
             neutral.append(i+1)
        else:
             like.append(i+1)

In [30]:
fig, ax = plt.subplots(1,1, figsize=(5,4))

ax.hist([dislike,like,neutral] , label=['Dislike', 'Neutral', 'Like'], color=[colours[0], colours[1], colours[2]], alpha=0.8)

# ax.set_title('Qualitative Feedback', fontdict={'size': 22})
ax.set_xticks([1.1,2.1,2.9])
ax.set_xticklabels(['Ranking', 'Cardinal Score', 'Plurality'], fontdict={'size': 16})
ax.legend(loc='center right', bbox_to_anchor=(0.55, 0.5), fontsize=16)
plt.show()



In [31]:
data_df


Out[31]:
question1 ... question3 feedback start_time end_time
1 2 3 4 5 6 7 8 9 10 ... 6 7 8 9 10 qustion_type end_time
respondent
0 0.0555556 0.0925926 0.0925926 0.166667 0.037037 0.0740741 0.166667 0.12963 0.148148 0.037037 ... 10 0 0 10 10 2 0:02:17.718000 Ranking was quite challenging, especially give... 2017-04-27 14:17:23.001000 2017-04-27 14:27:09.070000
1 0.15625 0.15625 0.15625 0.015625 0.015625 0.015625 0.15625 0.15625 0.15625 0.015625 ... 6 3 2 5 7 1 0:11:04.422000 2017-04-27 14:40:32.115000 2017-04-27 14:59:16.515000
2 10 10 10 0 0 0 0 0 10 10 ... 0.160714 0.0714286 0.0357143 0.0178571 0.107143 3 0:04:24.975000 ranking is difficult, scoring and/or selecting... 2017-04-27 14:44:51.384000 2017-04-27 15:00:08.110000
3 4 6 10 7 9 5 8 3 1 2 ... 0.176471 0.0196078 0.196078 0.0196078 0.156863 3 0:01:21.415000 Ticking the 5 most relevant points to support ... 2017-04-27 14:46:53.279000 2017-04-27 15:00:30.966000
4 3 2 1 8 7 10 5 4 9 6 ... 0.0526316 0.0526316 0.526316 0.0526316 0.0526316 3 0:05:27.783000 2017-04-27 15:07:22.730000 2017-04-27 15:20:44.715000
5 6 4 8 7 5 9 10 3 1 2 ... 0 10 10 0 10 2 0:01:50.762000 2017-04-27 15:47:02.165000 2017-04-27 15:54:57.614000
6 5 7 4 2 10 9 1 8 6 3 ... 10 10 0 10 0 2 0:01:21.468000 I liked the first metric (separate scores) mos... 2017-04-27 17:49:19.700000 2017-04-27 17:58:54.911000
8 0.241379 0.103448 0.0344828 0.172414 0.103448 0.0344828 0.0344828 0.172414 0.0344828 0.0689655 ... 0 0 0 0 10 2 0:01:39.260000 2017-04-27 17:40:01.636000 2017-04-27 18:44:42.758000
9 0.132075 0.150943 0.0377358 0.0188679 0.113208 0.132075 0.188679 0.0754717 0.132075 0.0188679 ... 6 5 9 10 8 1 0:00:51.064000 Selection was easiest to complete, but I think... 2017-04-27 19:09:53.142000 2017-04-27 19:36:42.114000
10 0.109091 0.0909091 0.0909091 0.0909091 0.109091 0.0909091 0.0909091 0.145455 0.0909091 0.0909091 ... 10 0 10 0 10 2 0:01:24.218000 2017-04-27 20:20:16.054000 2017-04-27 20:23:20.520000
11 0.0882353 0.205882 0.147059 0.0294118 0.0294118 0.0588235 0.147059 0.0294118 0.205882 0.0588235 ... 1 6 9 2 4 1 0:03:12.313000 2017-04-27 20:13:55.582000 2017-04-27 20:24:19.487000
12 6 3 9 5 4 10 1 2 8 7 ... 10 0 0 10 10 2 0:00:54.006000 2017-04-27 20:45:19.272000 2017-04-27 20:48:55.049000
13 0.078125 0.109375 0.09375 0.09375 0.125 0.0625 0.078125 0.125 0.09375 0.140625 ... 2 3 10 6 8 1 0:03:32.674000 Giving a score of 1 to 10 was the most difficu... 2017-04-27 20:51:08.794000 2017-04-27 21:06:33.623000
14 6 1 3 7 8 5 4 10 2 9 ... 0.157895 0.0175439 0.0175439 0.157895 0.0877193 3 0:03:20.558000 2017-04-27 23:37:29.434000 2017-04-27 23:49:04.291000
15 10 10 0 0 10 10 0 0 10 0 ... 0.196078 0.0392157 0.0980392 0.176471 0.0980392 3 0:07:33.466000 Ranking is the most difficult. I prefer the f... 2017-04-27 23:30:32.928000 2017-04-27 23:53:12.786000
16 10 0 0 0 10 0 0 10 10 10 ... 0.0958904 0.136986 0.0684932 0.0821918 0.0821918 3 0:03:26.556000 Assigning values from 1-10 was the most diffic... 2017-04-28 01:21:43.367000 2017-04-28 01:28:29.475000
18 0 0 0 0 10 10 10 10 10 0 ... 0.163636 0.0909091 0.0181818 0.109091 0.0545455 3 0:01:26.213000 2017-04-28 08:23:26.424000 2017-04-28 08:39:33.224000
19 0.117647 0.0235294 0.0823529 0.117647 0.0941176 0.117647 0.105882 0.117647 0.105882 0.117647 ... 10 9 5 1 8 1 0:01:29.336000 2017-04-28 08:34:39.694000 2017-04-28 08:44:19.413000
20 0.115385 0.0769231 0.0897436 0.102564 0.102564 0.0641026 0.0641026 0.128205 0.128205 0.128205 ... 7 6 4 5 1 1 0:02:02.332000 I think ranking was the easiest format in this... 2017-04-28 08:54:13.865000 2017-04-28 09:00:56.242000
21 0.125 0.111111 0.0972222 0.111111 0.0694444 0.0972222 0.111111 0.0694444 0.0833333 0.125 ... 3 7 9 8 6 1 0:02:17.449000 The checkbox question was the easiest to compl... 2017-04-28 08:55:47.607000 2017-04-28 09:04:06.216000
22 7 4 9 1 3 5 6 8 10 2 ... 10 10 10 0 10 2 0:02:10.011000 2/3rds of everything is irrelevant. What matte... 2017-04-28 09:26:14.238000 2017-04-28 09:36:20.315000
23 3 1 2 10 6 5 7 4 8 9 ... 10 10 0 0 0 2 0:02:23.703000 Question 1's ranking scheme was the easiest as... 2017-04-28 09:44:08.985000 2017-04-28 09:54:05.696000
24 10 10 10 0 0 0 10 0 10 0 ... 1 5 9 7 4 1 0:02:33.403000 Ranking was more difficult than selection.\nTh... 2017-04-28 10:16:32.505000 2017-04-28 10:34:30.335000
25 0.0357143 0.160714 0.125 0.125 0.0892857 0.0357143 0.125 0.142857 0.107143 0.0535714 ... 10 0 0 0 10 2 0:03:16.609000 Ranking was more difficult. Particularly when ... 2017-04-28 10:31:23.779000 2017-04-28 10:47:10.506000
26 10 10 10 0 0 0 10 0 10 0 ... 0.188679 0.113208 0.0188679 0.0566038 0.0754717 3 0:01:20.461000 Was confusing that the first two questions wen... 2017-04-28 10:48:33.597000 2017-04-28 10:54:47.054000
27 1 9 5 8 4 3 6 2 7 10 ... 0.0714286 0.0952381 0.214286 0.0238095 0.119048 3 0:02:58.553000 2017-04-28 11:54:20.225000 2017-04-28 12:00:05.339000
30 0.0862069 0.0862069 0.137931 0.0172414 0.137931 0.0689655 0.0862069 0.137931 0.137931 0.103448 ... 10 0 0 10 10 2 0:05:52.408000 I thought ranking the options (Question 3) was... 2017-04-28 12:01:02.944000 2017-04-28 12:28:23.361000
31 7 3 5 6 8 4 2 10 1 9 ... 0.14 0.08 0.04 0.12 0.1 3 0:03:42.542000 the drag-and-drop method facilitates focus on ... 2017-04-28 16:25:39.019000 2017-04-28 16:40:21.713000
32 0.106383 0.212766 0.212766 0.0851064 0.0425532 0.0638298 0.0851064 0.0425532 0.106383 0.0425532 ... 10 0 0 10 10 2 0:04:37.951000 Several theorems in game theory and social sci... 2017-04-28 16:28:03.486000 2017-04-28 17:00:47.559000
33 2 4 5 6 7 8 3 9 1 10 ... 0.147059 0.132353 0.0735294 0.147059 0.0441176 3 0:01:18.584000 2017-04-28 17:46:38.305000 2017-04-28 17:52:54.337000
34 2 5 4 7 6 9 3 8 1 10 ... 0.0545455 0.145455 0.127273 0.0181818 0.163636 3 0:06:40.669000 I liked the voting format that automatically r... 2017-04-28 20:35:57.810000 2017-04-28 20:57:54.747000
35 0.0555556 0.166667 0.222222 0.0555556 0.0555556 0.111111 0.0555556 0.0555556 0.0555556 0.166667 ... 1 5 8 4 6 1 0:04:59.009000 Drag drop provides the most feedback -- you ca... 2017-04-28 22:57:09.282000 2017-04-28 23:10:47.643000
36 0.120482 0.0843373 0.060241 0.060241 0.120482 0.0963855 0.120482 0.120482 0.120482 0.0963855 ... 2 4 7 1 5 1 0:04:41.069000 I found that question 1 required a fair amount... 2017-04-29 01:24:46.422000 2017-04-29 01:43:55.095000
37 0.142857 0.128571 0.0714286 0.0571429 0.0857143 0.114286 0.114286 0.114286 0.128571 0.0428571 ... 10 10 0 10 0 2 0:07:41.038000 I thought tanking was a bit more difficult. Tr... 2017-04-29 01:50:05.323000 2017-04-29 02:12:47.818000
38 0 0 0 0 10 10 10 0 10 10 ... 8 9 1 7 4 1 0:02:38.187000 2017-04-29 03:41:16.234000 2017-04-29 03:51:01.789000
39 2 4 1 9 8 7 3 6 5 10 ... 10 0 0 10 10 2 0:03:30.085000 2017-04-29 23:00:43.212000 2017-04-29 23:52:37.473000
40 3 4 5 8 7 2 6 10 1 9 ... 10 10 0 10 0 2 0:02:56.280000 2017-05-01 05:37:32.448000 2017-05-01 05:48:46.538000

37 rows × 39 columns

Find the combination of choices most frequently in the top 4:


In [34]:
result = []
for i in range(10000):
    
    row = np.random.choice(np.arange(1,11), size=10)
    res = dict()
    dist = []

    for row in [np.random.choice(np.arange(1,11), size=10, replace=False) for i in range(37)]:
        for window in np.concatenate([np.arange(0,3), np.arange(5,8)]):

            top3 = row[window:window+3]
        #     print(top3)

            for perm in itertools.combinations(top3, 2):

                if (tuple(perm) in res):
                    res[tuple(perm)] += 1
                elif tuple([perm[1], perm[0]]) in res:
                    res[tuple([perm[1], perm[0]])] += 1
                else:
                    res[tuple(perm)] = 1
                dist.append(np.abs(perm[0] - perm[1]))

    result.append(dist)
# plt.hist(dist, bins=8)
# plt.xlim([0,9])
# # plt.ylim([0,9])

In [35]:
# for i in result:
import collections
counts = []
for i in result:
    counts.append([c for i,c in collections.Counter(i).iteritems()])

In [36]:
means = np.array(counts).mean(axis=0)
stds = np.array(counts).std(axis=0)

In [57]:
import itertools
count_of_ones = []

fig, axes = plt.subplots(1,3,figsize=(12,4))
for k, (ax, question) in enumerate(zip(axes, ['question1','question2','question3'])):
    
    res = dict()
    dist = []

#     question = 'question1'

    for ix, group in data_df[question].groupby('qustion_type'):
#         print(group.shape)
        gp = group[[1,2,3,4,5,6,7,8,9,10]]

        for i, row in gp.iterrows():
#             print(row.sort_values())
            for window in np.concatenate([np.arange(0,3), np.arange(5,8)]):

                top3 = row.sort_values().index[window:window+3]
#                 print(top3)
                for perm in itertools.combinations(top3, 2):

                    if (tuple(perm) in res):
                        res[tuple(perm)] += 1
                    elif tuple([perm[1], perm[0]]) in res:
                        res[tuple([perm[1], perm[0]])] += 1
                    else:
                        res[tuple(perm)] = 1

                    dist.append(np.abs(perm[0] - perm[1]))
                    
    count_of_ones.append(np.sum(np.array(dist) == 1))
    ax.hist(dist, bins=9, label='Observed Distance')
    ax.set_title('Topic {}'.format(k+1))
    ax.set_xlabel('distance')
    if k == 0:
        ax.set_ylabel('frequency')
        
    ax.bar(np.arange(1,10),means, yerr=stds, color=colours[1], alpha=0.5, width=1, label='Expected Distance')
    
ax.legend(loc='center right', bbox_to_anchor=(2, 0.5), fontsize=16)

fig.tight_layout()
plt.show()



In [58]:
result = np.array(result)

In [59]:
ones = []
for r in result:
    ones.append(np.sum(r == 1))

In [60]:
plt.hist(ones)
plt.show()



In [61]:
mean = np.mean(ones)
std = np.std(ones)

In [62]:
mean


Out[62]:
133.4836

In [67]:
z_scores = (count_of_ones-mean)/std

In [68]:
count_of_ones


Out[68]:
[172, 207, 164]

In [69]:
p_values = stats.norm.sf(np.abs(z_scores))
p_values


Out[69]:
array([  2.22894748e-04,   1.02665579e-11,   2.70081974e-03])

In [70]:
2.70081974e-03


Out[70]:
0.00270081974

In [ ]: