CVS parser



In [9]:
file = "/root/git/SolarDataRESTfulAPI/testdata/h00t_1310160720_1406030410.csv"

url = "http://slb.nu/soldata/index.php?KEY=h00t&start=1310160720&stop=1406060410"

In [10]:
import pandas

In [18]:
df = pandas.read_csv(url,sep = ";",parse_dates=[[0, 1]],skiprows=8, header = None ,infer_datetime_format = True,na_values = ["     ","    ","  "," ",""])

In [19]:
df


Out[19]:
0_1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
0 13-10-16 07:20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...
1 13-10-16 07:30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...
2 13-10-16 07:40 0 0 0 0 0 0 0 0 32 0 0 32 0 0 0 0 0 0 0 ...
3 13-10-16 07:50 0 0 0 0 0 0 0 0 139 0 0 139 0 0 0 0 0 0 0 ...
4 13-10-16 08:00 0 0 0 0 0 0 0 0 297 0 0 297 0 0 0 0 0 0 0 ...
5 13-10-16 08:10 0 0 0 0 0 0 0 0 422 0 0 422 0 0 0 0 0 0 0 ...
6 13-10-16 08:20 0 0 0 0 0 0 0 0 650 0 0 650 0 0 0 0 0 0 0 ...
7 13-10-16 08:30 0 0 0 0 0 0 0 0 783 0 0 783 0 0 0 0 0 0 0 ...
8 13-10-16 08:40 0 0 0 0 0 0 0 0 1181 0 0 1181 0 0 0 0 0 0 0 ...
9 13-10-16 08:50 0 0 0 0 0 0 0 0 1217 0 0 1217 0 0 0 0 0 0 0 ...
10 13-10-16 09:00 0 0 0 0 0 0 0 0 1486 0 0 1486 0 0 0 0 0 0 0 ...
11 13-10-16 09:10 0 0 0 0 0 0 0 0 1821 0 0 1821 0 0 0 0 0 0 0 ...
12 13-10-16 09:20 3338 0 0 0 0 0 0 0 0 0 0 3338 0 0 0 0 0 0 0 ...
13 13-10-16 09:30 0 0 0 0 0 0 0 0 3338 0 0 3338 0 0 0 0 0 0 0 ...
14 13-10-16 09:40 0 0 0 0 0 0 0 0 3937 0 0 3937 0 0 0 0 0 0 0 ...
15 13-10-16 09:50 5545 0 0 0 0 0 0 0 0 0 0 5545 0 0 0 0 0 0 0 ...
16 13-10-16 10:00 8112 0 0 0 0 0 0 0 0 0 0 8112 0 0 0 0 0 0 0 ...
17 13-10-16 10:10 0 0 0 0 0 0 0 0 8112 0 0 8112 0 0 0 0 0 0 0 ...
18 13-10-16 10:20 0 0 0 0 0 0 0 0 8646 0 0 8646 0 0 0 0 0 0 0 ...
19 13-10-16 10:30 8337 0 0 0 0 0 0 0 0 0 0 8337 0 0 0 0 0 0 0 ...
20 13-10-16 10:40 9243 0 0 0 0 0 0 0 0 0 0 9243 0 0 0 0 0 0 0 ...
21 13-10-16 10:50 0 0 0 0 0 0 0 0 9243 0 0 9243 0 0 0 0 0 0 0 ...
22 13-10-16 11:00 0 0 0 0 0 0 0 0 3623 0 0 3623 0 0 0 0 0 0 0 ...
23 13-10-16 11:10 0 0 0 0 0 0 0 0 4487 0 0 4487 0 0 0 0 0 0 0 ...
24 13-10-16 11:20 0 0 0 0 0 0 0 0 4201 0 0 4201 0 0 0 0 0 0 0 ...
25 13-10-16 11:30 0 0 0 0 0 0 0 0 3759 0 0 3759 0 0 0 0 0 0 0 ...
26 13-10-16 11:40 0 0 0 0 0 0 0 0 3693 0 0 3693 0 0 0 0 0 0 0 ...
27 13-10-16 11:50 0 0 0 0 0 0 0 0 3501 0 0 3501 0 0 0 0 0 0 0 ...
28 13-10-16 12:00 0 0 0 0 0 0 0 0 5241 0 0 5241 0 0 0 0 0 0 0 ...
29 13-10-16 12:10 9600 0 0 0 0 0 0 0 0 0 0 9600 0 0 0 0 0 0 0 ...
30 13-10-16 12:20 0 0 0 0 0 0 0 0 9600 0 0 9600 0 0 0 0 0 0 0 ...
31 13-10-16 12:30 0 0 0 0 0 0 0 0 8126 0 0 8126 0 0 0 0 0 0 0 ...
32 13-10-16 12:40 0 0 0 0 0 0 0 0 10770 0 0 10770 0 0 0 0 0 0 0 ...
33 13-10-16 12:50 0 0 0 0 0 0 0 0 3107 0 0 3107 0 0 0 0 0 0 0 ...
34 13-10-16 13:00 0 0 0 0 0 0 0 0 2331 0 0 2331 0 0 0 0 0 0 0 ...
35 13-10-16 13:10 0 0 0 0 0 0 0 0 2212 0 0 2212 0 0 0 0 0 0 0 ...
36 13-10-16 13:20 0 0 0 0 0 0 0 0 2557 0 0 2557 0 0 0 0 0 0 0 ...
37 13-10-16 13:30 0 0 0 0 0 0 0 0 4100 0 0 4100 0 0 0 0 0 0 0 ...
38 13-10-16 13:40 0 0 0 0 0 0 0 0 4684 0 0 4684 0 0 0 0 0 0 0 ...
39 13-10-16 13:50 0 0 0 0 0 0 0 0 3176 0 0 3176 0 0 0 0 0 0 0 ...
40 13-10-16 14:00 0 0 0 0 0 0 0 0 6930 0 0 6930 0 0 0 0 0 0 0 ...
41 13-10-16 14:10 2039 0 0 0 0 0 0 0 0 0 0 2039 0 0 0 0 0 0 0 ...
42 13-10-16 14:20 0 0 0 0 0 0 0 0 2039 0 0 2039 0 0 0 0 0 0 0 ...
43 13-10-16 14:30 0 0 0 0 0 0 0 0 2102 0 0 2102 0 0 0 0 0 0 0 ...
44 13-10-16 14:40 0 0 0 0 0 0 0 0 2047 0 0 2047 0 0 0 0 0 0 0 ...
45 13-10-16 14:50 0 0 0 0 0 0 0 0 1771 0 0 1771 0 0 0 0 0 0 0 ...
46 13-10-16 15:00 1528 0 0 0 0 0 0 0 0 0 0 1528 0 0 0 0 0 0 0 ...
47 13-10-16 15:10 0 0 0 0 0 0 0 0 1528 0 0 1528 0 0 0 0 0 0 0 ...
48 13-10-16 15:20 0 0 0 0 0 0 0 0 1302 0 0 1302 0 0 0 0 0 0 0 ...
49 13-10-16 15:30 0 0 0 0 0 0 0 0 987 0 0 987 0 0 0 0 0 0 0 ...
50 13-10-16 15:40 0 0 0 0 0 0 0 0 916 0 0 916 0 0 0 0 0 0 0 ...
51 13-10-16 15:50 0 0 0 0 0 0 0 0 830 0 0 830 0 0 0 0 0 0 0 ...
52 13-10-16 16:00 0 0 0 0 0 0 0 0 746 0 0 746 0 0 0 0 0 0 0 ...
53 13-10-16 16:10 0 0 0 0 0 0 0 0 514 0 0 514 0 0 0 0 0 0 0 ...
54 13-10-16 16:20 0 0 0 0 0 0 0 0 381 0 0 381 0 0 0 0 0 0 0 ...
55 13-10-16 16:30 0 0 0 0 0 0 0 0 274 0 0 274 0 0 0 0 0 0 0 ...
56 13-10-16 16:40 0 0 0 0 0 0 0 0 213 0 0 213 0 0 0 0 0 0 0 ...
57 13-10-16 16:50 0 0 0 0 0 0 0 0 156 0 0 156 0 0 0 0 0 0 0 ...
58 13-10-16 17:00 0 0 0 0 0 0 0 0 76 0 0 76 0 0 0 0 0 0 0 ...
59 13-10-16 17:10 0 0 0 0 0 0 0 0 30 0 0 30 0 0 0 0 0 0 0 ...
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...

33535 rows × 87 columns


In [20]:
df.iloc[-100,2]


Out[20]:
nan

In [21]:
cl = pandas.read_csv(url,sep = ";", header = 6,error_bad_lines= False,na_values = [""],nrows=1)

In [22]:
cl


Out[22]:
KEY => Unnamed: 1 h00tM0Pac001 h00tM0Pac002 h00tM0Pac003 h00tM0Pac004 h00tM0Pac005 h00tM0Pac006 h00tM0Pac007 h00tM0Pac008 h00tM0Pac009 h00tM0Pac010 h00tM0Pac011 h00tM0PacTot h00tMErro001 h00tMErro002 h00tMErro003 h00tMErro004 h00tMErro005 h00tMErro006
0 Date Time Data Data Data Data Data Data Data Data Data Data Data Data Data Data Data Data Data Data ...

1 rows × 87 columns


In [23]:
cols = cl.keys()

In [24]:
cols = cols[1:]

In [25]:
col2 = cols.insert(-1,"NAN")

In [26]:
len(col2)


Out[26]:
87

In [27]:
df.columns = col2

In [28]:
df


Out[28]:
Unnamed: 1 h00tM0Pac001 h00tM0Pac002 h00tM0Pac003 h00tM0Pac004 h00tM0Pac005 h00tM0Pac006 h00tM0Pac007 h00tM0Pac008 h00tM0Pac009 h00tM0Pac010 h00tM0Pac011 h00tM0PacTot h00tMErro001 h00tMErro002 h00tMErro003 h00tMErro004 h00tMErro005 h00tMErro006 h00tMErro007
0 13-10-16 07:20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...
1 13-10-16 07:30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...
2 13-10-16 07:40 0 0 0 0 0 0 0 0 32 0 0 32 0 0 0 0 0 0 0 ...
3 13-10-16 07:50 0 0 0 0 0 0 0 0 139 0 0 139 0 0 0 0 0 0 0 ...
4 13-10-16 08:00 0 0 0 0 0 0 0 0 297 0 0 297 0 0 0 0 0 0 0 ...
5 13-10-16 08:10 0 0 0 0 0 0 0 0 422 0 0 422 0 0 0 0 0 0 0 ...
6 13-10-16 08:20 0 0 0 0 0 0 0 0 650 0 0 650 0 0 0 0 0 0 0 ...
7 13-10-16 08:30 0 0 0 0 0 0 0 0 783 0 0 783 0 0 0 0 0 0 0 ...
8 13-10-16 08:40 0 0 0 0 0 0 0 0 1181 0 0 1181 0 0 0 0 0 0 0 ...
9 13-10-16 08:50 0 0 0 0 0 0 0 0 1217 0 0 1217 0 0 0 0 0 0 0 ...
10 13-10-16 09:00 0 0 0 0 0 0 0 0 1486 0 0 1486 0 0 0 0 0 0 0 ...
11 13-10-16 09:10 0 0 0 0 0 0 0 0 1821 0 0 1821 0 0 0 0 0 0 0 ...
12 13-10-16 09:20 3338 0 0 0 0 0 0 0 0 0 0 3338 0 0 0 0 0 0 0 ...
13 13-10-16 09:30 0 0 0 0 0 0 0 0 3338 0 0 3338 0 0 0 0 0 0 0 ...
14 13-10-16 09:40 0 0 0 0 0 0 0 0 3937 0 0 3937 0 0 0 0 0 0 0 ...
15 13-10-16 09:50 5545 0 0 0 0 0 0 0 0 0 0 5545 0 0 0 0 0 0 0 ...
16 13-10-16 10:00 8112 0 0 0 0 0 0 0 0 0 0 8112 0 0 0 0 0 0 0 ...
17 13-10-16 10:10 0 0 0 0 0 0 0 0 8112 0 0 8112 0 0 0 0 0 0 0 ...
18 13-10-16 10:20 0 0 0 0 0 0 0 0 8646 0 0 8646 0 0 0 0 0 0 0 ...
19 13-10-16 10:30 8337 0 0 0 0 0 0 0 0 0 0 8337 0 0 0 0 0 0 0 ...
20 13-10-16 10:40 9243 0 0 0 0 0 0 0 0 0 0 9243 0 0 0 0 0 0 0 ...
21 13-10-16 10:50 0 0 0 0 0 0 0 0 9243 0 0 9243 0 0 0 0 0 0 0 ...
22 13-10-16 11:00 0 0 0 0 0 0 0 0 3623 0 0 3623 0 0 0 0 0 0 0 ...
23 13-10-16 11:10 0 0 0 0 0 0 0 0 4487 0 0 4487 0 0 0 0 0 0 0 ...
24 13-10-16 11:20 0 0 0 0 0 0 0 0 4201 0 0 4201 0 0 0 0 0 0 0 ...
25 13-10-16 11:30 0 0 0 0 0 0 0 0 3759 0 0 3759 0 0 0 0 0 0 0 ...
26 13-10-16 11:40 0 0 0 0 0 0 0 0 3693 0 0 3693 0 0 0 0 0 0 0 ...
27 13-10-16 11:50 0 0 0 0 0 0 0 0 3501 0 0 3501 0 0 0 0 0 0 0 ...
28 13-10-16 12:00 0 0 0 0 0 0 0 0 5241 0 0 5241 0 0 0 0 0 0 0 ...
29 13-10-16 12:10 9600 0 0 0 0 0 0 0 0 0 0 9600 0 0 0 0 0 0 0 ...
30 13-10-16 12:20 0 0 0 0 0 0 0 0 9600 0 0 9600 0 0 0 0 0 0 0 ...
31 13-10-16 12:30 0 0 0 0 0 0 0 0 8126 0 0 8126 0 0 0 0 0 0 0 ...
32 13-10-16 12:40 0 0 0 0 0 0 0 0 10770 0 0 10770 0 0 0 0 0 0 0 ...
33 13-10-16 12:50 0 0 0 0 0 0 0 0 3107 0 0 3107 0 0 0 0 0 0 0 ...
34 13-10-16 13:00 0 0 0 0 0 0 0 0 2331 0 0 2331 0 0 0 0 0 0 0 ...
35 13-10-16 13:10 0 0 0 0 0 0 0 0 2212 0 0 2212 0 0 0 0 0 0 0 ...
36 13-10-16 13:20 0 0 0 0 0 0 0 0 2557 0 0 2557 0 0 0 0 0 0 0 ...
37 13-10-16 13:30 0 0 0 0 0 0 0 0 4100 0 0 4100 0 0 0 0 0 0 0 ...
38 13-10-16 13:40 0 0 0 0 0 0 0 0 4684 0 0 4684 0 0 0 0 0 0 0 ...
39 13-10-16 13:50 0 0 0 0 0 0 0 0 3176 0 0 3176 0 0 0 0 0 0 0 ...
40 13-10-16 14:00 0 0 0 0 0 0 0 0 6930 0 0 6930 0 0 0 0 0 0 0 ...
41 13-10-16 14:10 2039 0 0 0 0 0 0 0 0 0 0 2039 0 0 0 0 0 0 0 ...
42 13-10-16 14:20 0 0 0 0 0 0 0 0 2039 0 0 2039 0 0 0 0 0 0 0 ...
43 13-10-16 14:30 0 0 0 0 0 0 0 0 2102 0 0 2102 0 0 0 0 0 0 0 ...
44 13-10-16 14:40 0 0 0 0 0 0 0 0 2047 0 0 2047 0 0 0 0 0 0 0 ...
45 13-10-16 14:50 0 0 0 0 0 0 0 0 1771 0 0 1771 0 0 0 0 0 0 0 ...
46 13-10-16 15:00 1528 0 0 0 0 0 0 0 0 0 0 1528 0 0 0 0 0 0 0 ...
47 13-10-16 15:10 0 0 0 0 0 0 0 0 1528 0 0 1528 0 0 0 0 0 0 0 ...
48 13-10-16 15:20 0 0 0 0 0 0 0 0 1302 0 0 1302 0 0 0 0 0 0 0 ...
49 13-10-16 15:30 0 0 0 0 0 0 0 0 987 0 0 987 0 0 0 0 0 0 0 ...
50 13-10-16 15:40 0 0 0 0 0 0 0 0 916 0 0 916 0 0 0 0 0 0 0 ...
51 13-10-16 15:50 0 0 0 0 0 0 0 0 830 0 0 830 0 0 0 0 0 0 0 ...
52 13-10-16 16:00 0 0 0 0 0 0 0 0 746 0 0 746 0 0 0 0 0 0 0 ...
53 13-10-16 16:10 0 0 0 0 0 0 0 0 514 0 0 514 0 0 0 0 0 0 0 ...
54 13-10-16 16:20 0 0 0 0 0 0 0 0 381 0 0 381 0 0 0 0 0 0 0 ...
55 13-10-16 16:30 0 0 0 0 0 0 0 0 274 0 0 274 0 0 0 0 0 0 0 ...
56 13-10-16 16:40 0 0 0 0 0 0 0 0 213 0 0 213 0 0 0 0 0 0 0 ...
57 13-10-16 16:50 0 0 0 0 0 0 0 0 156 0 0 156 0 0 0 0 0 0 0 ...
58 13-10-16 17:00 0 0 0 0 0 0 0 0 76 0 0 76 0 0 0 0 0 0 0 ...
59 13-10-16 17:10 0 0 0 0 0 0 0 0 30 0 0 30 0 0 0 0 0 0 0 ...
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...

33535 rows × 87 columns


In [29]:
import time

In [33]:
time.strftime("%y%m%d%H%M")


Out[33]:
'1406051916'

In [ ]: