In [9]:

    

import os
import sys
import subprocess
import re
from astropy.time import Time
from datetime import datetime

#cluster = 'MACSJ2243.3-0935'
#cluster = 'CL0016+16'
cluster = '3C295'

top = '/sps/lsst/dev/lsstprod/clusters'

filters = ['u', 'g', 'r', 'i', 'z']

steps = ['processCcd', 'makeCoaddTempExp', 'assembleCoadd', 'detectCoaddSources', 'mergeCoaddDetections',
         'measureCoaddSources', 'mergeCoaddMeasurements', 'forcedPhotCoadd', 'forcedPhotCcd']
#steps = ['processCcd']


    

In [10]:

    

# number of visits

path = os.path.join(top, cluster)

visits = []
for f in filters :
    cmd = 'wc -l ' + path + '/utils/processCcd/' + f + '.list'
    r = subprocess.check_output(cmd, shell=True).split(' ')[0]
    visits.append(r)

print("Number of visits per filter :", visits)

# Raw data size
cmd = 'du -hs ' + path + '/rawDownload/'
total_size = subprocess.check_output(cmd, shell=True).split('\t')[0]
print('Dataset size : ', total_size)

# calexp
cmd = 'du -hs '+ path + '/output/calexp/'
size = subprocess.check_output(cmd, shell=True).split('\t')[0]
print('Calexp size : ', size)

# deepCoadd
cmd = 'du -hs '+ path + '/output/coadd_dir/deepCoadd/'
size = subprocess.check_output(cmd, shell=True).split('\t')[0]
print('deepCoadd size : ', size)

# deepCoadd-results
cmd = 'du -hs '+ path + '/output/coadd_dir/deepCoadd-results/'
size = subprocess.check_output(cmd, shell=True).split('\t')[0]
print('deepCoadd results size : ', size)

# deepCoadd-results merged
cmd = 'du -hs '+ path + '/output/coadd_dir/deepCoadd-results/merged/'
size = subprocess.check_output(cmd, shell=True).split('\t')[0]
print('deepCoadd results merged size : ', size)


    

    




('Number of visits per filter :', ['25', '17', '21', '20', '13'])
('Dataset size : ', '31G')
('Calexp size : ', '301G')
('deepCoadd size : ', '579G')
('deepCoadd results size : ', '104G')
('deepCoadd results merged size : ', '6,0G')

In [11]:

    

# Time

Time_elaps = []
Time_cpu = []
for task in steps :
    elaps = 0.
    cpu = 0.
    for f in filters :
        if task[0:5] == "merge" :
            path = os.path.join(top, cluster, 'utils', task, 'log')
        else :
            path = os.path.join(top, cluster, 'utils', task, 'log', f)
        log = os.listdir(path)
        for l in log :
            p = os.path.join(path, l)
            fd = open(p, "r")
        
            for line in fd:
                if re.search("JobID:", line):
                    jobId = line.split('JobID')[1].split('*')[0].split(':')[1].strip()
                if re.search("Started on", line):
                    start = Time(datetime.strptime(line.split('Started on:')[1].split('*')[0].strip(), '%c'))
                if re.search("cpu time:", line):
                    c = line.split('cpu time:')[1].split('*')[0].split('/')[0].strip()
                if re.search("Ended on", line):
                    end = Time(datetime.strptime(line.split('Ended on:')[1].split('*')[0].strip(), '%c'))
            diff = end - start
            elaps += diff.sec
            cpu += float(c)
            
            y = str(start).split('-')[0]
            m = str(start).split('-')[1]
#            acc_cmd = 'qacct -f /opt/sge/ccin2p3/common/accounting.' + y + '.' + m + ' -j ' + str(jobId)
#            print acc_cmd
#            r = subprocess.check_output(acc_cmd, shell=True)
#            print r
            
            
    print('%s : %.1f %.1f %.1f')%(task, elaps/3600, cpu/3600, 100*cpu/elaps)
    Time_elaps.append(elaps)
    Time_cpu.append(cpu)
    
print('Total time : %.1f %.1f %.1f')%(sum(Time_elaps)/3600, sum(Time_cpu)/3600, 100*sum(Time_cpu)/sum(Time_elaps))


    

    




processCcd : 52.0 371.7 715.2
makeCoaddTempExp : 44.6 15.3 34.2
assembleCoadd : 51.0 12.8 25.1
detectCoaddSources : 1.6 1.2 74.4
mergeCoaddDetections : 17.5 17.3 98.4
measureCoaddSources : 156.2 215.9 138.2
mergeCoaddMeasurements : 19.4 17.9 92.5
forcedPhotCoadd : 26.0 19.8 76.3
forcedPhotCcd : 179.4 175.0 97.5
Total time : 547.8 846.8 154.6

In [120]:

    

from astropy.time import Time
from datetime import datetime

a = "Wed Aug 10 15:12:39 2016"
b = "Wed Aug 10 15:51:08 2016"
aa = Time(datetime.strptime(a, '%c'))
bb = Time(datetime.strptime(b, '%c'))

delta = bb - aa
print delta.sec
print 100*2738/delta.sec


    

    




2309.0
118.579471633

In [133]:

    

print 'Total time', sum(Time_elaps)/3600, sum(Time_cpu)/3600


    

    




 Total time 1646.0725 1579.80083333

In [20]:

    

print str(start).split('-')[1]


    

    




08

In [ ]: