notebook.community

Edit and run 





In [1]:



    


library(jsonlite)



    











In [2]:



    


setwd("C:/Users/Johannes/Projects/results/output/20161218")
jsonData <- fromJSON("20161218-logstash.json", flatten = TRUE)



    











In [3]:



    


print(head(jsonData))



    


















    






               _index _type                  _id _score         sort
1 logstash-2016.12.18  logs AVkR139EbpknfgCFxFy4     NA 1.482063e+12
2 logstash-2016.12.18  logs AVkR139NbpknfgCFxFy5     NA 1.482063e+12
3 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyH     NA 1.482063e+12
4 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyG     NA 1.482063e+12
5 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyE     NA 1.482063e+12
6 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyF     NA 1.482063e+12
  _source.source_host _source.level                _source.created
1       40.68.164.185             6 2016-12-18T12:10:35.454949019Z
2       40.68.164.185             6 2016-12-18T12:10:35.058709599Z
3       40.68.164.185             6 2016-12-18T12:10:35.528087556Z
4       40.68.164.185             6 2016-12-18T12:10:35.528087556Z
5       40.68.164.185             3 2016-12-18T12:10:35.058709599Z
6       40.68.164.185             6 2016-12-18T12:10:35.528087556Z
                                                      _source.message
1                                        worker:ready Worker started.
2                                        worker:ready Worker started.
3                                                                <NA>
4                                                       > node app.js
5 npm info lifecycle elastic-worker@1.0.0~start: elastic-worker@1.0.0
6                           > elastic-worker@1.0.0 start /usr/src/app
  _source.version _source.command _source.tags             _source.image_name
1             1.1       npm start         NULL djbnjack/elastic-worker:latest
2             1.1       npm start         NULL djbnjack/elastic-worker:latest
3             1.1       npm start         NULL djbnjack/elastic-worker:latest
4             1.1       npm start         NULL djbnjack/elastic-worker:latest
5             1.1       npm start         NULL djbnjack/elastic-worker:latest
6             1.1       npm start         NULL djbnjack/elastic-worker:latest
        _source.@timestamp                       _source.container_name
1 2016-12-18T12:10:52.628Z  elastic-workers.7.614ldil5k7w0bahkgkmi3ncx4
2 2016-12-18T12:10:52.486Z elastic-workers.41.b93a0xuk5vbw0540rqu3gjlza
3 2016-12-18T12:10:50.141Z elastic-workers.11.2y9axe7110ubv3j11fpmpya68
4 2016-12-18T12:10:50.140Z elastic-workers.11.2y9axe7110ubv3j11fpmpya68
5 2016-12-18T12:10:50.131Z elastic-workers.41.b93a0xuk5vbw0540rqu3gjlza
6 2016-12-18T12:10:50.131Z elastic-workers.11.2y9axe7110ubv3j11fpmpya68
  _source.host _source.@version _source.tag
1      node-06                1            
2      node-06                1            
3      node-06                1            
4      node-06                1            
5      node-06                1            
6      node-06                1            
                                                         _source.image_id
1 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
2 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
3 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
4 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
5 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
6 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
                                              _source.container_id
1 1c00dc95b1e8fb17b98836d076236c37d6978ffef5b4c207b9824d8aeadde34d
2 db265d4d66b7cd561934b23ae597974d0aa9b28f54c7ff4267d68b4ff439287f
3 e20a08f6bea03a3a90d1589ef89f565b38232f8256e21c09808c57e362ff0055
4 e20a08f6bea03a3a90d1589ef89f565b38232f8256e21c09808c57e362ff0055
5 db265d4d66b7cd561934b23ae597974d0aa9b28f54c7ff4267d68b4ff439287f
6 e20a08f6bea03a3a90d1589ef89f565b38232f8256e21c09808c57e362ff0055
  _source.short_message fields.created fields.@timestamp
1                  <NA>   1.482063e+12      1.482063e+12
2                  <NA>   1.482063e+12      1.482063e+12
3                         1.482063e+12      1.482063e+12
4                  <NA>   1.482063e+12      1.482063e+12
5                  <NA>   1.482063e+12      1.482063e+12
6                  <NA>   1.482063e+12      1.482063e+12

















In [5]:



    


colnames(jsonData)



    


















    







    
	
  1. '_index'
    2. 
	
  2. '_type'
    3. 
	
  3. '_id'
    4. 
	
  4. '_score'
    5. 
	
  5. 'sort'
    6. 
	
  6. '_source.source_host'
    7. 
	
  7. '_source.level'
    8. 
	
  8. '_source.created'
    9. 
	
  9. '_source.message'
    10. 
	
  10. '_source.version'
    11. 
	
  11. '_source.command'
    12. 
	
  12. '_source.tags'
    13. 
	
  13. '_source.image_name'
    14. 
	
  14. '_source.@timestamp'
    15. 
	
  15. '_source.container_name'
    16. 
	
  16. '_source.host'
    17. 
	
  17. '_source.@version'
    18. 
	
  18. '_source.tag'
    19. 
	
  19. '_source.image_id'
    20. 
	
  20. '_source.container_id'
    21. 
	
  21. '_source.short_message'
    22. 
	
  22. 'fields.created'
    23. 
	
  23. 'fields.@timestamp'
    24. 



















In [9]:



    


head(jsonData[,c("_source.created","_source.message")])



    


















    









_source.created_source.message


	
2016-12-18T12:10:35.454949019Z                                     worker:ready Worker started.                                       

	
2016-12-18T12:10:35.058709599Z                                     worker:ready Worker started.                                       

	
2016-12-18T12:10:35.528087556Z                                     NA                                                                 

	
2016-12-18T12:10:35.528087556Z                                                                                > node app.js                                                      

	
2016-12-18T12:10:35.058709599Z                                     npm info lifecycle elastic-worker@1.0.0~start: elastic-worker@1.0.0

	
2016-12-18T12:10:35.528087556Z                                                                                > elastic-worker@1.0.0 start /usr/src/app                          





















In [56]:



    


workerStart <- jsonData[grep("worker:start", jsonData$"_source.message", ignore.case=T),]
head(workerStart[,c("fields.@timestamp","_source.host","_source.message")])
cat("N:", nrow(workerStart))



    


















    









fields.@timestamp_source.host_source.message


	
42211.482062e+12                                                                        node-01                                                                             worker:start Executing task H:CC:S for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
42701.482062e+12                                                                        node-09                                                                             worker:start Executing task H:CC:S for workflow e1c24850-c515-11e6-99c6-755b0ed3b6d2

	
43591.482062e+12                                                                        node-01                                                                             worker:start Executing task G:CC:M for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
44141.482062e+12                                                                        node-05                                                                             worker:start Executing task G:CC:S for workflow 1911fb20-c516-11e6-99c6-755b0ed3b6d2

	
44191.482062e+12                                                                        node-10                                                                             worker:start Executing task H:CC:S for workflow cd2fb620-c515-11e6-99c6-755b0ed3b6d2

	
44231.482062e+12                                                                        node-05                                                                             worker:start Executing task F:CI:M for workflow 1911fb20-c516-11e6-99c6-755b0ed3b6d2














    






N: 436
















In [64]:



    


workerDone <- jsonData[grep("worker:done", jsonData$"_source.message", ignore.case=T),]
head(workerDone[,c("fields.@timestamp","_source.host","_source.message")])
cat("N:", nrow(workerDone))



    


















    









fields.@timestamp_source.host_source.message


	
41781.482062e+12                                                                           node-01                                                                                worker:done Finished with task H:CC:S for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
42221.482062e+12                                                                           node-09                                                                                worker:done Finished with task H:CC:S for workflow e1c24850-c515-11e6-99c6-755b0ed3b6d2

	
42231.482062e+12                                                                           node-01                                                                                worker:done Finished with task G:CC:M for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
42711.482062e+12                                                                           node-09                                                                                worker:done Finished with task G:CC:M for workflow e1c24850-c515-11e6-99c6-755b0ed3b6d2

	
43601.482062e+12                                                                           node-09                                                                                worker:done Finished with task D:CC:M for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
43681.482062e+12                                                                           node-05                                                                                worker:done Finished with task G:CC:S for workflow 1911fb20-c516-11e6-99c6-755b0ed3b6d2














    






N: 436
















In [59]:



    


workerStart$start <- 1



    











In [67]:



    


workerStart$time <- as.POSIXct(workerStart$"fields.@timestamp", origin="1970-01-01")



    


















    






Error in as.POSIXct.default(workerStart$"fields.@timestamp", origin = "1970-01-01"): do not know how to convert 'workerStart$"fields.@timestamp"' to class "POSIXct"
Traceback:

1. as.POSIXct(workerStart$"fields.@timestamp", origin = "1970-01-01")
2. as.POSIXct.default(workerStart$"fields.@timestamp", origin = "1970-01-01")
3. stop(gettextf("do not know how to convert '%s' to class %s", 
 .     deparse(substitute(x)), dQuote("POSIXct")), domain = NA)
















In [62]:



    


head(workerStart[,c("fields.@timestamp","start")])



    


















    









fields.@timestampstart


	
42211.482062e+121           

	
42701.482062e+121           

	
43591.482062e+121           

	
44141.482062e+121           

	
44191.482062e+121           

	
44231.482062e+121           

	
44241.482062e+121           

	
44281.482062e+121           

	
45231.482062e+121           

	
45271.482062e+121           

	
45681.482062e+121           

	
45741.482062e+121           

	
45771.482062e+121           

	
45781.482062e+121           

	
45811.482062e+121           

	
45821.482062e+121           

	
45831.482062e+121           

	
45851.482062e+121           

	
45861.482062e+121           

	
45871.482062e+121           

	
46291.482062e+121           

	
46341.482062e+121           

	
46751.482062e+121           

	
46811.482062e+121           

	
46891.482062e+121           

	
47101.482062e+121           

	
47111.482062e+121           

	
47231.482062e+121           

	
47241.482062e+121           

	
47251.482062e+121           

	
.........

	
101661.482059e+121           

	
101671.482059e+121           

	
102021.482059e+121           

	
102211.482059e+121           

	
102451.482059e+121           

	
102471.482059e+121           

	
102491.482059e+121           

	
102551.482059e+121           

	
102561.482059e+121           

	
102571.482059e+121           

	
102591.482059e+121           

	
103061.482059e+121           

	
103071.482059e+121           

	
103141.482059e+121           

	
103161.482059e+121           

	
103171.482059e+121           

	
103191.482059e+121           

	
103211.482059e+121           

	
103231.482059e+121           

	
103481.482059e+121           

	
103511.482059e+121           

	
103681.482059e+121           

	
103771.482059e+121           

	
103981.482059e+121           

	
103991.482059e+121           

	
104001.482059e+121           

	
104171.482059e+121           

	
104911.482059e+121           

	
104921.482059e+121           

	
105881.482059e+121           





















In [65]:



    


plot(workerStart$"fields.@timestamp", workerStart$start)



    


















    
























In [66]:



    


hist(workerStart$"fields.@timestamp")



    


















    






Error in hist.default(workerStart$"fields.@timestamp"): 'x' must be numeric
Traceback:

1. hist(workerStart$"fields.@timestamp")
2. hist.default(workerStart$"fields.@timestamp")
3. stop("'x' must be numeric")
















In [ ]:

Content source: elastic-bpm/results

Similar notebooks:

notebook.community | gallery | about