notebook.community

Edit and run 





In [1]:



    


library(jsonlite)



    











In [2]:



    


setwd("C:/Users/Johannes/Projects/results/output/20161218")
jsonData <- fromJSON("20161218-logstash.json", flatten = TRUE)



    











In [3]:



    


print(head(jsonData))



    


















    






               _index _type                  _id _score         sort
1 logstash-2016.12.18  logs AVkR139EbpknfgCFxFy4     NA 1.482063e+12
2 logstash-2016.12.18  logs AVkR139NbpknfgCFxFy5     NA 1.482063e+12
3 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyH     NA 1.482063e+12
4 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyG     NA 1.482063e+12
5 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyE     NA 1.482063e+12
6 logstash-2016.12.18  logs AVkR13WMbpknfgCFxFyF     NA 1.482063e+12
  _source.source_host _source.level                _source.created
1       40.68.164.185             6 2016-12-18T12:10:35.454949019Z
2       40.68.164.185             6 2016-12-18T12:10:35.058709599Z
3       40.68.164.185             6 2016-12-18T12:10:35.528087556Z
4       40.68.164.185             6 2016-12-18T12:10:35.528087556Z
5       40.68.164.185             3 2016-12-18T12:10:35.058709599Z
6       40.68.164.185             6 2016-12-18T12:10:35.528087556Z
                                                      _source.message
1                                        worker:ready Worker started.
2                                        worker:ready Worker started.
3                                                                <NA>
4                                                       > node app.js
5 npm info lifecycle elastic-worker@1.0.0~start: elastic-worker@1.0.0
6                           > elastic-worker@1.0.0 start /usr/src/app
  _source.version _source.command _source.tags             _source.image_name
1             1.1       npm start         NULL djbnjack/elastic-worker:latest
2             1.1       npm start         NULL djbnjack/elastic-worker:latest
3             1.1       npm start         NULL djbnjack/elastic-worker:latest
4             1.1       npm start         NULL djbnjack/elastic-worker:latest
5             1.1       npm start         NULL djbnjack/elastic-worker:latest
6             1.1       npm start         NULL djbnjack/elastic-worker:latest
        _source.@timestamp                       _source.container_name
1 2016-12-18T12:10:52.628Z  elastic-workers.7.614ldil5k7w0bahkgkmi3ncx4
2 2016-12-18T12:10:52.486Z elastic-workers.41.b93a0xuk5vbw0540rqu3gjlza
3 2016-12-18T12:10:50.141Z elastic-workers.11.2y9axe7110ubv3j11fpmpya68
4 2016-12-18T12:10:50.140Z elastic-workers.11.2y9axe7110ubv3j11fpmpya68
5 2016-12-18T12:10:50.131Z elastic-workers.41.b93a0xuk5vbw0540rqu3gjlza
6 2016-12-18T12:10:50.131Z elastic-workers.11.2y9axe7110ubv3j11fpmpya68
  _source.host _source.@version _source.tag
1      node-06                1            
2      node-06                1            
3      node-06                1            
4      node-06                1            
5      node-06                1            
6      node-06                1            
                                                         _source.image_id
1 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
2 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
3 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
4 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
5 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
6 sha256:82850a5886f9bde2fef20e81e7e9339d65796066bd246b856375ce8dedffd90b
                                              _source.container_id
1 1c00dc95b1e8fb17b98836d076236c37d6978ffef5b4c207b9824d8aeadde34d
2 db265d4d66b7cd561934b23ae597974d0aa9b28f54c7ff4267d68b4ff439287f
3 e20a08f6bea03a3a90d1589ef89f565b38232f8256e21c09808c57e362ff0055
4 e20a08f6bea03a3a90d1589ef89f565b38232f8256e21c09808c57e362ff0055
5 db265d4d66b7cd561934b23ae597974d0aa9b28f54c7ff4267d68b4ff439287f
6 e20a08f6bea03a3a90d1589ef89f565b38232f8256e21c09808c57e362ff0055
  _source.short_message fields.created fields.@timestamp
1                  <NA>   1.482063e+12      1.482063e+12
2                  <NA>   1.482063e+12      1.482063e+12
3                         1.482063e+12      1.482063e+12
4                  <NA>   1.482063e+12      1.482063e+12
5                  <NA>   1.482063e+12      1.482063e+12
6                  <NA>   1.482063e+12      1.482063e+12

















In [5]:



    


colnames(jsonData)



    


















    







    
	
  1. '_index'
    2. 
	
  2. '_type'
    3. 
	
  3. '_id'
    4. 
	
  4. '_score'
    5. 
	
  5. 'sort'
    6. 
	
  6. '_source.source_host'
    7. 
	
  7. '_source.level'
    8. 
	
  8. '_source.created'
    9. 
	
  9. '_source.message'
    10. 
	
  10. '_source.version'
    11. 
	
  11. '_source.command'
    12. 
	
  12. '_source.tags'
    13. 
	
  13. '_source.image_name'
    14. 
	
  14. '_source.@timestamp'
    15. 
	
  15. '_source.container_name'
    16. 
	
  16. '_source.host'
    17. 
	
  17. '_source.@version'
    18. 
	
  18. '_source.tag'
    19. 
	
  19. '_source.image_id'
    20. 
	
  20. '_source.container_id'
    21. 
	
  21. '_source.short_message'
    22. 
	
  22. 'fields.created'
    23. 
	
  23. 'fields.@timestamp'
    24. 



















In [9]:



    


head(jsonData[,c("_source.created","_source.message")])



    


















    









_source.created_source.message


	
2016-12-18T12:10:35.454949019Z                                     worker:ready Worker started.                                       

	
2016-12-18T12:10:35.058709599Z                                     worker:ready Worker started.                                       

	
2016-12-18T12:10:35.528087556Z                                     NA                                                                 

	
2016-12-18T12:10:35.528087556Z                                                                                > node app.js                                                      

	
2016-12-18T12:10:35.058709599Z                                     npm info lifecycle elastic-worker@1.0.0~start: elastic-worker@1.0.0

	
2016-12-18T12:10:35.528087556Z                                                                                > elastic-worker@1.0.0 start /usr/src/app                          





















In [55]:



    


workerStart <- jsonData[grep("worker:start", jsonData$"_source.message", ignore.case=T),]
head(workerStart[,c("_source.created","_source.host","_source.message")])
cat("N:", nrow(workerStart))



    


















    






Error in `[.data.frame`(workerStart, , c("@timestamp", "_source.host", : undefined columns selected
Traceback:

1. head(workerStart[, c("@timestamp", "_source.host", "_source.message")])
2. workerStart[, c("@timestamp", "_source.host", "_source.message")]
3. `[.data.frame`(workerStart, , c("@timestamp", "_source.host", 
 .     "_source.message"))
4. stop("undefined columns selected")
















In [41]:



    


workerDone <- jsonData[grep("worker:done", jsonData$"_source.message", ignore.case=T),]
head(workerDone[,c("_source.created","_source.host","_source.message")])
cat("N:", nrow(workerDone))



    


















    









_source.created_source.host_source.message


	
41782016-12-18T11:08:00.109593474Z                                                         node-01                                                                                worker:done Finished with task H:CC:S for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
42222016-12-18T11:08:00.254009919Z                                                         node-09                                                                                worker:done Finished with task H:CC:S for workflow e1c24850-c515-11e6-99c6-755b0ed3b6d2

	
42232016-12-18T11:08:00.109593474Z                                                         node-01                                                                                worker:done Finished with task G:CC:M for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
42712016-12-18T11:08:00.254009919Z                                                         node-09                                                                                worker:done Finished with task G:CC:M for workflow e1c24850-c515-11e6-99c6-755b0ed3b6d2

	
43602016-12-18T11:08:00.449816519Z                                                         node-09                                                                                worker:done Finished with task D:CC:M for workflow 08359320-c516-11e6-99c6-755b0ed3b6d2

	
43682016-12-18T11:08:02.211040263Z                                                         node-05                                                                                worker:done Finished with task G:CC:S for workflow 1911fb20-c516-11e6-99c6-755b0ed3b6d2














    






N: 436
















In [50]:



    


workerStart$start <- 1



    











In [51]:



    


workerStart$time <- as.POSIXlt(workerStart$"_source.created", format="%Y-%m-%dT%H:%M:%S", 
                     origin="1970-01-01",tz="GMT")



    











In [54]:



    


workerStart[,c("time","start")]



    


















    









timestart


	
42212016-12-18 11:08:001                  

	
42702016-12-18 11:08:001                  

	
43592016-12-18 11:08:001                  

	
44142016-12-18 11:08:021                  

	
44192016-12-18 11:08:001                  

	
44232016-12-18 11:08:021                  

	
44242016-12-18 11:07:591                  

	
44282016-12-18 11:08:001                  

	
45232016-12-18 11:07:591                  

	
45272016-12-18 11:07:591                  

	
45682016-12-18 11:08:001                  

	
45742016-12-18 11:08:001                  

	
45772016-12-18 11:07:591                  

	
45782016-12-18 11:07:591                  

	
45812016-12-18 11:08:001                  

	
45822016-12-18 11:07:591                  

	
45832016-12-18 11:07:591                  

	
45852016-12-18 11:07:591                  

	
45862016-12-18 11:08:001                  

	
45872016-12-18 11:08:021                  

	
46292016-12-18 11:08:021                  

	
46342016-12-18 11:08:021                  

	
46752016-12-18 11:08:021                  

	
46812016-12-18 11:08:021                  

	
46892016-12-18 11:08:001                  

	
47102016-12-18 11:08:001                  

	
47112016-12-18 11:08:001                  

	
47232016-12-18 11:07:591                  

	
47242016-12-18 11:08:021                  

	
47252016-12-18 11:08:001                  

	
.........

	
101662016-12-18 11:08:001                  

	
101672016-12-18 11:08:001                  

	
102022016-12-18 11:08:001                  

	
102212016-12-18 11:08:011                  

	
102452016-12-18 11:08:001                  

	
102472016-12-18 11:08:001                  

	
102492016-12-18 11:08:021                  

	
102552016-12-18 11:08:001                  

	
102562016-12-18 11:08:001                  

	
102572016-12-18 11:08:001                  

	
102592016-12-18 11:07:591                  

	
103062016-12-18 11:08:001                  

	
103072016-12-18 11:08:001                  

	
103142016-12-18 11:08:001                  

	
103162016-12-18 11:07:591                  

	
103172016-12-18 11:08:001                  

	
103192016-12-18 11:08:001                  

	
103212016-12-18 11:07:591                  

	
103232016-12-18 11:08:021                  

	
103482016-12-18 11:07:591                  

	
103512016-12-18 11:08:021                  

	
103682016-12-18 11:07:591                  

	
103772016-12-18 11:08:001                  

	
103982016-12-18 11:08:001                  

	
103992016-12-18 11:08:001                  

	
104002016-12-18 11:08:021                  

	
104172016-12-18 11:08:001                  

	
104912016-12-18 11:08:001                  

	
104922016-12-18 11:07:591                  

	
105882016-12-18 11:08:021                  





















In [53]:



    


plot(workerStart$time, workerStart$start)



    


















    
























In [ ]:

Content source: elastic-bpm/results

Similar notebooks:

notebook.community | gallery | about