Thursday, December 18, 2014

Using jq to talk with elasticsearch

I've recently found a new tool, quite useful, if you often talk to json services. It's named jq, please do not confuse it with jQuery. Eg. let's take Elasticsearch. I want to know which JVM it runs within. So I need to ask like this:
curl -XGET es1.grey:9200/_cluster/stats
so here's the output:
{"timestamp":1418913590962,"cluster_name":"gc-pb-grey","status":"green","indices":{"count":31,"shards":{"total":252,"primaries":126,"replication":1.0,"index":{"shards":{"min":4,"max":10,"avg":8.129032258064516},"primaries":{"min":2,"max":5,"avg":4.064516129032258},"replication":{"min":1.0,"max":1.0,"avg":1.0}}},"docs":{"count":162713289,"deleted":188051},"store":{"size":"44.2gb","size_in_bytes":47543046975,"throttle_time":"1.9d","throttle_time_in_millis":172301216},"fielddata":{"memory_size":"2.6gb","memory_size_in_bytes":2858909795,"evictions":0},"filter_cache":{"memory_size":"4.8gb","memory_size_in_bytes":5161692177,"evictions":504563},"id_cache":{"memory_size":"0b","memory_size_in_bytes":0},"completion":{"size":"0b","size_in_bytes":0},"segments":{"count":2000,"memory":"2.9gb","memory_in_bytes":3169080575}},"nodes":{"count":{"total":8,"master_only":0,"data_only":0,"master_data":8,"client":0},"versions":["0.90.12"],"os":{"available_processors":32,"mem":{"total":"188.7gb","total_in_bytes":202616012800},"cpu":[{"vendor":"AMD","model":"Opteron","mhz":2599,"total_cores":4,"total_sockets":4,"cores_per_socket":1,"cache_size":"512b","cache_size_in_bytes":512,"count":8}]},"process":{"cpu":{"percent":115},"open_file_descriptors":{"min":1356,"max":1537,"avg":1429}},"jvm":{"max_uptime":"8d","max_uptime_in_millis":693602733,"versions":[{"version":"1.6.0_30","vm_name":"OpenJDK 64-Bit Server VM","vm_version":"23.25-b01","vm_vendor":"Sun Microsystems Inc.","count":8}],"mem":{"heap_used":"59.5gb","heap_used_in_bytes":63949850024,"heap_max":"127.7gb","heap_max_in_bytes":137160032256},"threads":621},"fs":{"total":"94.4gb","total_in_bytes":101461655552,"free":"48.9gb","free_in_bytes":52530081792,"available":"44.1gb","available_in_bytes":47376134144,"disk_reads":570224,"disk_writes":164405664,"disk_io_op":164975888,"disk_read_size":"16.1gb","disk_read_size_in_bytes":17347735552,"disk_write_size":"612.6gb","disk_write_size_in_bytes":657792794624,"disk_io_size":"628.7gb","disk_io_size_in_bytes":675140530176,"disk_queue":"0","disk_service_time":"0.9"},"plugins":[{"name":"bigdesk","description":"No description found for bigdesk.","url":"/_plugin/bigdesk/","jvm":false,"site":true},{"name":"hq","description":"No description found for hq.","url":"/_plugin/hq/","jvm":false,"site":true},{"name":"head","description":"No description found for head.","url":"/_plugin/head/","jvm":false,"site":true}]}}
Quite human unreadable. Let's try now with jq:
curl -XGET es1.grey:9200/_cluster/stats | jq "."
Seems more elegant now:
{
  "nodes": {
    "plugins": [
      {
        "site": true,
        "jvm": false,
        "url": "/_plugin/bigdesk/",
        "description": "No description found for bigdesk.",
        "name": "bigdesk"
      },
      {
        "site": true,
        "jvm": false,
        "url": "/_plugin/hq/",
        "description": "No description found for hq.",
        "name": "hq"
      },
      {
        "site": true,
        "jvm": false,
        "url": "/_plugin/head/",
        "description": "No description found for head.",
        "name": "head"
      }
    ],
    "fs": {
      "disk_service_time": "0.3",
      "disk_writes": 164465520,
      "disk_reads": 570266,
      "available_in_bytes": 47369240576,
      "available": "44.1gb",
      "free_in_bytes": 52523188224,
      "free": "48.9gb",
      "total_in_bytes": 101461655552,
      "total": "94.4gb",
      "disk_io_op": 165035786,
      "disk_read_size": "16.1gb",
      "disk_read_size_in_bytes": 17348456448,
      "disk_write_size": "612.8gb",
      "disk_write_size_in_bytes": 658024804352,
      "disk_io_size": "628.9gb",
      "disk_io_size_in_bytes": 675373260800,
      "disk_queue": "8.3E-5"
    },
    "jvm": {
      "threads": 621,
      "mem": {
        "heap_max_in_bytes": 137160032256,
        "heap_max": "127.7gb",
        "heap_used_in_bytes": 53713885664,
        "heap_used": "50gb"
      },
      "versions": [
        {
          "count": 8,
          "vm_vendor": "Sun Microsystems Inc.",
          "vm_version": "23.25-b01",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "version": "1.6.0_30"
        }
      ],
      "max_uptime_in_millis": 694176092,
      "max_uptime": "8d"
    },
    "process": {
      "open_file_descriptors": {
        "avg": 1432,
        "max": 1546,
        "min": 1355
      },
      "cpu": {
        "percent": 134
      }
    },
    "os": {
      "cpu": [
        {
          "count": 8,
          "vendor": "AMD",
          "model": "Opteron",
          "mhz": 2599,
          "total_cores": 4,
          "total_sockets": 4,
          "cores_per_socket": 1,
          "cache_size": "512b",
          "cache_size_in_bytes": 512
        }
      ],
      "mem": {
        "total_in_bytes": 202616012800,
        "total": "188.7gb"
      },
      "available_processors": 32
    },
    "versions": [
      "0.90.12"
    ],
    "count": {
      "client": 0,
      "master_data": 8,
      "data_only": 0,
      "master_only": 0,
      "total": 8
    }
  },
  "indices": {
    "segments": {
      "memory_in_bytes": 3169089995,
      "memory": "2.9gb",
      "count": 2023
    },
    "count": 31,
    "shards": {
      "index": {
        "replication": {
          "avg": 1,
          "max": 1,
          "min": 1
        },
        "primaries": {
          "avg": 4.064516129032258,
          "max": 5,
          "min": 2
        },
        "shards": {
          "avg": 8.129032258064516,
          "max": 10,
          "min": 4
        }
      },
      "replication": 1,
      "primaries": 126,
      "total": 252
    },
    "docs": {
      "deleted": 188053,
      "count": 162713334
    },
    "store": {
      "throttle_time_in_millis": 172306131,
      "throttle_time": "1.9d",
      "size_in_bytes": 47552519067,
      "size": "44.2gb"
    },
    "fielddata": {
      "evictions": 0,
      "memory_size_in_bytes": 2858939869,
      "memory_size": "2.6gb"
    },
    "filter_cache": {
      "evictions": 504563,
      "memory_size_in_bytes": 5181806342,
      "memory_size": "4.8gb"
    },
    "id_cache": {
      "memory_size_in_bytes": 0,
      "memory_size": "0b"
    },
    "completion": {
      "size_in_bytes": 0,
      "size": "0b"
    }
  },
  "status": "green",
  "cluster_name": "gc-pb-grey",
  "timestamp": 1418914164326
}

Ok, this does not changes much however, it's easy to get json formatted. But try this: curl -XGET es1.grey:9200/_cluster/stats | jq ".nodes.jvm" and we get:
{
  "threads": 619,
  "mem": {
    "heap_max_in_bytes": 137160032256,
    "heap_max": "127.7gb",
    "heap_used_in_bytes": 54125025648,
    "heap_used": "50.4gb"
  },
  "versions": [
    {
      "count": 8,
      "vm_vendor": "Sun Microsystems Inc.",
      "vm_version": "23.25-b01",
      "vm_name": "OpenJDK 64-Bit Server VM",
      "version": "1.6.0_30"
    }
  ],
  "max_uptime_in_millis": 694408309,
  "max_uptime": "8d"
}
I took the outer object by first dot, 'nodes' property, and from inside it 'jvm' property. It is now trivial to get actual jvm version: curl -XGET es1.grey:9200/_cluster/stats | jq ".nodes.jvm.versions[0].version" Tadam:
"1.6.0_30"

Wednesday, January 22, 2014

can your language talk json?

Recently I discovered Python. It's really amazing that the language is that flexible so it can treat json literals as it's usual code. Think about constructing all objects in Java with any JSON library. Is this flexibility specific to dynamic languages? Btw I have found an example that well known and loved ;> C++ also is flexible enough to handle JSON:
std::string json_cxx = { 
  key: "value", 
  number: 53     
}; 
This will compile. But... don't be fooled by compiler letting pass that code! The result is not what javascript developer could expect :/