Quick Overview

inserting records

# import from a file (one record per line)
sybil ingest -table my_table < record.json

# import from a mongo DB, making sure to exclude the _id column
mongoexport -collection my_collection | sybil ingest -table my_table -exclude _id

# import from a CSV file
sybil ingest -csv -table my_csv_table < some_csv.csv

# import samples from a subkey of an already existing JSON doc
# the pathname here would be used for the following document:
# {
#    records: [ sample, sample, sample ]
# }
sybil ingest -table my_other_table -path "$.records" < my_json_doc.json

# check out the db file structure
ls -R db/

collating records

# turn the ingest log into column store
sybil digest -table my_collection

querying records

# list tables
sybil query -tables

# query that table
sybil query -table my_table -info
sybil query -table my_table -print

# run a more complicated query (get status, host and histogram of pings)
sybil query -table uptime -group status,host -int ping -print -op hist

# make that a time series JSON blob
sybil query -table uptime -group status,host -int ping -json -op hist -time

# filter the previous query to samples that with host ~= mysite.*
sybil query -table uptime -group status,host -int ping -json -op hist -time -str-filter host:re:mysite.*

# add a time filter for data newer than a week week old
sybil query -table uptime -group status,host -int-filter time:lt:`date --date="-1 week" +%s`

# add a time filter for data between 1 month and 1 week old
sybil query -table uptime -group status,host \
  -int-filter time:lt:`date --date="-1 week" +%s`,time:gt:`date --date="-1 month" +%s`

trimming old records

# list all blocks that won't fit in 100MB (give or take a few MB)
sybil trim -table uptime -time-col time -mb 100

# list all blocks that have no data newer than a week ago
# TIP: other time durations can be used, like: day, month & year
sybil trim -table uptime -time-col time -before `date --date "-1 week" +%s`

# the above two trimming commands can be combined!
# list all blocks that have no data newer than a week and fit within 100mb
sybil trim -table uptime -time-col time -before `date --date "-1 week" +%s` -mb 100

# delete the blocks that won't fit in memory
# TIP: use -really flag if you don't want to be prompted (for use in scripts)
sybil trim -table uptime -time-col time -mb 100 -delete