Merge pull request #196 from ambiata/topic/csv

Sample-extraction skeleton

README.md

@@ -77,6 +77,18 @@ warden schema validate schema.json
 find _warden -name \*.warden -print0 | xargs -0 warden marker failed
 ```
 
+Extracting numeric samples
+--------------------------
+
+Samples can be extracted as CSV files from view markers for easier
+interactive analysis.
+
+```
+warden-sample extract \
+    -o samples.csv \
+    _warden/$view/$data_dates/*/*.json
+```
+
 Future Work
 -----------
 
@@ -91,4 +103,4 @@ Further reading
  - [Documentation](https://github.com/ambiata/warden/tree/master/doc)
    (or find a built version on S3 under
    `s3://ambiata-dispensary-v2/doc/master/warden/`).
- - [Talk slides](https://github.com/ambiata/talks/blob/master/warden-2016-05-20/slides.pdf).
+ - [Talk slides](https://github.com/ambiata/talks/blob/master/warden-2016-05-20/slides.pdf).

ambiata-warden.cabal

@@ -215,6 +215,24 @@ executable warden-anomalies
                      , text                            == 1.2.*
                      , vector                          == 0.10.*
 
+executable warden-sample
+  main-is:             warden-sample.hs
+
+  ghc-options:         -Wall -threaded -O2
+
+  hs-source-dirs:
+                       main
+
+  build-depends:
+                       base                            >= 3          && < 5
+                     , ambiata-p
+                     , ambiata-warden
+                     , ambiata-x-eithert
+                     , ambiata-x-optparse
+                     , optparse-applicative            == 0.11.*
+                     , text                            == 1.2.*
+                     , vector                          == 0.10.*
+
 test-suite test
   type:                exitcode-stdio-1.0
 

main/warden-sample.hs

@@ -0,0 +1,54 @@
+{-# LANGUAGE NoImplicitPrelude #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+import           BuildInfo_ambiata_warden
+import           DependencyInfo_ambiata_warden
+
+import           Options.Applicative (Parser)
+import           Options.Applicative (subparser)
+import           Options.Applicative (long, short, help)
+import           Options.Applicative (metavar, strArgument, strOption)
+
+import           P
+
+import           System.IO (IO, BufferMode(..), FilePath)
+import           System.IO (stdout, stderr, hSetBuffering)
+import           System.IO (hPutStrLn)
+import           System.Exit (exitFailure)
+
+import           X.Options.Applicative (cli, command')
+
+data Command =
+    Extract !FilePath ![FilePath]
+  deriving (Eq, Show)
+
+main :: IO ()
+main = do
+  hSetBuffering stdout LineBuffering
+  hSetBuffering stderr LineBuffering
+  cli "warden-sample" buildInfoVersion dependencyInfo commandP $ \cmd ->
+    case cmd of
+      Extract _op _fs -> do
+        hPutStrLn stderr $ "implement me!"
+        exitFailure
+
+commandP :: Parser Command
+commandP = subparser $
+     command'
+       "extract"
+       "Extract the numeric samples from a set of marker files into a CSV format more suited to interactive data analysis."
+       (Extract <$> csvOutputFileP <*> some markerFileP)
+
+csvOutputFileP :: Parser FilePath
+csvOutputFileP = strOption $
+     metavar "CSV-OUTPUT-FILE"
+  <> long "csv-output-file"
+  <> short 'o'
+  <> help "Path to write CSV output."
+
+-- FIXME: dedupe (also in main warden cli)
+markerFileP :: Parser FilePath
+markerFileP = strArgument $
+     metavar "MARKER-FILE"
+  <> help "Path to view marker file(s) from `warden check`."
+

main/warden.hs

@@ -116,9 +116,9 @@ markerCommandP = subparser $
        (SummarizeMarkers <$> some markerFileP)
 
   <> command'
-    "failed"
-    "Given a list of view marker paths, output all those containing failed checks."
-    (FailedMarkers <$> some markerFileP)
+       "failed"
+       "Given a list of view marker paths, output all those containing failed checks."
+       (FailedMarkers <$> some markerFileP)
 
 schemaCommandP :: Parser Command
 schemaCommandP = subparser $

master.toml

@@ -6,14 +6,14 @@
 [build.profiling-7-10]
   PROFILING = "true"
   PUBLISH = "true"
-  PUBLISH_EXECUTABLES = "warden warden-gen"
+  PUBLISH_EXECUTABLES = "warden warden-gen warden-sample"
   PUBLISH_S3 = "$AMBIATA_ARTEFACTS_PROFILING"
   GHC_VERSION="7.10.2"
   CABAL_VERSION = "1.22.4.0"
 
 [build.dist-7-10]
   PUBLISH = "true"
-  PUBLISH_EXECUTABLES = "warden warden-gen"
+  PUBLISH_EXECUTABLES = "warden warden-gen warden-sample"
   PUBLISH_S3 = "$AMBIATA_ARTEFACTS_MASTER"
   HADDOCK = "true"
   HADDOCK_S3 = "$AMBIATA_HADDOCK_MASTER"
@@ -22,7 +22,7 @@
 
 [build.branches-7-10]
   PUBLISH = "true"
-  PUBLISH_EXECUTABLES = "warden warden-gen"
+  PUBLISH_EXECUTABLES = "warden warden-gen warden-sample"
   PUBLISH_S3 = "$AMBIATA_ARTEFACTS_BRANCHES"
   HADDOCK = "true"
   HADDOCK_S3 = "$AMBIATA_HADDOCK_BRANCHES"

src/Warden/Commands.hs

@@ -3,6 +3,7 @@
 
 module Warden.Commands(
     check
+  , extractNumericFields
   , failedMarkers
   , fileCheck
   , infer
@@ -117,3 +118,11 @@ failedMarkers fs =
     markerFailed mf = do
       vm <- readViewMarker mf
       pure $ or (fmap resultSummaryFailed $ vmCheckResults vm)
+
+extractNumericFields
+  :: FilePath
+  -> [FilePath]
+  -> EitherT WardenError (ResourceT IO) ()
+extractNumericFields _outp _fs =
+  left WardenNotImplementedError
+

test/cli/basic-usage-sample/run

@@ -0,0 +1,21 @@
+#!/bin/sh -eu
+
+. $(dirname $0)/../core/setup.sh
+
+banner Display Version
+#---------------------
+
+$WARDEN_SAMPLE -v | grep -q warden-sample
+
+$WARDEN_SAMPLE --version
+
+banner Display Help
+#------------------
+
+$WARDEN_SAMPLE -h
+
+$WARDEN_SAMPLE --help
+
+WARDEN_SAMPLE="$WARDEN_SAMPLE --dry-run"
+
+$WARDEN_SAMPLE extract -o foo.csv bar.json baz.json

test/cli/core/setup.sh

@@ -1,5 +1,6 @@
 WARDEN=${1:-./dist/build/warden/warden}
 WARDEN_GEN=${2:-./dist/build/warden-gen/warden-gen}
+WARDEN_SAMPLE=${3:-./dist/build/warden-sample/warden-sample}
 export AWS_DEFAULT_REGION="ap-southeast-2"
 export AMBIATA_ENV="test"
 

test/test-cli.hs

@@ -5,4 +5,5 @@ main =
   disorderCliMain [
       "./dist/build/warden/warden"
     , "./dist/build/warden-gen/warden-gen"
+    , "./dist/build/warden-sample/warden-sample"
     ]