diff --git a/.gitignore b/.gitignore index b6fb7b6..842f940 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /_build/ -/genspio.byte \ No newline at end of file +/genspio.byte +/.merlin +/.ocamlinit +/genspio-examples.byte +/genspio-test.byte diff --git a/myocamlbuild.ml b/myocamlbuild.ml index 932284d..f47e1ff 100644 --- a/myocamlbuild.ml +++ b/myocamlbuild.ml @@ -44,7 +44,11 @@ let test : Project.item list = Project.app (project_name ^ "-test") ~thread:() ~file:"src/test/main.ml" - ~internal_deps:[lib; test_lib] + ~internal_deps:[lib; test_lib]; + Project.app (project_name ^ "-examples") + ~thread:() + ~file:"src/test/examples.ml" + ~internal_deps:[lib; test_lib]; ] else [] diff --git a/src/lib/EDSL.ml b/src/lib/EDSL.ml index 42e67b7..3dddfde 100644 --- a/src/lib/EDSL.ml +++ b/src/lib/EDSL.ml @@ -1,6 +1,62 @@ type 'a t = 'a Language.t -type cli_option = Language.cli_option +type 'a cli_option = 'a Language.cli_option type 'a option_spec = 'a Language.option_spec type ('a, 'b) cli_options = ('a, 'b) Language.cli_options +let (//) = Filename.concat + include Language.Construct +open Nonstd +module String = Sosa.Native_string + +let case condition body = `Case (condition, seq body) +let default d = `Default (seq d) +let switch l = + let default = ref None in + let cases = + List.filter_map l ~f:(function + | `Default d when !default <> None -> + failwith "Cannot build switch with >1 defaults" + | `Default d -> default := (Some d); None + | `Case t -> Some t) + in + make_switch ~default:(Option.value ~default:nop !default) cases + +let string_concat sl = + (* This is a pretty unefficient implementation: *) + let out s = call [string "printf"; string "%s"; s] in + seq (List.map sl ~f:out) |> output_as_string + +type string_variable = < + get : string Language.t; + set : string Language.t -> unit Language.t; + > +let tmp_file ?(tmp_dir = string "/tmp") name : string_variable = + let path = + let clean = + String.map name ~f:(function + | 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '-' as c -> c + | other -> '_') in + string_concat [ + tmp_dir; + string "/"; + string + (sprintf "genspio-tmp-file-%s-%s" clean Digest.(string name |> to_hex)); + ] + in + let tmp = string_concat [path; string "-tmp"] in + object + method get = output_as_string (call [string "cat"; path]) + method set v = + seq [ + call [string "echo"; string "Setting"]; + call [string "echo"; tmp]; + v >> exec ["cat"] |> write_output ~stdout:tmp; + call [string "mv"; string "-f"; tmp; path]; + ] + end + +let if_seq ~t ?e c = + match e with + | None -> if_then c (seq t) + | Some f -> if_then_else c (seq t) (seq f) diff --git a/src/lib/EDSL.mli b/src/lib/EDSL.mli index 3bd6b6e..b432c8c 100644 --- a/src/lib/EDSL.mli +++ b/src/lib/EDSL.mli @@ -1,13 +1,27 @@ type 'a t = 'a Language.t -type cli_option = Language.cli_option -type 'a option_spec = 'a Language.option_spec -type ('a, 'b) cli_options = ('a, 'b) Language.cli_options val fail: unit t (** Abort the script/command immediately. *) +val call : string t list -> unit t +(** Call a command from its list of “arguments” (including the first + argument being the actual command). + + Note that UNIX does not allow strings passed as arguments to + executables to contain NUL-characters (['\x00']). + The function {!Language.to_many_lines} raises an exception + if an argument is a literal and contains a NUL, but if the + argument is the result of some other expression the behavior is + for now undefined. +*) + val exec : string list -> unit t +(** Like {!call} but with string literals; i.e. [exec ["a"; "b"]] is + actually [call [string "a"; string "b"]] which is the usual shell command + ["a b"] (with proper escaping). *) + + val ( &&& ) : bool t -> bool t -> bool t val ( ||| ) : bool t -> bool t -> bool t val ( =$= ) : string t -> string t -> bool t @@ -23,36 +37,97 @@ val nop : unit t val if_then_else : bool t -> unit t -> unit t -> unit t val if_then : bool t -> unit t -> unit t + val seq : unit t list -> unit t +(** Sequence a list of expressions into an expression. *) + +val if_seq: + t:unit t list -> + ?e:unit t list -> + bool t -> + unit t +(** [if_seq c ~t ~e] is an alternate API for {!if_then_else} (when + [?e] is provided) or {!if_then} (otherwise) that assumes “then” + and “else” bodies to be lists for {!seq} construct. *) + val not : bool t -> bool t -val printf : ('a, unit, string, unit t) format4 -> 'a -val file_exists : string -> bool t + +val file_exists : string t -> bool t + +(** {3 Switch Statements } *) + val switch : - (bool t * unit t) list -> - default:unit t -> unit t + [ `Case of bool t * unit t | `Default of unit t ] list -> unit t +(** Create a switch statement from a list of {!case} and optionally a + {!default} (the function raises an exception if there are more + than one default cases). *) + +val case : + bool t -> + unit t list -> + [> `Case of bool t * unit t ] +(** Create a normal case for a {!switch} statement. *) + +val default : unit t list -> [> `Default of unit t ] +(** Create the default case for a {!switch} statement. *) + + +(**/**) +val make_switch : + (bool Language.t * unit Language.t) list -> + default:unit Language.t -> unit Language.t +(**/**) + + val write_output : - ?stdout:string -> - ?stderr:string -> - ?return_value:string -> unit t -> unit t -val write_stdout : path:string -> unit t -> unit t + ?stdout:string t -> + ?stderr:string t -> + ?return_value:string t -> unit t -> unit t + +val write_stdout : path: string t -> unit t -> unit t + +(** {3 Literals } *) val string : string -> string t val int : int -> int t -val bool : bool t +val bool : bool -> bool t + val output_as_string : unit t -> string t val feed : string:string t -> unit t -> unit t val ( >> ) : string t -> unit t -> unit t val loop_while : bool t -> body:unit t -> unit t +type 'argument_type cli_option = 'argument_type Language.cli_option +type 'argument_type option_spec = 'argument_type Language.option_spec +type ('parse_function, 'return_type) cli_options = ('parse_function, 'return_type) Language.cli_options module Option_list : sig val string : - doc:string -> char -> string t option_spec - val flag : doc:string -> char -> bool t option_spec + ?default: string t -> + doc:string -> char -> + string t option_spec + val flag : + ?default: bool t -> + doc:string -> char -> + bool t option_spec val ( & ) : - 'a option_spec -> - ('b, 'c) cli_options -> ('a -> 'b, 'c) cli_options - val usage : string -> ('a, 'a) cli_options + 'argument_type option_spec -> + ('parse_function, 'return_type) cli_options -> + ('argument_type -> 'parse_function, 'return_type) cli_options + val usage : string -> ('last_return_type, 'last_return_type) cli_options end val parse_command_line : - ('a, unit t) cli_options -> 'a -> unit t + ('parse_function, unit t) cli_options -> 'parse_function -> unit t + +val string_concat: string t list -> string t + +type string_variable = < + get : string Language.t; + set : string Language.t -> unit Language.t; + > +val tmp_file: + ?tmp_dir: string t -> + string -> + string_variable +(** Create a temporary file that may contain arbitrary strings (can be + used as variable containing [string t] values). *) diff --git a/src/lib/language.ml b/src/lib/language.ml index 3d26a03..b3b47ff 100644 --- a/src/lib/language.ml +++ b/src/lib/language.ml @@ -17,22 +17,36 @@ module Literal = struct ); str "'" end |> fst - | Bool true -> "0" - | Bool false -> "1" + | Bool true -> "true" + | Bool false -> "false" + + module String = struct + let easy_to_escape s = + String.for_all s + ~f:(function + | 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '-' | '_' | '*' | '&' | '^' + | '=' | '+' | '%' | '$' | '"' | '\'' | '/' | '#' | '@' | '!' | ' ' + | '~' | '`' | '\\' | '|' | '?' | '>' | '<' | '.' | ',' | ':' | ';' + | '{' | '}' | '(' | ')' | '[' | ']' -> true + | other -> false) + let impossible_to_escape_for_variable = String.exists ~f:((=) '\x00') + end + end -type cli_option = { +type 'a cli_option = { switch: char; doc: string; + default: 'a; } type _ option_spec = - | Opt_flag: cli_option -> bool t option_spec - | Opt_string: cli_option -> string t option_spec + | Opt_flag: bool t cli_option -> bool t option_spec + | Opt_string: string t cli_option -> string t option_spec and (_, _) cli_options = | Opt_end: string -> ('a, 'a) cli_options | Opt_cons: 'c option_spec * ('a, 'b) cli_options -> ('c -> 'a, 'b) cli_options and _ t = - | Exec: string list -> unit t + | Exec: string t list -> unit t | Raw_cmd: string -> unit t | Bool_operator: bool t * [ `And | `Or ] * bool t -> bool t | String_operator: string t * [ `Eq | `Neq ] * string t -> bool t @@ -45,9 +59,9 @@ and _ t = | Output_as_string: unit t -> string t | Write_output: { expr: unit t; - stdout: string option; - stderr: string option; - return_value: string option; + stdout: string t option; + stderr: string t option; + return_value: string t option; } -> unit t | Feed: string t * unit t -> unit t | While: {condition: bool t; body: unit t} -> unit t @@ -58,7 +72,8 @@ and _ t = | Fail: unit t module Construct = struct - let exec l = Exec l + let exec l = Exec (List.map l ~f:(fun s -> Literal (Literal.String s))) + let call l = Exec l let (&&&) a b = Bool_operator (a, `And, b) let (|||) a b = Bool_operator (a, `Or, b) let (=$=) a b = String_operator (a, `Eq, b) @@ -75,15 +90,9 @@ module Construct = struct let not t = Not t - let printf fmt = - ksprintf (fun s -> exec ["printf"; "%s"; s]) fmt - - let file_exists p = - exec ["test"; "-f"; p] |> succeeds - let fail = Fail - let switch: type a. (bool t * unit t) list -> default: unit t -> unit t = + let make_switch: type a. (bool t * unit t) list -> default: unit t -> unit t = fun conds ~default -> List.fold_right conds ~init:default ~f:(fun (x, body) prev -> if_then_else x body prev) @@ -96,7 +105,10 @@ module Construct = struct let literal l = Literal l let string s = Literal.String s |> literal let int s = Literal.Int s |> literal - let bool = Literal.Bool true |> literal + let bool t = Literal.Bool t |> literal + + let file_exists p = + call [string "test"; string "-f"; p] |> succeeds let output_as_string e = Output_as_string e @@ -106,8 +118,10 @@ module Construct = struct let loop_while condition ~body = While {condition; body} module Option_list = struct - let string ~doc switch = Opt_string {switch; doc} - let flag ~doc switch = Opt_flag {switch; doc} + let string ?(default = string "") ~doc switch = + Opt_string {switch; doc; default} + let flag ?(default = bool false) ~doc switch = + Opt_flag {switch; doc; default} let (&) x y = Opt_cons (x, y) let usage s = Opt_end s @@ -132,40 +146,35 @@ let rec to_shell: type a. _ -> a t -> string = sprintf {sh| printf "$(printf '%%s' %s | sed -e 's/\(.\{3\}\)/\\\1/g')" |sh} s in - (* let expand_output_to_string = *) - (* sprintf "\"$(%s)\"" in *) + let to_argument varname = + function + | Literal (Literal.String s) when Literal.String.easy_to_escape s -> + None, Filename.quote s + | Literal (Literal.String s) when + Literal.String.impossible_to_escape_for_variable s -> + ksprintf failwith "to_shell: sorry literal %S is impossible to \ + escape as `exec` argument" s + | v -> + let var = + sprintf "%s=$(%s; printf 'x')" + varname (continue v |> expand_octal) + in + Some var, sprintf "\"${%s%%?}\"" varname + in match e with | Exec l -> - let easy_to_escape = - function - | 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '-' | '_' | '*' | '&' | '^' - | '=' | '+' | '%' | '$' | '"' | '\'' | '/' | '#' | '@' | '!' | ' ' - | '~' | '`' | '\\' | '|' | '?' | '>' | '<' | '.' | ',' | ':' | ';' - | '{' | '}' | '(' | ')' | '[' | ']' -> true - | other -> false in - let impossible_to_escape = String.exists ~f:((=) '\x00') in let variables = ref [] in let args = - List.mapi l ~f:(fun index -> function - | arg when String.for_all arg ~f:easy_to_escape -> - Filename.quote arg - | arg when impossible_to_escape arg -> - ksprintf failwith "to_shell: sorry %S is impossible to escape as \ - `exec` argument" arg - | arg -> - let var, () = - with_buffer begin fun str -> - ksprintf str "argument_%d=$(printf '" index; - String.iter arg ~f:(fun c -> - Char.code c |> sprintf "\\%03o" |> str - ); - str "'; printf 'x') ; " - end in - variables := var :: !variables; - sprintf "\"${argument_%d%%?}\"" index - ) - in - (List.rev !variables) @ args |> String.concat ~sep:" " + List.mapi l ~f:(fun index v -> + let varname = sprintf "argument_%d" index in + match to_argument varname v with + | None, v -> v + | Some vardef, v -> + variables := sprintf "%s ; " vardef :: !variables; + v) in + (List.rev !variables) @ args + |> String.concat ~sep:" " + |> sprintf " { %s ; } " | Raw_cmd s -> s | Returns {expr; value} -> sprintf " { %s ; [ $? -eq %d ] ; }" (continue expr) value @@ -197,13 +206,25 @@ let rec to_shell: type a. _ -> a t -> string = | Not t -> sprintf "! { %s ; }" (continue t) | Write_output { expr; stdout; stderr; return_value } -> - sprintf " ( %s %s ) %s %s" + let make_argument name option = + Option.value_map ~default:(None, None) option ~f:(fun v -> + let v, e = to_argument name v in + v, Some e) in + let varstdout, exprstdout = make_argument "stdoutfile" stdout in + let varstderr, exprstderr = make_argument "stderrfile" stderr in + let varret, exprret = make_argument "retfile" return_value in + let vars = + List.filter_map [varstdout; varstderr; varret] + ~f:(Option.map ~f:(sprintf "export %s ; ")) + |> String.concat ~sep:" " in + sprintf "%s ( %s %s ) %s %s" + vars (continue expr) - (Option.value_map return_value ~default:"" ~f:(fun path -> + (Option.value_map exprret ~default:"" ~f:(fun path -> sprintf "; echo \"$?\" > %s" path)) - (Option.value_map stdout ~default:"" ~f:(fun path -> + (Option.value_map exprstdout ~default:"" ~f:(fun path -> sprintf " > %s" path)) - (Option.value_map stderr ~default:"" ~f:(fun path -> + (Option.value_map exprstderr ~default:"" ~f:(fun path -> sprintf "2> %s" path)) | Literal lit -> Literal.to_shell lit @@ -228,7 +249,7 @@ let rec to_shell: type a. _ -> a t -> string = let string_of_var var = Output_as_string (Raw_cmd (sprintf "printf \"${%s}\"" var)) in let bool_of_var var = - Construct.succeeds (Raw_cmd (sprintf "[ \"${%s}\" -eq 0 ]" var)) in + Construct.succeeds (Raw_cmd (sprintf "{ ${%s} ; } " var)) in let unit_t = let rec loop : type a b. @@ -239,7 +260,8 @@ let rec to_shell: type a. _ -> a t -> string = f | Opt_cons (Opt_string x, more) -> let var = variable x in - to_init (sprintf "export %s= " var); + to_init (sprintf "export %s=$(%s)" + var (continue x.default |> expand_octal)); to_case (sprintf "-%c) %s ;;" x.switch (seq [ @@ -256,11 +278,13 @@ let rec to_shell: type a. _ -> a t -> string = loop (f (string_of_var var)) more | Opt_cons (Opt_flag x, more) -> let var = variable x in - to_init (sprintf "export %s=1 " var); + to_init (sprintf + "export %s=$(if %s ; then printf 'true' ; else printf 'false' ; fi)" var + (continue x.default)); to_case ( sprintf "-%c) %s ;;" x.switch (seq [ - sprintf "export %s=0" var; + sprintf "export %s=true" var; "shift"; ]) ); @@ -279,7 +303,7 @@ let rec to_shell: type a. _ -> a t -> string = [ "while :;"; " do case $1 in"; "-h|-help|--help) "; - sprintf "export %s_help=0 ; " prefix; + sprintf "export %s_help=true ; " prefix; sprintf "%s ;" (continue Construct.(string help_msg >> exec ["cat"])); @@ -305,7 +329,7 @@ let rec to_shell: type a. _ -> a t -> string = in seq ( sprintf "export %s_args=" prefix - :: sprintf "export %s_help=1" prefix + :: sprintf "export %s_help=false" prefix :: List.rev !inits @ [ while_loop; continue Construct.( diff --git a/src/test-lib/test_lib.ml b/src/test-lib/test_lib.ml index 76d2985..f1f112f 100644 --- a/src/test-lib/test_lib.ml +++ b/src/test-lib/test_lib.ml @@ -13,26 +13,38 @@ let babble fmt = else ()) fmt let check_command s ~verifies = - babble "check_command\n %s\n%!" (String.sub s ~index:0 ~length:100 |> Option.value ~default:s); - Pvem_lwt_unix.System.Shell.execute s - >>= fun (out, err, exit_status) -> - List.fold verifies ~init:(return []) ~f:(fun prev_m v -> - prev_m >>= fun prev -> - match v with - | `Exits_with i -> - let l = - if exit_status = `Exited i - then (true, "exited well") :: prev - else ( - false, - sprintf "%s:\nout:\n%s\nerr:\n%s\ncall:\n%s\n" - (Pvem_lwt_unix.System.Shell.status_to_string exit_status) - out - err - s - ) :: prev - in - return l) + babble "check_command (%s)\n %s\n%!" + (List.map ~f:(function `Exits_with n -> sprintf "exits with %d" n) verifies + |> String.concat ~sep:", ") + (String.sub s ~index:0 ~length:300 |> Option.value ~default:s); + begin + Pvem_lwt_unix.System.with_timeout 5. ~f:begin fun () -> + Pvem_lwt_unix.System.Shell.execute s + end + >>< begin function + | `Ok (out, err, exit_status) -> + List.fold verifies ~init:(return []) ~f:(fun prev_m v -> + prev_m >>= fun prev -> + match v with + | `Exits_with i -> + let l = + if exit_status = `Exited i + then (true, "exited well") :: prev + else ( + false, + sprintf "%s (instead of %d):\nout:\n%s\nerr:\n%s\ncall:\n%s\n" + (Pvem_lwt_unix.System.Shell.status_to_string exit_status) + i out err s + ) :: prev + in + return l) + | `Error (`System (`With_timeout _, _)) -> assert false + | `Error (`Shell (_, `Exn e)) -> + return [false, sprintf "Shell EXN : %s" (Printexc.to_string e)] + | `Error (`Timeout _) -> + return [false, sprintf "Timeout !!"] + end + end >>= fun results -> List.filter ~f:(fun (t, _) -> t = false) results |> return diff --git a/src/test/examples.ml b/src/test/examples.ml new file mode 100644 index 0000000..4cc38ab --- /dev/null +++ b/src/test/examples.ml @@ -0,0 +1,214 @@ + +open Nonstd +module String = Sosa.Native_string + +let downloader () = + let open Genspio.EDSL in + let say strings = + let sayone ?(prompt = false) s = + let prompt = + if prompt then "downloader: " else "" in + call (string "printf" :: string (prompt ^ "%s") :: s :: []) in + match strings with + | [] -> nop + | s :: more -> + seq ( + sayone ~prompt:true s :: List.map more ~f:sayone @ [ + sayone (string "\n"); + ] + ) + in + let sayf fmt = ksprintf (fun s -> say [string s]) fmt in + let fail l = seq [say (string "ERROR: " :: l); fail] in + let failf fmt = ksprintf (fun s -> fail [string s]) fmt in + let (//) = Filename.concat in + let silent ~name unit = + object (self) + method stdout = "/tmp" // sprintf "output-of-%s-%s" name "out" |> string + method stderr = "/tmp" // sprintf "output-of-%s-%s" name "err" |> string + method exec = + seq [ + (* say [string "Silent "; string name; self#stdout; self#stderr;]; *) + write_output (seq unit) ~stdout:self#stdout ~stderr:self#stderr; + ] + method succeed_or_fail = + if_seq (self#exec|> succeeds) + ~t:[sayf "%s: Success" name] + ~e:[ + sayf "Expression %s failed!" name; + call [string "cat"; self#stderr]; + failf "Fatal failure of %s" name; + ] + end in + let silence ~name unit = + let s = silent ~name [unit] in + s#exec in + let succeed_in_silence_or_fail ~name units = + let s = silent ~name units in + s#succeed_or_fail in + let download ~url ~output = + let try_help ?(opt = "--help") cmd = + exec [cmd; opt] |> silence ~name:(cmd ^ opt) |> succeeds in + let do_call exec args = [ + sayf "Using `%s`." exec; + succeed_in_silence_or_fail ~name:exec [ + call (string exec :: args); + ] + ] in + switch [ + case (try_help "wget") + (do_call "wget" [url; string "--output-document"; output]); + case (try_help "curl") + (do_call "wget" [url; string "--output-document"; output]); + default [failf "Can't find a downloading application"]; + ] + in + let string_matches_any string regexp_list = + (* Cf. http://pubs.opengroup.org/onlinepubs/009695399/utilities/grep.html *) + let options = List.concat_map regexp_list ~f:(fun r -> ["-e"; r]) in + string >> exec (["grep"; "-q"] @ options) |> succeeds in + + let module Unwrapper = struct + + type cmd = unit t + type t = { + extension: string; + verb: string; + commands: string_variable -> cmd list; + } + let make ~ext ~verb commands = + {extension = ext; verb; commands} + + let remove_suffix v suf = + v >> exec ["sed"; sprintf "s:^\\(.*\\)%s$:\\1:" suf] + |> output_as_string + + let to_switch name_variable t_list = + let make_case t = + case (string_matches_any + name_variable#get [sprintf "\\.%s$" t.extension]) [ + say [ksprintf string "%s: " t.verb; name_variable#get]; + succeed_in_silence_or_fail + ~name:(sprintf "%s-%s" t.verb t.extension) + (t.commands name_variable); + name_variable#set + (remove_suffix name_variable#get (sprintf "\\.%s" t.extension)); + ] in + seq [ + say [string "Extract loop: "; name_variable#get]; + switch (List.map t_list ~f:make_case) + ] + + let to_loop name_variable t_list = + loop_while + (string_matches_any name_variable#get + (List.map t_list (fun t -> sprintf "\\.%s$" t.extension))) + ~body:(to_switch name_variable t_list) + + let all = [ + make ~ext:"gz" ~verb:"Gunzipping" (fun current_name -> [ + call [string "gunzip"; string "-f"; current_name#get]; + ]); + make ~ext:"bz2" ~verb:"Bunzip2-ing" (fun current_name -> [ + call [string "bunzip2"; string "-f"; current_name#get]; + ]); + make ~ext:"zip" ~verb:"Unzipping" (fun current_name -> [ + call [string "unzip"; current_name#get]; + ]); + make ~ext:"tar" ~verb:"Untarring" (fun current_name -> [ + call [string "tar"; string "xf"; current_name#get]; + ]); + make ~ext:"tgz" ~verb:"Untar-gzip-ing" (fun name -> [ + call [string "tar"; string "zxf"; name#get]; + ]); + make ~ext:"tbz2" ~verb:"Untar-bzip2-ing" (fun name -> [ + call [string "tar"; string "xfj"; name#get]; + ]); + make ~ext:"gpg" ~verb:"Decyphering" (fun name -> [ + call [string "gpg"; + string "--output"; + (remove_suffix name#get "\\.gpg"); + string "-d"; name#get;]; + ]); + ] + end in + let no_value = sprintf "none_%x" (Random.int 100_000) |> string in + let cli_spec = + Option_list.( + string + ~doc:"The URL to the stuff" 'u' + ~default:no_value + & flag 'c' ~doc:"Do everything in the temp-dir" + & string 'f' + ~doc:"Override the downloaded file-name" + ~default:no_value + & string 't' + ~doc:"Use