From d07f4c314881ef06cb11d20961021711d52fedcd Mon Sep 17 00:00:00 2001 From: sgargula Date: Mon, 12 Feb 2018 11:34:27 +0100 Subject: [PATCH 001/205] pretty printing JSON --- cliparser/cliparser.go | 2 ++ converter/converter.go | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index 13437ad..5fd6c1d 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -51,6 +51,7 @@ type CliArguments struct { Sandbox *bool Version *bool Stack *string + PrettyPrint *bool } // Get and validate CLI arguments. Returns error if validation fails. @@ -71,6 +72,7 @@ func ParseCliArguments() (cliArguments CliArguments, err error) { cliArguments.Sandbox = kingpin.Flag("sandbox", "Do not use configuration files hierarchy.").Bool() cliArguments.Version = kingpin.Flag("version", "Print version number together with release name and exit immediately.").Bool() cliArguments.Stack = kingpin.Flag("stack", "An AWS stack name.").String() + cliArguments.PrettyPrint = kingpin.Flag("pretty-print", "Pretty print JSON").Bool() kingpin.Parse() diff --git a/converter/converter.go b/converter/converter.go index 05d90b1..29d8d17 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -19,6 +19,7 @@ package converter import ( + "encoding/json" "errors" "github.com/Appliscale/perun/cliparser" "github.com/Appliscale/perun/context" @@ -46,10 +47,16 @@ func Convert(context *context.Context) error { } if *context.CliArguments.OutputFileFormat == cliparser.JSON { - outputTemplate, err := toJSON(rawTemplate) + var outputTemplate []byte + if *context.CliArguments.PrettyPrint == false { + outputTemplate, err = toJSON(rawTemplate) + } else if *context.CliArguments.PrettyPrint == true { + outputTemplate, err = prettyJSON(rawTemplate) + } if err != nil { return err } + err = saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) if err != nil { return err @@ -71,12 +78,25 @@ func toYAML(jsonTemplate []byte) ([]byte, error) { func toJSON(yamlTemplate []byte) ([]byte, error) { jsonTemplate, error := yaml.YAMLToJSON(yamlTemplate) + if !govalidator.IsJSON(string(jsonTemplate)) { + return nil, errors.New("This is not a valid YAML file") + } + return jsonTemplate, error +} + +func prettyJSON(yamlTemplate []byte) ([]byte, error) { + var yamlObj interface{} + templateError := yaml.Unmarshal(yamlTemplate, &yamlObj) + if templateError != nil { + return nil, errors.New("Incorrect yaml") + } + jsonTemplate, indentError := json.MarshalIndent(yamlObj, "", " ") if !govalidator.IsJSON(string(jsonTemplate)) { return nil, errors.New("This is not a valid YAML file") } + return jsonTemplate, indentError - return jsonTemplate, error } func saveToFile(template []byte, path string, logger *logger.Logger) error { From 9f30d2025f9224377d695ea5fd878c4d8e6e9c3d Mon Sep 17 00:00:00 2001 From: sgargula Date: Mon, 12 Feb 2018 12:14:02 +0100 Subject: [PATCH 002/205] pretty printing new names --- converter/converter.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/converter/converter.go b/converter/converter.go index 29d8d17..4aa4fec 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -49,9 +49,9 @@ func Convert(context *context.Context) error { if *context.CliArguments.OutputFileFormat == cliparser.JSON { var outputTemplate []byte if *context.CliArguments.PrettyPrint == false { - outputTemplate, err = toJSON(rawTemplate) + outputTemplate, err = yamlToJSON(rawTemplate) } else if *context.CliArguments.PrettyPrint == true { - outputTemplate, err = prettyJSON(rawTemplate) + outputTemplate, err = yamlToPrettyJSON(rawTemplate) } if err != nil { return err @@ -76,7 +76,7 @@ func toYAML(jsonTemplate []byte) ([]byte, error) { return yamlTemplate, error } -func toJSON(yamlTemplate []byte) ([]byte, error) { +func yamlToJSON(yamlTemplate []byte) ([]byte, error) { jsonTemplate, error := yaml.YAMLToJSON(yamlTemplate) if !govalidator.IsJSON(string(jsonTemplate)) { return nil, errors.New("This is not a valid YAML file") @@ -84,18 +84,16 @@ func toJSON(yamlTemplate []byte) ([]byte, error) { return jsonTemplate, error } -func prettyJSON(yamlTemplate []byte) ([]byte, error) { - var yamlObj interface{} - templateError := yaml.Unmarshal(yamlTemplate, &yamlObj) - if templateError != nil { - return nil, errors.New("Incorrect yaml") - } - jsonTemplate, indentError := json.MarshalIndent(yamlObj, "", " ") +func yamlToPrettyJSON(yamlTemplate []byte) ([]byte, error) { + var YAMLObj interface{} + templateError := yaml.Unmarshal(yamlTemplate, &YAMLObj) + + jsonTemplate, templateError := json.MarshalIndent(YAMLObj, "", " ") if !govalidator.IsJSON(string(jsonTemplate)) { return nil, errors.New("This is not a valid YAML file") } - return jsonTemplate, indentError + return jsonTemplate, templateError } From 65fb2eee5b6795afeb09a1dc648117c10c5c55b2 Mon Sep 17 00:00:00 2001 From: sgargula Date: Mon, 12 Feb 2018 12:59:51 +0100 Subject: [PATCH 003/205] pretty print in cliparser --- cliparser/cliparser.go | 4 +++- converter/converter.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index 6c682e5..c3c4dd8 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -51,7 +51,7 @@ type CliArguments struct { Region *string Sandbox *bool Stack *string - PrettyPrint *bool + PrettyPrint *string } func availableFormats() []string { @@ -83,6 +83,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() convertOutputFormat = convert.Arg("format", "Output format: "+strings.ToUpper(JSON)+" | "+strings.ToUpper(YAML)+".").HintAction(availableFormats).Required().String() + prettyPrint = convert.Arg("pretty-print", "Pretty printing JSON").String() configure = app.Command(ConfigureMode, "Create your own configuration mode") @@ -113,6 +114,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { cliArguments.TemplatePath = convertTemplate cliArguments.OutputFilePath = convertOutputFile cliArguments.OutputFileFormat = convertOutputFormat + cliArguments.PrettyPrint = prettyPrint // configure case configure.FullCommand(): diff --git a/converter/converter.go b/converter/converter.go index 4aa4fec..67549bf 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -48,9 +48,9 @@ func Convert(context *context.Context) error { if *context.CliArguments.OutputFileFormat == cliparser.JSON { var outputTemplate []byte - if *context.CliArguments.PrettyPrint == false { + if *context.CliArguments.PrettyPrint == "" { outputTemplate, err = yamlToJSON(rawTemplate) - } else if *context.CliArguments.PrettyPrint == true { + } else if *context.CliArguments.PrettyPrint == "pretty-print" { outputTemplate, err = yamlToPrettyJSON(rawTemplate) } if err != nil { From 9d2ba1b405d7d60bdcdaf2f96f33797b94215e22 Mon Sep 17 00:00:00 2001 From: sgargula Date: Mon, 12 Feb 2018 13:39:00 +0100 Subject: [PATCH 004/205] pretty print flag --- cliparser/cliparser.go | 4 ++-- converter/converter.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index c3c4dd8..eaac68c 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -51,7 +51,7 @@ type CliArguments struct { Region *string Sandbox *bool Stack *string - PrettyPrint *string + PrettyPrint *bool } func availableFormats() []string { @@ -83,7 +83,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() convertOutputFormat = convert.Arg("format", "Output format: "+strings.ToUpper(JSON)+" | "+strings.ToUpper(YAML)+".").HintAction(availableFormats).Required().String() - prettyPrint = convert.Arg("pretty-print", "Pretty printing JSON").String() + prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() configure = app.Command(ConfigureMode, "Create your own configuration mode") diff --git a/converter/converter.go b/converter/converter.go index 67549bf..4aa4fec 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -48,9 +48,9 @@ func Convert(context *context.Context) error { if *context.CliArguments.OutputFileFormat == cliparser.JSON { var outputTemplate []byte - if *context.CliArguments.PrettyPrint == "" { + if *context.CliArguments.PrettyPrint == false { outputTemplate, err = yamlToJSON(rawTemplate) - } else if *context.CliArguments.PrettyPrint == "pretty-print" { + } else if *context.CliArguments.PrettyPrint == true { outputTemplate, err = yamlToPrettyJSON(rawTemplate) } if err != nil { From 1dbf98fd9c8565dbdf08d57a38025a16ff67f52c Mon Sep 17 00:00:00 2001 From: sgargula Date: Mon, 12 Feb 2018 16:41:59 +0100 Subject: [PATCH 005/205] file format --- cliparser/cliparser.go | 30 +++++------------------------- converter/converter.go | 37 +++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index eaac68c..3813d70 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -23,7 +23,7 @@ import ( "github.com/Appliscale/perun/logger" "github.com/Appliscale/perun/utilities" "gopkg.in/alecthomas/kingpin.v2" - "strings" + //"strings" ) var ValidateMode = "validate" @@ -33,14 +33,10 @@ var ConfigureMode = "configure" var CreateStackMode = "create-stack" var DestroyStackMode = "delete-stack" -const JSON = "json" -const YAML = "yaml" - type CliArguments struct { Mode *string TemplatePath *string OutputFilePath *string - OutputFileFormat *string ConfigurationPath *string Quiet *bool Yes *bool @@ -54,10 +50,6 @@ type CliArguments struct { PrettyPrint *bool } -func availableFormats() []string { - return []string{JSON, YAML} -} - // Get and validate CLI arguments. Returns error if validation fails. func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { var ( @@ -79,11 +71,10 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { offlineValidate = app.Command(OfflineValidateMode, "Offline Template Validation") offlineValidateTemplate = offlineValidate.Arg("template", "A path to the template file.").Required().String() - convert = app.Command(ConvertMode, "Convertion between JSON and YAML of template files") - convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() - convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() - convertOutputFormat = convert.Arg("format", "Output format: "+strings.ToUpper(JSON)+" | "+strings.ToUpper(YAML)+".").HintAction(availableFormats).Required().String() - prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() + convert = app.Command(ConvertMode, "Convertion between JSON and YAML of template files") + convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() + convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() + prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() configure = app.Command(ConfigureMode, "Create your own configuration mode") @@ -113,7 +104,6 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { cliArguments.Mode = &ConvertMode cliArguments.TemplatePath = convertTemplate cliArguments.OutputFilePath = convertOutputFile - cliArguments.OutputFileFormat = convertOutputFormat cliArguments.PrettyPrint = prettyPrint // configure @@ -121,7 +111,6 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { cliArguments.Mode = &ConfigureMode // create Stack - case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode cliArguments.Stack = createStackName @@ -158,14 +147,5 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { return } - if *cliArguments.Mode == ConvertMode { - *cliArguments.OutputFileFormat = strings.ToLower(*cliArguments.OutputFileFormat) - if *cliArguments.OutputFileFormat != JSON && *cliArguments.OutputFileFormat != YAML { - err = errors.New("Invalid output file format. Use JSON or YAML") - return - } - - } - return } diff --git a/converter/converter.go b/converter/converter.go index 4aa4fec..164a5f7 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -21,7 +21,6 @@ package converter import ( "encoding/json" "errors" - "github.com/Appliscale/perun/cliparser" "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/logger" "github.com/asaskevich/govalidator" @@ -37,17 +36,16 @@ func Convert(context *context.Context) error { if err != nil { return err } + format := chooseformat(rawTemplate) + var outputTemplate []byte - if *context.CliArguments.OutputFileFormat == cliparser.YAML { - outputTemplate, err := toYAML(rawTemplate) + if format == "JSON" { + outputTemplate, err = toYAML(rawTemplate) if err != nil { return err } saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) - } - - if *context.CliArguments.OutputFileFormat == cliparser.JSON { - var outputTemplate []byte + } else if format == "YAML" { if *context.CliArguments.PrettyPrint == false { outputTemplate, err = yamlToJSON(rawTemplate) } else if *context.CliArguments.PrettyPrint == true { @@ -56,11 +54,13 @@ func Convert(context *context.Context) error { if err != nil { return err } - - err = saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) - if err != nil { - return err - } + } else { + context.Logger.Always(format) + return nil + } + err = saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) + if err != nil { + return err } return nil @@ -112,3 +112,16 @@ func saveToFile(template []byte, path string, logger *logger.Logger) error { return nil } + +func chooseformat(rawTemplate []byte) (format string) { + + _, errorYAML := toYAML(rawTemplate) + _, errorJSON := yamlToJSON(rawTemplate) + + if errorYAML == nil { + return "JSON" + } else if errorJSON == nil { + return "YAML" + } + return "Invalid output file format. Use JSON or YAML" +} From 50ec38e4543d310e81e7774c4a125d173d9b0949 Mon Sep 17 00:00:00 2001 From: jlampar Date: Mon, 12 Feb 2018 16:43:16 +0100 Subject: [PATCH 006/205] Conversion - Improvements: transform shorthand syntax intrinsic functions --- converter/converter.go | 19 +++--- intrinsicsolver/elongateForms.go | 40 +++++++++++++ intrinsicsolver/fixFunctions.go | 60 +++++++++++++++---- intrinsicsolver/fixLongFormCorrectness.go | 18 ++++++ intrinsicsolver/fixMultiLineMap.go | 38 +++++++----- intrinsicsolver/intrinsicsolver_test.go | 36 ++++++++++- intrinsicsolver/preprocessed.yml | 7 --- .../manual_test_correctlong.yaml | 1 + .../test_resources/manual_test_elongate.yaml | 1 + .../test_resources/test_elongate.yaml | 1 + offlinevalidator/offlinevalidator.go | 2 +- 11 files changed, 181 insertions(+), 42 deletions(-) create mode 100644 intrinsicsolver/elongateForms.go create mode 100644 intrinsicsolver/fixLongFormCorrectness.go delete mode 100644 intrinsicsolver/preprocessed.yml create mode 100644 intrinsicsolver/test_resources/manual_test_correctlong.yaml create mode 100644 intrinsicsolver/test_resources/manual_test_elongate.yaml create mode 100644 intrinsicsolver/test_resources/test_elongate.yaml diff --git a/converter/converter.go b/converter/converter.go index 05d90b1..85e4526 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -20,13 +20,15 @@ package converter import ( "errors" + "io/ioutil" + "os" + "github.com/Appliscale/perun/cliparser" "github.com/Appliscale/perun/context" + "github.com/Appliscale/perun/intrinsicsolver" "github.com/Appliscale/perun/logger" "github.com/asaskevich/govalidator" "github.com/ghodss/yaml" - "io/ioutil" - "os" ) // Read template from the file, convert it and check if it has valid structure. @@ -46,7 +48,14 @@ func Convert(context *context.Context) error { } if *context.CliArguments.OutputFileFormat == cliparser.JSON { - outputTemplate, err := toJSON(rawTemplate) + if !govalidator.IsJSON(string(rawTemplate)) { + return errors.New("This is not a valid YAML file") + } + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(rawTemplate, context.Logger, "multiline", "elongate", "correctlong") + if preprocessingError != nil { + context.Logger.Error(preprocessingError.Error()) + } + outputTemplate, err := toJSON(preprocessed) if err != nil { return err } @@ -72,10 +81,6 @@ func toYAML(jsonTemplate []byte) ([]byte, error) { func toJSON(yamlTemplate []byte) ([]byte, error) { jsonTemplate, error := yaml.YAMLToJSON(yamlTemplate) - if !govalidator.IsJSON(string(jsonTemplate)) { - return nil, errors.New("This is not a valid YAML file") - } - return jsonTemplate, error } diff --git a/intrinsicsolver/elongateForms.go b/intrinsicsolver/elongateForms.go new file mode 100644 index 0000000..14ec374 --- /dev/null +++ b/intrinsicsolver/elongateForms.go @@ -0,0 +1,40 @@ +package intrinsicsolver + +import ( + "strings" +) + +/* Function elongateForms is investigating for short-form functions and changes them for their long equivalent. */ +func elongateForms(line *string, lines *[]string, idx int, name string) { + var currentFunctions int + pLines := *lines + totalFunctions := strings.Count(*line, "!") + for (currentFunctions != totalFunctions+1) && !strings.Contains(*line, "#!/bin/bash") && strings.Contains(*line, "!") { + short := shortForm(name) + long := longForm(name) + full := fullForm(long) + split := strings.Split(*line, short) + if idx+1 < len(pLines) { + if strings.Contains(*line, name) && strings.Contains(pLines[idx+1], "-") && (len(split) != 2) { + // If so - we don't have to surround it with quotes. + if strings.Contains(*line, short) && !strings.Contains(*line, "|") { + *line = strings.Replace(*line, short, full, -1) + } else if strings.Contains(*line, short) && strings.Contains(*line, "|") { + *line = strings.Replace(*line, (short + " |"), full, -1) + } + } else if strings.Contains(*line, name) { + if strings.Contains(*line, short) && !strings.Contains(*line, "|") { + *line = strings.Replace(*line, short, ("\"" + long + "\":"), -1) + } else if strings.Contains(*line, short) && strings.Contains(*line, "|") { + *line = strings.Replace(*line, (short + " |"), ("\"" + long + "\":"), -1) + } else if strings.Contains(*line, full) && !strings.Contains(*line, "|") { + *line = strings.Replace(*line, full, ("\"" + long + "\":"), -1) + } else if strings.Contains(*line, full) && strings.Contains(*line, "|") { + *line = strings.Replace(*line, (full + " |"), ("\"" + long + "\":"), -1) + } + } + } + currentFunctions++ + } + +} diff --git a/intrinsicsolver/fixFunctions.go b/intrinsicsolver/fixFunctions.go index 2cf8363..9a10fde 100644 --- a/intrinsicsolver/fixFunctions.go +++ b/intrinsicsolver/fixFunctions.go @@ -13,12 +13,16 @@ import ( /* FixFunctions : takes []byte file and firstly converts all single quotation marks to double ones (anything between single ones is treated as the rune in GoLang), -then deconstructs file into lines, checks for intrinsic functions of a map nature where the function name is located in one line and it's body (map elements) +then deconstructs file into lines, checks for intrinsic functions. The FixFunctions has modes: `multiline`, `elongate` and `correctlong`. +Mode `multiline` looks for functions of a map nature where the function name is located in one line and it's body (map elements) are located in the following lines (if this would be not fixed an error would be thrown: `json: unsupported type: map[interface {}]interface {}`). -The function changes the notation by putting function name in the next line with proper indentation and saves the result to temporary file, -then opens it and returns []byte array. +The function changes the notation by putting function name in the next line with proper indentation. +Mode `elongate` exchanges the short function names into their proper, long equivalent. +Mode `correctlong` prepares the file for conversion into JSON. If the file is a YAML with every line being solicitously indented, there is no problem and the `elongate` mode is all we need. +But if there is any mixed notation (e.g. indented maps along with one-line maps, functions in one line with the key), parsing must be preceded with some additional operations. +The result is saved to temporary file, then opened and returned as a []byte array. */ -func FixFunctions(template []byte, logger *logger.Logger) ([]byte, error) { +func FixFunctions(template []byte, logger *logger.Logger, mode ...string) ([]byte, error) { var quotationProcessed, temporaryResult []string preLines, err := parseFileIntoLines(template, logger) @@ -34,27 +38,41 @@ func FixFunctions(template []byte, logger *logger.Logger) ([]byte, error) { quotationProcessed = append(quotationProcessed, fixed) } + // In case the intrinsic function is in the last line and the the next line is investigated in search for it's multi-line body, we have to add one, blank line. + quotationProcessed = append(quotationProcessed, "") + lines := quotationProcessed - // These are the YAML short names of a functions which take the arguments in a form of a map. - multiLiners := []string{"!FindInMap", "!Join", "!Select", "!Split", "!Sub", "!And", "!Equals", "!If", "!Not", "!Or"} + var functions = []string{"Base64", "GetAtt", "GetAZs", "ImportValue", "Ref", "FindInMap", "Join", "Select", "Split", "Sub", "And", "Equals", "If", "Not", "Or"} for idx, d := range lines { - for _, function := range multiLiners { - fixMultiLineMap(&d, &lines, idx, function) + for _, m := range mode { + if m == "multiline" { + for _, function := range functions[5:] { + fixMultiLineMap(&d, &lines, idx, function) + } + } + if m == "elongate" { + for _, function := range functions { + elongateForms(&d, &lines, idx, function) + } + } + if m == "correctlong" { + fixLongFormCorrectness(&d) + } } temporaryResult = append(temporaryResult, d) } // Function writeLines saves the processed result to a file (if there would be any errors, it could be investigated there). - if err := writeLines(temporaryResult, "preprocessed.yml"); err != nil { + if err := writeLines(temporaryResult, ".preprocessed.yml"); err != nil { logger.Error(err.Error()) return nil, err } // Then the temporary result is opened and returned as a []byte. - preprocessedTemplate, err := ioutil.ReadFile("preprocessed.yml") + preprocessedTemplate, err := ioutil.ReadFile(".preprocessed.yml") if err != nil { logger.Error(err.Error()) return preprocessedTemplate, err @@ -63,6 +81,28 @@ func FixFunctions(template []byte, logger *logger.Logger) ([]byte, error) { return preprocessedTemplate, nil } +// Expands the function name to it's long form without a colon. For example - Fn::FindInMap. +func longForm(name string) string { + var fullName string + if name != "Ref" { + fullName = "Fn::" + name + } else { + fullName = name + } + return fullName +} + +/* Expands the function name by adding a colon. For example - Fn::FindInMap:. +It is crucial to pass here the output from the longForm function.*/ +func fullForm(name string) string { + return (name + ":") +} + +// Expands the function name to it's short form. For example - !FindInMap. +func shortForm(name string) string { + return ("!" + name) +} + // Function parseFileIntoLines is reading the []byte file and returns it line by line as []string slice. func parseFileIntoLines(template []byte, logger *logger.Logger) ([]string, error) { bytesReader := bytes.NewReader(template) diff --git a/intrinsicsolver/fixLongFormCorrectness.go b/intrinsicsolver/fixLongFormCorrectness.go new file mode 100644 index 0000000..4e327bc --- /dev/null +++ b/intrinsicsolver/fixLongFormCorrectness.go @@ -0,0 +1,18 @@ +package intrinsicsolver + +import ( + "strings" +) + +/* Unfortunately the short-to-long-form function names exchange isn't solving the issue of YAML being ready for the YAML-JSON conversion. +In some cases the parser is misinterpretating function in it's long form with additional key and throws an error. We must enclose functions in curly braces. */ +func fixLongFormCorrectness(line *string) { + keyValue := strings.SplitAfterN(*line, ":", 2) + if len(keyValue) == 2 && !strings.Contains(keyValue[0], "Fn:") { + if strings.Contains(keyValue[1], "\"Fn::") && !strings.Contains(keyValue[0], "Fn") { + *line = strings.Replace(*line, keyValue[1], (" {" + keyValue[1] + "}"), 1) + } else if strings.Contains(keyValue[1], "\"Ref") && !strings.Contains(keyValue[0], "Ref") { + *line = strings.Replace(*line, keyValue[1], (" {" + keyValue[1] + "}"), 1) + } + } +} diff --git a/intrinsicsolver/fixMultiLineMap.go b/intrinsicsolver/fixMultiLineMap.go index d0f88a8..fe30572 100644 --- a/intrinsicsolver/fixMultiLineMap.go +++ b/intrinsicsolver/fixMultiLineMap.go @@ -4,23 +4,31 @@ import ( "strings" ) -// Function fixMultiLineMap detects if a function is of a multi-line map nature by checking what follows the function name. At the moment the goformation library is inappropriately handling the case where the function name is in the same line as the key and the body of a function isn't in the same line. There are many ways to solve this problem but the fastest is to move the function name to the next line, indent it and transform it to it's full name. Other solutions include rewriting the whole function and it's body in one line but due the lack of knowledge of how nested the map internal structure is and where it ends, this solution is not chosen. +/* Function fixMultiLineMap detects if a function is of a multi-line map nature by checking what follows the function name. +At the moment the goformation library is inappropriately handling the case where the function name is in the same line as the key and the body of a function isn't in the same line. +There are many ways to solve this problem but the fastest is to move the function name to the next line, indent it and transform it to it's full name. +Other solutions include rewriting the whole function and it's body in one line but due the lack of knowledge of how nested the map internal structure is and where it ends, +this solution is not chosen. */ func fixMultiLineMap(line *string, lines *[]string, idx int, name string) { pLines := *lines - longName := "Fn::" + strings.Split(name, "!")[1] + ":" - if strings.Contains(*line, name) && !strings.Contains(*line, "|") { - split := strings.Split(*line, name) - if strings.Contains(pLines[idx+1], "-") && split[1] == "" { - // If so - we have multiple-level function with a body created of a map elements as the hyphen-noted structures. - if strings.Contains(*line, ":") { - // If so - we have key and a function name in one line. We have to relocate the function name into the next line, indent it and change it to the long form. - nextLineIndents := indentations(pLines[idx+1]) - fullIndents := strings.Repeat(" ", nextLineIndents) - replacement := "\n" + fullIndents + longName - *line = strings.Replace(*line, name, replacement, -1) - } else { - // If so - we have function as the element of another map - we assume that it is well indented so we only change the form to the long one. - *line = strings.Replace(*line, name, longName, -1) + short := shortForm(name) + long := longForm(name) + full := fullForm(long) + if strings.Contains(*line, short) && !strings.Contains(*line, "|") { + split := strings.Split(*line, short) + if idx+1 < len(pLines) { + if strings.Contains(pLines[idx+1], "-") && (len(split) == 1 || split[1] == "") { + // If so - we have multiple-level function with a body created of a map elements as the hyphen-noted structures. + if strings.Contains(*line, ":") { + // If so - we have key and a function name in one line. We have to relocate the function name into the next line, indent it and change it to the long form. + nextLineIndents := indentations(pLines[idx+1]) + fullIndents := strings.Repeat(" ", nextLineIndents) + replacement := "\n" + fullIndents + full + *line = strings.Replace(*line, short, replacement, -1) + } else { + // If so - we have function as the element of another map - we assume that it is well indented so we only change the form to the long one. + *line = strings.Replace(*line, short, full, -1) + } } } } diff --git a/intrinsicsolver/intrinsicsolver_test.go b/intrinsicsolver/intrinsicsolver_test.go index b892058..0f309da 100644 --- a/intrinsicsolver/intrinsicsolver_test.go +++ b/intrinsicsolver/intrinsicsolver_test.go @@ -45,12 +45,44 @@ func TestIndentations(t *testing.T) { assert.Equal(t, "K", firstLetter, "MSG") } -func TestFixFunctions(t *testing.T) { +func TestMultiline(t *testing.T) { rawTemplate, _ := ioutil.ReadFile("./test_resources/test_map.yaml") expectedTemplate, _ := ioutil.ReadFile("./test_resources/manual_test_map.yaml") - fixed, _ := FixFunctions(rawTemplate, &sink) + fixed, _ := FixFunctions(rawTemplate, &sink, "multiline") expected, _ := parseFileIntoLines(expectedTemplate, &sink) actual, _ := parseFileIntoLines(fixed, &sink) + if string(actual[len(actual)-1]) == "" { + actual = actual[:(len(actual) - 1)] + } + + assert.Equal(t, expected, actual, "MSG") +} + +func TestElongate(t *testing.T) { + rawTemplate, _ := ioutil.ReadFile("./test_resources/test_elongate.yaml") + expectedTemplate, _ := ioutil.ReadFile("./test_resources/manual_test_elongate.yaml") + fixed, _ := FixFunctions(rawTemplate, &sink, "elongate") + expected, _ := parseFileIntoLines(expectedTemplate, &sink) + actual, _ := parseFileIntoLines(fixed, &sink) + + if string(actual[len(actual)-1]) == "" { + actual = actual[:(len(actual) - 1)] + } + + assert.Equal(t, expected, actual, "MSG") +} + +func TestCorrectLong(t *testing.T) { + rawTemplate, _ := ioutil.ReadFile("./test_resources/manual_test_elongate.yaml") + expectedTemplate, _ := ioutil.ReadFile("./test_resources/manual_test_correctlong.yaml") + fixed, _ := FixFunctions(rawTemplate, &sink, "correctlong") + expected, _ := parseFileIntoLines(expectedTemplate, &sink) + actual, _ := parseFileIntoLines(fixed, &sink) + + if string(actual[len(actual)-1]) == "" { + actual = actual[:(len(actual) - 1)] + } + assert.Equal(t, expected, actual, "MSG") } diff --git a/intrinsicsolver/preprocessed.yml b/intrinsicsolver/preprocessed.yml deleted file mode 100644 index 58f1834..0000000 --- a/intrinsicsolver/preprocessed.yml +++ /dev/null @@ -1,7 +0,0 @@ -Key: - Fn::Equals: - - "value_1" - - Fn::FindInMap: - - MapName - - TopLevelKey - - SecondLevelKey diff --git a/intrinsicsolver/test_resources/manual_test_correctlong.yaml b/intrinsicsolver/test_resources/manual_test_correctlong.yaml new file mode 100644 index 0000000..451ac50 --- /dev/null +++ b/intrinsicsolver/test_resources/manual_test_correctlong.yaml @@ -0,0 +1 @@ +Key: { "Fn::Equals": [ value_1, "Fn::FindInMap": [ MapName, "Ref": TopLevelKeyRef, SecondLevelKey ] ]} \ No newline at end of file diff --git a/intrinsicsolver/test_resources/manual_test_elongate.yaml b/intrinsicsolver/test_resources/manual_test_elongate.yaml new file mode 100644 index 0000000..88792c5 --- /dev/null +++ b/intrinsicsolver/test_resources/manual_test_elongate.yaml @@ -0,0 +1 @@ +Key: "Fn::Equals": [ value_1, "Fn::FindInMap": [ MapName, "Ref": TopLevelKeyRef, SecondLevelKey ] ] \ No newline at end of file diff --git a/intrinsicsolver/test_resources/test_elongate.yaml b/intrinsicsolver/test_resources/test_elongate.yaml new file mode 100644 index 0000000..7afcca6 --- /dev/null +++ b/intrinsicsolver/test_resources/test_elongate.yaml @@ -0,0 +1 @@ +Key: !Equals [ value_1, !FindInMap [ MapName, !Ref TopLevelKeyRef, SecondLevelKey ] ] \ No newline at end of file diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index da88809..9c53237 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -145,7 +145,7 @@ func parseYAML(templateFile []byte, refTemplate template.Template, logger *logge return template, err } - preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger) + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline") if preprocessingError != nil { logger.Error(preprocessingError.Error()) } From f440ad0a883709cde5bc8bcce8307f3a840219a2 Mon Sep 17 00:00:00 2001 From: jlampar Date: Mon, 12 Feb 2018 17:48:04 +0100 Subject: [PATCH 007/205] fixed converter issues --- intrinsicsolver/elongateForms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intrinsicsolver/elongateForms.go b/intrinsicsolver/elongateForms.go index 14ec374..80489b0 100644 --- a/intrinsicsolver/elongateForms.go +++ b/intrinsicsolver/elongateForms.go @@ -9,7 +9,7 @@ func elongateForms(line *string, lines *[]string, idx int, name string) { var currentFunctions int pLines := *lines totalFunctions := strings.Count(*line, "!") - for (currentFunctions != totalFunctions+1) && !strings.Contains(*line, "#!/bin/bash") && strings.Contains(*line, "!") { + for (currentFunctions != totalFunctions+1) && !strings.Contains(*line, "#!") && strings.Contains(*line, "!") { short := shortForm(name) long := longForm(name) full := fullForm(long) From 7f0d5eefda8aa0f175dd257949b82cf7f134d2b7 Mon Sep 17 00:00:00 2001 From: jlampar Date: Mon, 12 Feb 2018 18:01:34 +0100 Subject: [PATCH 008/205] fixed converter --- intrinsicsolver/fixFunctions.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/intrinsicsolver/fixFunctions.go b/intrinsicsolver/fixFunctions.go index 9a10fde..6317ec5 100644 --- a/intrinsicsolver/fixFunctions.go +++ b/intrinsicsolver/fixFunctions.go @@ -11,6 +11,9 @@ import ( "github.com/Appliscale/perun/logger" ) +var functions = []string{"Base64", "GetAtt", "GetAZs", "ImportValue", "Ref", "FindInMap", "Join", "Select", "Split", "Sub", "And", "Equals", "If", "Not", "Or"} +var mapNature = functions[5:] + /* FixFunctions : takes []byte file and firstly converts all single quotation marks to double ones (anything between single ones is treated as the rune in GoLang), then deconstructs file into lines, checks for intrinsic functions. The FixFunctions has modes: `multiline`, `elongate` and `correctlong`. @@ -43,12 +46,10 @@ func FixFunctions(template []byte, logger *logger.Logger, mode ...string) ([]byt lines := quotationProcessed - var functions = []string{"Base64", "GetAtt", "GetAZs", "ImportValue", "Ref", "FindInMap", "Join", "Select", "Split", "Sub", "And", "Equals", "If", "Not", "Or"} - for idx, d := range lines { for _, m := range mode { if m == "multiline" { - for _, function := range functions[5:] { + for _, function := range mapNature { fixMultiLineMap(&d, &lines, idx, function) } } From de3a4e3a984ea1e7bfc739e81677d67b8e9debd7 Mon Sep 17 00:00:00 2001 From: sgargula Date: Tue, 13 Feb 2018 09:47:45 +0100 Subject: [PATCH 009/205] auto-detection of input file format --- cliparser/cliparser.go | 1 - cliparser/cliparser_test.go | 5 ----- converter/converter.go | 6 +++--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index 3813d70..5f3094d 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -23,7 +23,6 @@ import ( "github.com/Appliscale/perun/logger" "github.com/Appliscale/perun/utilities" "gopkg.in/alecthomas/kingpin.v2" - //"strings" ) var ValidateMode = "validate" diff --git a/cliparser/cliparser_test.go b/cliparser/cliparser_test.go index 8d1c84f..ce6afd6 100644 --- a/cliparser/cliparser_test.go +++ b/cliparser/cliparser_test.go @@ -21,11 +21,6 @@ import ( "testing" ) -func TestInvalidOutputFormatInConvertMode(t *testing.T) { - assert.Equal(t, "Invalid output file format. Use JSON or YAML", - parseCliArguments([]string{"cmd", "convert", "some_path", "some_path", "wrong_format"}).Error()) -} - func TestInvalidVerbosity(t *testing.T) { assert.Equal(t, "You specified invalid value for --verbosity flag", parseCliArguments([]string{"cmd", "validate", "some_path", "--verbosity=TEST"}).Error()) diff --git a/converter/converter.go b/converter/converter.go index 164a5f7..80a9568 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -36,7 +36,7 @@ func Convert(context *context.Context) error { if err != nil { return err } - format := chooseformat(rawTemplate) + format := detectFormatFromContent(rawTemplate) var outputTemplate []byte if format == "JSON" { @@ -113,7 +113,7 @@ func saveToFile(template []byte, path string, logger *logger.Logger) error { return nil } -func chooseformat(rawTemplate []byte) (format string) { +func detectFormatFromContent(rawTemplate []byte) (format string) { _, errorYAML := toYAML(rawTemplate) _, errorJSON := yamlToJSON(rawTemplate) @@ -123,5 +123,5 @@ func chooseformat(rawTemplate []byte) (format string) { } else if errorJSON == nil { return "YAML" } - return "Invalid output file format. Use JSON or YAML" + return "Unsupported file format. The input file must be either a valid _JSON_ or _YAML_ file." } From 037b3a1b11f5b765cc69852facb414fc5a450aa8 Mon Sep 17 00:00:00 2001 From: sgargula Date: Tue, 13 Feb 2018 12:46:44 +0100 Subject: [PATCH 010/205] format type file and fixFunctions --- .preprocessed.yml | 168 ++++++++++++++++++++++++++++++ converter/converter.go | 22 ++-- intrinsicsolver/.preprocessed.yml | 2 + 3 files changed, 179 insertions(+), 13 deletions(-) create mode 100644 .preprocessed.yml create mode 100644 intrinsicsolver/.preprocessed.yml diff --git a/.preprocessed.yml b/.preprocessed.yml new file mode 100644 index 0000000..7e16f0b --- /dev/null +++ b/.preprocessed.yml @@ -0,0 +1,168 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Description: > + HB S2S Creatives Scripts stack which creates all necessary dependencies for that, based on environment. Including + external Elastic IP needed for production system, for communication with 3rd party used for creatives scanning done + on our side. + +Parameters: + + ENV: + Description: > + This points out which region and stage should be used - we call it an environment. + Type: String + Default: eu-central-1.dev + AllowedValues: + - eu-central-1.dev + - eu-central-1.qa + - eu-central-1.prod + - ap-southeast-1.prod + - us-east-1.prod + - us-west-2.prod + ConstraintDescription: > + Must be one of the defined allowed values. + +Mappings: + + EnvSettings: + eu-central-1.dev: + SecurityGroupId: sg-b30848da # private + SubnetIds: + - subnet-74c0a61d # eu-central-1a, private + - subnet-db7cf8a0 # eu-central-1b, private + HostedZone: dev.adtech.aolcloud.net + AmiId: ami-183a9e77 + Stage: dev + Capacity: 1 + OrbProjectId: "68748238" + + eu-central-1.qa: + SecurityGroupId: sg-4b074722 # private + SubnetIds: + - subnet-8bc2a4e2 # eu-central-1a, private + - subnet-0873f773 # eu-central-1b, private + HostedZone: qa.adtech.aolcloud.net + AmiId: ami-d8389cb7 + Stage: qa + Capacity: 1 + OrbProjectId: "68748252" + + eu-central-1.prod: + SecurityGroupId: sg-5af3b233 # public + SubnetIds: + - subnet-e06ce89b # eu-central-1b, public + - subnet-10f39579 # eu-central-1a, public + HostedZone: prod.adtech.aolcloud.net + AmiId: ami-673f9208 + Stage: prod + Capacity: 2 + OrbProjectId: "68748228" + + ap-southeast-1.prod: + SecurityGroupId: sg-672bc601 # public + SubnetIds: + - subnet-91a8a2e7 # ap-southeast-1a, public + - subnet-54907133 # ap-southeast-1b, public + HostedZone: prod.adtech.aolcloud.net + AmiId: ami-fd196f9e + Stage: prod + Capacity: 2 + OrbProjectId: "68748228" + + us-east-1.prod: + SecurityGroupId: sg-d4cbd4af # public + SubnetIds: + - subnet-56b18e0e # us-east-1a, public + - subnet-3d291617 # us-east-1c, public + HostedZone: prod.adtech.aolcloud.net + AmiId: ami-c81935de + Stage: prod + Capacity: 2 + OrbProjectId: "68748228" + + us-west-2.prod: + SecurityGroupId: sg-1f2e7f79 # public + SubnetIds: + - subnet-adc286c9 # us-west-2a, public + - subnet-fa30ac8c # us-west-2b, public + HostedZone: prod.adtech.aolcloud.net + AmiId: ami-7b8c8602 + Stage: prod + Capacity: 2 + OrbProjectId: "68748228" + +Conditions: + OnlyProduction: { "Fn::Equals": [ prod, "Fn::FindInMap": [ EnvSettings, "Ref": ENV, Stage ] ]} + +Resources: + + # We need 2 ElasticIPs and it will require manual + # association after creating it. No scripting for it. + + YumRepositoryAccessRole: + Type: AWS::IAM::Role + Properties: + RoleName: + Fn::Sub: + - "adtech-s2s-yum-access-rtb-cs-${AWS::Region}-${Stage}" + - Stage: { "Fn::FindInMap": [ EnvSettings, "Ref": ENV, Stage ]} + Policies: + - PolicyName: + Fn::Sub: + - "adtech-s2s-yum-access-policy-rtb-cs-${AWS::Region}-${Stage}" + - Stage: { "Fn::FindInMap": [ EnvSettings, "Ref": ENV, Stage ]} + PolicyDocument: + Version: "2012-10-17" + AvailabilityZone: + Fn::Select: + - "Fn::FindInMap": [ RegionMap, "Ref": "AWS::Region", 32 ] + - Fn::GetAZs: + Ref: "AWS::Region" + Statement: + - Action: + - s3:ListAllMyBuckets + Effect: Allow + Resource: + - arn:aws:s3:::* + - Action: + - s3:ListBucket + - s3:GetBucketLocation + Effect: Allow + Resource: + - Fn::Sub: + - "arn:aws:s3:::${BucketName}" + - BucketName: { "Fn::ImportValue": "adtech-s2s-rtb-yum-repository-bucket-name"} + - Action: + - s3:* + Effect: Allow + Resource: + - Fn::Sub: + - "arn:aws:s3:::${BucketName}/*" + - BucketName: { "Fn::ImportValue": "adtech-s2s-rtb-yum-repository-bucket-name"} + + CreativesScriptsLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + DependsOn: + - ReadAccessToYumRepository + Properties: + InstanceType: m4.xlarge + InstanceMonitoring: true + IamInstanceProfile: { "Fn::GetAtt": "ReadAccessToYumRepository.Arn"} + ImageId: { "Fn::FindInMap": [ EnvSettings, "Ref": ENV, AmiId ]} + SecurityGroups: + - "Fn::FindInMap": [ EnvSettings, "Ref": ENV, SecurityGroupId ] + EbsOptimized: true + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: 30 + UserData: + "Fn::Base64": + "Fn::Sub": + #!/bin/bash -xe + /usr/bin/cfn-init -v --stack ${AWS::StackName} --resource CreativesScriptsLaunchConfiguration --region ${AWS::Region} + /usr/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource CreativesScriptsAutoScalingGroup --region ${AWS::Region} + + +Outputs: {} + diff --git a/converter/converter.go b/converter/converter.go index 5e8ef41..8e15345 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -21,14 +21,13 @@ package converter import ( "encoding/json" "errors" - "io/ioutil" - "os" - "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/intrinsicsolver" "github.com/Appliscale/perun/logger" "github.com/asaskevich/govalidator" "github.com/ghodss/yaml" + "io/ioutil" + "os" ) // Read template from the file, convert it and check if it has valid structure. @@ -41,16 +40,16 @@ func Convert(context *context.Context) error { format := detectFormatFromContent(rawTemplate) var outputTemplate []byte + // If input type file is JSON convert to YAML. if format == "JSON" { outputTemplate, err = toYAML(rawTemplate) if err != nil { return err } saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) + + // If input type file is YAML, check all functions and create JSON (with or not --pretty-print flag). } else if format == "YAML" { - if !govalidator.IsJSON(string(rawTemplate)) { - return errors.New("This is not a valid YAML file") - } preprocessed, preprocessingError := intrinsicsolver.FixFunctions(rawTemplate, context.Logger, "multiline", "elongate", "correctlong") if preprocessingError != nil { context.Logger.Error(preprocessingError.Error()) @@ -63,14 +62,14 @@ func Convert(context *context.Context) error { if err != nil { return err } + err = saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) + if err != nil { + return err + } } else { context.Logger.Always(format) return nil } - err = saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) - if err != nil { - return err - } return nil } @@ -96,9 +95,6 @@ func yamlToPrettyJSON(yamlTemplate []byte) ([]byte, error) { jsonTemplate, templateError := json.MarshalIndent(YAMLObj, "", " ") - if !govalidator.IsJSON(string(jsonTemplate)) { - return nil, errors.New("This is not a valid YAML file") - } return jsonTemplate, templateError } diff --git a/intrinsicsolver/.preprocessed.yml b/intrinsicsolver/.preprocessed.yml new file mode 100644 index 0000000..32a417d --- /dev/null +++ b/intrinsicsolver/.preprocessed.yml @@ -0,0 +1,2 @@ +Key: { "Fn::Equals": [ value_1, "Fn::FindInMap": [ MapName, "Ref": TopLevelKeyRef, SecondLevelKey ] ]} + From ece2e8a670bea81544496445144ae0cd3e89224c Mon Sep 17 00:00:00 2001 From: Sylwia <30501500+SylwiaGargula@users.noreply.github.com> Date: Tue, 13 Feb 2018 13:08:51 +0100 Subject: [PATCH 011/205] Auto detection of file format (#76) --- .gitignore | 2 +- cliparser/cliparser.go | 29 +++---------------- cliparser/cliparser_test.go | 5 ---- converter/converter.go | 58 +++++++++++++++++++++---------------- 4 files changed, 38 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 8a4399d..f2ba980 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ perun .perun # This is a temporary file which is being created when FixFunctions is called from intrinsicsolver package. -preprocessed.yml +.preprocessed.yml # Internal Visual Studio Code config .vscode diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index eaac68c..5f3094d 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -23,7 +23,6 @@ import ( "github.com/Appliscale/perun/logger" "github.com/Appliscale/perun/utilities" "gopkg.in/alecthomas/kingpin.v2" - "strings" ) var ValidateMode = "validate" @@ -33,14 +32,10 @@ var ConfigureMode = "configure" var CreateStackMode = "create-stack" var DestroyStackMode = "delete-stack" -const JSON = "json" -const YAML = "yaml" - type CliArguments struct { Mode *string TemplatePath *string OutputFilePath *string - OutputFileFormat *string ConfigurationPath *string Quiet *bool Yes *bool @@ -54,10 +49,6 @@ type CliArguments struct { PrettyPrint *bool } -func availableFormats() []string { - return []string{JSON, YAML} -} - // Get and validate CLI arguments. Returns error if validation fails. func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { var ( @@ -79,11 +70,10 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { offlineValidate = app.Command(OfflineValidateMode, "Offline Template Validation") offlineValidateTemplate = offlineValidate.Arg("template", "A path to the template file.").Required().String() - convert = app.Command(ConvertMode, "Convertion between JSON and YAML of template files") - convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() - convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() - convertOutputFormat = convert.Arg("format", "Output format: "+strings.ToUpper(JSON)+" | "+strings.ToUpper(YAML)+".").HintAction(availableFormats).Required().String() - prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() + convert = app.Command(ConvertMode, "Convertion between JSON and YAML of template files") + convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() + convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() + prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() configure = app.Command(ConfigureMode, "Create your own configuration mode") @@ -113,7 +103,6 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { cliArguments.Mode = &ConvertMode cliArguments.TemplatePath = convertTemplate cliArguments.OutputFilePath = convertOutputFile - cliArguments.OutputFileFormat = convertOutputFormat cliArguments.PrettyPrint = prettyPrint // configure @@ -121,7 +110,6 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { cliArguments.Mode = &ConfigureMode // create Stack - case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode cliArguments.Stack = createStackName @@ -158,14 +146,5 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { return } - if *cliArguments.Mode == ConvertMode { - *cliArguments.OutputFileFormat = strings.ToLower(*cliArguments.OutputFileFormat) - if *cliArguments.OutputFileFormat != JSON && *cliArguments.OutputFileFormat != YAML { - err = errors.New("Invalid output file format. Use JSON or YAML") - return - } - - } - return } diff --git a/cliparser/cliparser_test.go b/cliparser/cliparser_test.go index 8d1c84f..ce6afd6 100644 --- a/cliparser/cliparser_test.go +++ b/cliparser/cliparser_test.go @@ -21,11 +21,6 @@ import ( "testing" ) -func TestInvalidOutputFormatInConvertMode(t *testing.T) { - assert.Equal(t, "Invalid output file format. Use JSON or YAML", - parseCliArguments([]string{"cmd", "convert", "some_path", "some_path", "wrong_format"}).Error()) -} - func TestInvalidVerbosity(t *testing.T) { assert.Equal(t, "You specified invalid value for --verbosity flag", parseCliArguments([]string{"cmd", "validate", "some_path", "--verbosity=TEST"}).Error()) diff --git a/converter/converter.go b/converter/converter.go index 12a1787..b8ba90a 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -21,15 +21,13 @@ package converter import ( "encoding/json" "errors" - "io/ioutil" - "os" - - "github.com/Appliscale/perun/cliparser" "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/intrinsicsolver" "github.com/Appliscale/perun/logger" "github.com/asaskevich/govalidator" "github.com/ghodss/yaml" + "io/ioutil" + "os" ) // Read template from the file, convert it and check if it has valid structure. @@ -39,43 +37,44 @@ func Convert(context *context.Context) error { if err != nil { return err } + format := detectFormatFromContent(rawTemplate) + var outputTemplate []byte - if *context.CliArguments.OutputFileFormat == cliparser.YAML { - if !govalidator.IsJSON(string(rawTemplate)) { - return errors.New("This is not a valid YAML file") - } - preprocessed, preprocessingError := intrinsicsolver.FixFunctions(rawTemplate, context.Logger, "multiline", "elongate", "correctlong") - if preprocessingError != nil { - context.Logger.Error(preprocessingError.Error()) - } - outputTemplate, err := yamlToJSON(preprocessed) + // If input type file is JSON convert to YAML. + if format == "JSON" { + outputTemplate, err = jsonToYaml(rawTemplate) if err != nil { return err } saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) - } - if *context.CliArguments.OutputFileFormat == cliparser.JSON { - var outputTemplate []byte + // If input type file is YAML, check all functions and create JSON (with or not --pretty-print flag). + } else if format == "YAML" { + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(rawTemplate, context.Logger, "multiline", "elongate", "correctlong") + if preprocessingError != nil { + context.Logger.Error(preprocessingError.Error()) + } if *context.CliArguments.PrettyPrint == false { - outputTemplate, err = yamlToJSON(rawTemplate) + outputTemplate, err = yamlToJson(preprocessed) } else if *context.CliArguments.PrettyPrint == true { - outputTemplate, err = yamlToPrettyJSON(rawTemplate) + outputTemplate, err = yamlToPrettyJson(preprocessed) } if err != nil { return err } - err = saveToFile(outputTemplate, *context.CliArguments.OutputFilePath, context.Logger) if err != nil { return err } + } else { + context.Logger.Always(format) + return nil } return nil } -func toYAML(jsonTemplate []byte) ([]byte, error) { +func jsonToYaml(jsonTemplate []byte) ([]byte, error) { if !govalidator.IsJSON(string(jsonTemplate)) { return nil, errors.New("This is not a valid JSON file") } @@ -85,20 +84,17 @@ func toYAML(jsonTemplate []byte) ([]byte, error) { return yamlTemplate, error } -func yamlToJSON(yamlTemplate []byte) ([]byte, error) { +func yamlToJson(yamlTemplate []byte) ([]byte, error) { jsonTemplate, error := yaml.YAMLToJSON(yamlTemplate) return jsonTemplate, error } -func yamlToPrettyJSON(yamlTemplate []byte) ([]byte, error) { +func yamlToPrettyJson(yamlTemplate []byte) ([]byte, error) { var YAMLObj interface{} templateError := yaml.Unmarshal(yamlTemplate, &YAMLObj) jsonTemplate, templateError := json.MarshalIndent(YAMLObj, "", " ") - if !govalidator.IsJSON(string(jsonTemplate)) { - return nil, errors.New("This is not a valid YAML file") - } return jsonTemplate, templateError } @@ -118,3 +114,15 @@ func saveToFile(template []byte, path string, logger *logger.Logger) error { return nil } + +func detectFormatFromContent(rawTemplate []byte) (format string) { + _, errorYAML := jsonToYaml(rawTemplate) + _, errorJSON := yamlToJson(rawTemplate) + + if errorYAML == nil { + return "JSON" + } else if errorJSON == nil { + return "YAML" + } + return "Unsupported file format. The input file must be either a valid JSON or YAML file." +} From 8982c80ad03659f4e5366a22b7cba86f8c76ac23 Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Tue, 13 Feb 2018 13:45:50 +0100 Subject: [PATCH 012/205] cli Arguments Fix Fixed create-stack mode arguments --- cliparser/cliparser.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index 5f3094d..316f674 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -79,6 +79,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { createStack = app.Command(CreateStackMode, "Creates a stack on aws") createStackName = createStack.Arg("stack", "An AWS stack name.").Required().String() + createStackTemplate = createStack.Arg("template", "A path to the template file.").Required().String() deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") deleteStackName = deleteStack.Arg("stack", "An AWS stack name.").Required().String() @@ -112,6 +113,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { // create Stack case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode + cliArguments.TemplatePath = createStackTemplate cliArguments.Stack = createStackName // delete Stack From 57fd1edec1cd1c02bed1e19ed831aaafccbdcd26 Mon Sep 17 00:00:00 2001 From: pfigwer Date: Thu, 15 Feb 2018 13:11:17 +0100 Subject: [PATCH 013/205] Deployment to GitHub Releases (#80) * Deployment * Release only from master branch --- .travis.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 98f961f..2c1a84b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,4 +7,28 @@ install: make config-install get-deps script: - make code-analysis - - make test \ No newline at end of file + - make test + +before_deploy: + - 'mkdir -p release' + - 'GOOS=linux GOARCH=amd64 go build -o release/perun-linux-amd64' + - 'GOOS=darwin GOARCH=amd64 go build -o release/perun-darwin-amd64' + - 'GOOS=windows GOARCH=amd64 go build -o release/perun-windows-amd64.exe' + - 'GOOS=windows GOARCH=386 go build -o release/perun-windows-386.exe' + - 'tar -C release -czf release/perun-linux-amd64.tar.gz perun-linux-amd64' + - 'tar -C release -czf release/perun-darwin-amd64.tar.gz perun-darwin-amd64' + - 'tar -C release -czf release/perun-windows-amd64.tar.gz perun-windows-amd64.exe' + - 'tar -C release -czf release/perun-windows-386.tar.gz perun-windows-386.exe' + +deploy: + provider: releases + api_key: + secure: eLTZravNiDVvQ1dekb7NvWYJBIG2X6CzfHEqUKJ2JCDlKSqYXrvqZF/B3XxmYvmF1tEAmAa08LMDzzzApl9IML1DVSoW8i6uy+uetg+xbvumAf9fq14nMd0JQEEA1qruE7pwjyQs7h9gXYtyAR01CPhj/xNUQmYV1i8NCTHoljBkO+NsMFyi3WMbW7HTRQZQZXbPBagI06L3tSOCfN/w5KVmRsKFQ3lvmnzs+mTrIvOy2CBQC+0Cp3PQ/p7yyhEWRFd5J6n2jYGxneetnBq0FAfbOF4RIwvrWuu9XI/znxhYMOB5lra0qUwuG+prJStB6oaQ/vHStRcxQorV75Jtm4u/EHcFmmaxTQvPksdZQ8VSIbFonz1qbnuurP5sloiAR1RnJQtQWZKj7I7ioknEBh4kqCGvLUIbt0VpHTNoPKN0a8GYiPSE9UO6J+CNS+FR5mahW3xsHx5dHMV+R4mxcbt16dlg0g8m4tah06bd3P/t91kkgliTWmkHDMX4ES4hh+ribMnsLB0k7iqtuoO2P+gFn80CR5ooAX9Z3u8P8MaEovuPSaO7DqsGfX3uCaFInyBpc5EteCNwgN9dGAfh4mscJlijx28qgJ5quNU56fhcfQ8DoC5nXTM7RRRSu0OB1xSDa9OEf5Nh1AlkDwQKxjAYD+ujYFCXxqSWcntUbqE= + file: + - release/perun-linux-amd64.tar.gz + - release/perun-darwin-amd64.tar.gz + - release/perun-windows-amd64.tar.gz + - release/perun-windows-386.tar.gz + skip_cleanup: true + on: + tags: true From 96ed015a41939900aa4b1432e318b59ed9a0921b Mon Sep 17 00:00:00 2001 From: jlampar Date: Tue, 20 Feb 2018 15:13:05 +0100 Subject: [PATCH 014/205] task in progress --- intrinsicsolver/fixFunctions.go | 35 +++-- offlinevalidator/offlinevalidator.go | 175 +++++++++++++++++++--- offlinevalidator/offlinevalidator_test.go | 41 ++--- 3 files changed, 202 insertions(+), 49 deletions(-) diff --git a/intrinsicsolver/fixFunctions.go b/intrinsicsolver/fixFunctions.go index 6317ec5..8672d9a 100644 --- a/intrinsicsolver/fixFunctions.go +++ b/intrinsicsolver/fixFunctions.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "io/ioutil" "os" "strings" @@ -28,6 +27,10 @@ The result is saved to temporary file, then opened and returned as a []byte arra func FixFunctions(template []byte, logger *logger.Logger, mode ...string) ([]byte, error) { var quotationProcessed, temporaryResult []string preLines, err := parseFileIntoLines(template, logger) + if err != nil { + logger.Error(err.Error()) + return nil, err + } // All single quotation marks are transformed to double ones. for _, line := range preLines { @@ -66,20 +69,26 @@ func FixFunctions(template []byte, logger *logger.Logger, mode ...string) ([]byt temporaryResult = append(temporaryResult, d) } - // Function writeLines saves the processed result to a file (if there would be any errors, it could be investigated there). - if err := writeLines(temporaryResult, ".preprocessed.yml"); err != nil { - logger.Error(err.Error()) - return nil, err - } + stringStream := strings.Join(temporaryResult, "\n") + output := []byte(stringStream) - // Then the temporary result is opened and returned as a []byte. - preprocessedTemplate, err := ioutil.ReadFile(".preprocessed.yml") - if err != nil { - logger.Error(err.Error()) - return preprocessedTemplate, err - } + return output, nil + + // To investigate preprocessed template, uncomment the next lines: + /* + if err := writeLines(temporaryResult, ".preprocessed.yml"); err != nil { + logger.Error(err.Error()) + return nil, err + } + + preprocessedTemplate, err := ioutil.ReadFile(".preprocessed.yml") + if err != nil { + logger.Error(err.Error()) + return preprocessedTemplate, err + } - return preprocessedTemplate, nil + return preprocessedTemplate, nil + */ } // Expands the function name to it's long form without a colon. For example - Fn::FindInMap. diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index f102e79..7d8f70d 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -24,6 +24,8 @@ import ( "io/ioutil" "path" "reflect" + "strconv" + "strings" "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/intrinsicsolver" @@ -85,28 +87,35 @@ func Validate(context *context.Context) bool { return false } - resources := obtainResources(goFormationTemplate, perunTemplate) + deNilizedTemplate, _ := nilNeutralize(goFormationTemplate, context.Logger) + resources := obtainResources(deNilizedTemplate, perunTemplate, context.Logger) + deadResources := getNilResources(resources) + deadProperties := getNilProperties(resources) - valid = validateResources(resources, &specification, context.Logger) + valid = validateResources(resources, &specification, context.Logger, deadProperties, deadResources) return valid } -func validateResources(resources map[string]template.Resource, specification *specification.Specification, sink *logger.Logger) bool { +func validateResources(resources map[string]template.Resource, specification *specification.Specification, sink *logger.Logger, deadProp []string, deadRes []string) bool { for resourceName, resourceValue := range resources { - resourceValidation := sink.AddResourceForValidation(resourceName) + if deadResource := sliceContains(deadRes, resourceName); !deadResource { + resourceValidation := sink.AddResourceForValidation(resourceName) - if resourceSpecification, ok := specification.ResourceTypes[resourceValue.Type]; ok { - for propertyName, propertyValue := range resourceSpecification.Properties { - validateProperties(specification, resourceValue, propertyName, propertyValue, resourceValidation) + if resourceSpecification, ok := specification.ResourceTypes[resourceValue.Type]; ok { + for propertyName, propertyValue := range resourceSpecification.Properties { + if deadProperty := sliceContains(deadProp, propertyName); !deadProperty { + validateProperties(specification, resourceValue, propertyName, propertyValue, resourceValidation) + } + } + } else { + resourceValidation.AddValidationError("Type needs to be specified") + } + if validator, ok := validatorsMap[resourceValue.Type]; ok { + validator.(func(template.Resource, *logger.ResourceValidation) bool)(resourceValue, resourceValidation) } - } else { - resourceValidation.AddValidationError("Type needs to be specified") - } - if validator, ok := validatorsMap[resourceValue.Type]; ok { - validator.(func(template.Resource, *logger.ResourceValidation) bool)(resourceValue, resourceValidation) - } + } } return !sink.HasValidationErrors() } @@ -226,7 +235,7 @@ func parseYAML(templateFile []byte, refTemplate template.Template, logger *logge return template, err } - preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline") + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong") if preprocessingError != nil { logger.Error(preprocessingError.Error()) } @@ -240,7 +249,7 @@ func parseYAML(templateFile []byte, refTemplate template.Template, logger *logge return returnTemplate, nil } -func obtainResources(goformationTemplate cloudformation.Template, perunTemplate template.Template) map[string]template.Resource { +func obtainResources(goformationTemplate cloudformation.Template, perunTemplate template.Template, logger *logger.Logger) map[string]template.Resource { perunResources := perunTemplate.Resources goformationResources := goformationTemplate.Resources @@ -258,6 +267,54 @@ func obtainResources(goformationTemplate cloudformation.Template, perunTemplate */ } + for propertyName, propertyContent := range perunResources { // Searching through PROPERTIES + if propertyContent.Properties == nil { + logger.Always("WARNING! " + propertyName + " <--- is nil.") + } else { + for element, elementValue := range propertyContent.Properties { // Searching through PROPERTIES.ELEMENT + if elementValue == nil { + logger.Always("WARNING! " + propertyName + ": " + element + " <--- is nil.") + } else if elementMap, ok := elementValue.(map[string]interface{}); ok { // Searching through PROPERTIES.ELEMENT.ELEMENT when it is map + for key, value := range elementMap { + if value == nil { + logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + " <--- is nil.") + } else if elementOfElement, ok := value.(map[string]interface{}); ok { + for subKey, subValue := range elementOfElement { + if subValue == nil { + logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + ": " + subKey + " <--- is nil.") + } + } + } else if sliceOfElement, ok := value.([]interface{}); ok { + for indexKey, indexValue := range sliceOfElement { + if indexValue == nil { + logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + "[" + strconv.Itoa(indexKey) + "] <--- is nil.") + } + } + } + } + } else if elementSlice, ok := elementValue.([]interface{}); ok { // Searching through PROPERTIES.ELEMENT.ELEMENT when is is slice + for index, value := range elementSlice { + if value == nil { + logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "] <--- is nil.") + } else if elementOfElement, ok := value.(map[string]interface{}); ok { + for subKey, subValue := range elementOfElement { + if subValue == nil { + logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "]: " + subKey + " <--- is nil.") + } + } + } else if sliceOfElement, ok := value.([]interface{}); ok { + for indexKey, indexValue := range sliceOfElement { + if indexValue == nil { + logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "][" + strconv.Itoa(indexKey) + "] <--- is nil.") + } + } + } + } + } + } + } + } + return perunResources } @@ -268,7 +325,9 @@ func toMapList(resourceProperties map[string]interface{}, propertyName string) [ } mapList := make([]map[string]interface{}, len(subproperties)) for index, value := range subproperties { - mapList[index] = value.(map[string]interface{}) + if _, ok := value.(map[string]interface{}); ok { + mapList[index] = value.(map[string]interface{}) + } } return mapList } @@ -280,7 +339,9 @@ func toStringList(resourceProperties map[string]interface{}, propertyName string } list := make([]string, len(subproperties)) for index, value := range subproperties { - list[index] = value.(string) + if value != nil { + list[index] = value.(string) + } } return list } @@ -292,3 +353,83 @@ func toMap(resourceProperties map[string]interface{}, propertyName string) map[s } return subproperties } + +// There is a possibility that a map[string]interface{} inside the template would have one of it's element's being an intrinsic function designed to output `key : value` pair. +// If this function would be unresolved, it would output of type interface{}. It would be an alien element in a map[string]interface{} surrounding. +// To fix this and alert that there is a missing element, we replace a lonely `nil` inside a map with a `MISSING: nil` pair. +func nilNeutralize(template cloudformation.Template, logger *logger.Logger) (output cloudformation.Template, err error) { + bytes, initErr := json.Marshal(template) + if initErr != nil { + logger.Error(err.Error()) + } + byteSlice := string(bytes) + + var info int + var check1, check2, check3 string + if strings.Contains(byteSlice, ",null,") { + check1 = strings.Replace(byteSlice, ",null,", ",{\"MISSING\":null},", -1) + info++ + } else { + check1 = byteSlice + } + if strings.Contains(check1, "[null,") { + check2 = strings.Replace(check1, "[null,", "[{\"MISSING\":null},", -1) + info++ + } else { + check2 = check1 + } + if strings.Contains(check2, ",null]") { + check3 = strings.Replace(check2, ",null]", ",{\"MISSING\":null}]", -1) + info++ + } else { + check3 = check2 + } + + byteSliceCorrected := []byte(check3) + + tempJSON, err := goformation.ParseJSON(byteSliceCorrected) + if err != nil { + logger.Error(err.Error()) + } + + if info > 0 { + logger.Info("There are intrinsic functions which would output `key : value` pair but are unresolved and are evaluated to . As this element of a template should be a hash table element, the is exchanged with a mock `MISSING : ` pair.") + } + + returnTemplate := *tempJSON + + return returnTemplate, nil +} + +func getNilProperties(resources map[string]template.Resource) []string { + list := make([]string, 0) + for _, resourceContent := range resources { + properties := resourceContent.Properties + for propertyName, propertyContent := range properties { + if propertyContent == nil { + list = append(list, propertyName) + } + } + } + return list +} + +func getNilResources(resources map[string]template.Resource) []string { + list := make([]string, 0) + for resourceName, resourceContent := range resources { + if resourceContent.Properties == nil { + list = append(list, resourceName) + } + + } + return list +} + +func sliceContains(slice []string, match string) bool { + for _, s := range slice { + if s == match { + return true + } + } + return false +} diff --git a/offlinevalidator/offlinevalidator_test.go b/offlinevalidator/offlinevalidator_test.go index 23b125f..d2fd5d9 100644 --- a/offlinevalidator/offlinevalidator_test.go +++ b/offlinevalidator/offlinevalidator_test.go @@ -30,6 +30,9 @@ var spec specification.Specification var sink logger.Logger +var deadProp = make([]string, 0) +var deadRes = make([]string, 0) + func setup() { var err error @@ -51,7 +54,7 @@ func TestValidResource(t *testing.T) { resources := make(map[string]template.Resource) resources["ExampleResource"] = createResourceWithOneProperty("ExampleResourceType", "ExampleProperty", "Property value") - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestInvalidResourceType(t *testing.T) { @@ -59,7 +62,7 @@ func TestInvalidResourceType(t *testing.T) { resources := make(map[string]template.Resource) resources["ExampleResource"] = createResourceWithOneProperty("InvalidType", "ExampleProperty", "Property value") - assert.False(t, validateResources(resources, &spec, &sink), "This resource should be invalid, it has invalid resource type") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be invalid, it has invalid resource type") } func TestLackOfRequiredPropertyInResource(t *testing.T) { @@ -67,7 +70,7 @@ func TestLackOfRequiredPropertyInResource(t *testing.T) { resources := make(map[string]template.Resource) resources["ExampleResource"] = createResourceWithOneProperty("ExampleResourceType", "SomeProperty", "Property value") - assert.False(t, validateResources(resources, &spec, &sink), "This resource should not be valid, it does not have required property") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should not be valid, it does not have required property") } func TestLackOfSubpropertyWithSpecification(t *testing.T) { sink = logger.Logger{} @@ -77,7 +80,7 @@ func TestLackOfSubpropertyWithSpecification(t *testing.T) { } resources["cluster"] = createResourceWithNestedProperties("AWS::Nested3::Cluster", "SomeProperty", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource should not be valid, it does not have property with specification") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should not be valid, it does not have property with specification") } func TestValidPrimitiveTypeInProperty(t *testing.T) { sink = logger.Logger{} @@ -87,7 +90,7 @@ func TestValidPrimitiveTypeInProperty(t *testing.T) { } resources["cluster"] = createResourceWithNestedProperties("AWS::Nested3::Cluster", "Instances", properties) - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestLackOfPrimitiveTypeInProperty(t *testing.T) { @@ -98,7 +101,7 @@ func TestLackOfPrimitiveTypeInProperty(t *testing.T) { } resources["cluster"] = createResourceWithNestedProperties("AWS::Nested3::Cluster", "Instances", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource shouldn't be valid") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource shouldn't be valid") } func TestLackOfPrimitiveTypeInPropertyNestedInProperty(t *testing.T) { @@ -112,7 +115,7 @@ func TestLackOfPrimitiveTypeInPropertyNestedInProperty(t *testing.T) { } resources["cluster"] = createResourceWithNestedProperties("AWS::Nested1::Cluster", "Instances", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource shouldn't be valid, it lacks required property") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource shouldn't be valid, it lacks required property") } func TestLackOfRequiredSubproperty(t *testing.T) { sink = logger.Logger{} @@ -124,7 +127,7 @@ func TestLackOfRequiredSubproperty(t *testing.T) { } resources["cluster"] = createResourceWithNestedProperties("AWS::Nested1::Cluster", "Instances", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource shouldn't be valid, required subproperty is missing") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource shouldn't be valid, required subproperty is missing") } func TestLackOfRequiredPrimitiveTypeInNonrequiredSubproperty(t *testing.T) { sink = logger.Logger{} @@ -134,7 +137,7 @@ func TestLackOfRequiredPrimitiveTypeInNonrequiredSubproperty(t *testing.T) { } resources["ApiGatewayResource"] = createResourceWithNestedProperties("AWS::Nested2::RestApi", "BodyS3Location", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource shouldn't be valid, required primitive property in nonrequired subproperty is missing") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource shouldn't be valid, required primitive property in nonrequired subproperty is missing") } func TestLackOfRequiredPropertyInNonRequiredProperty(t *testing.T) { @@ -148,7 +151,7 @@ func TestLackOfRequiredPropertyInNonRequiredProperty(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::Nested4::Method", "Definition", properties) - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestLackOfRequiredNestedPrimitivePropertyInListItem(t *testing.T) { @@ -168,7 +171,7 @@ func TestLackOfRequiredNestedPrimitivePropertyInListItem(t *testing.T) { resource.Properties["BootstrapActions"] = properties resources["ExampleResource"] = resource - assert.False(t, validateResources(resources, &spec, &sink), "This resource should not be valid, List is empty") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should not be valid, List is empty") } func TestLackOfRequiredListItemSubpropertyInList(t *testing.T) { @@ -185,7 +188,7 @@ func TestLackOfRequiredListItemSubpropertyInList(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::List2::Bucket", "WebsiteConfiguration", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource should not be valid, It must contain RedirectRule property") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should not be valid, It must contain RedirectRule property") } func TestLackOfRequiredPrimitiveTypeListItemInList(t *testing.T) { @@ -209,7 +212,7 @@ func TestLackOfRequiredPrimitiveTypeListItemInList(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::List2::Bucket", "WebsiteConfiguration", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource should not be valid, RedirectRule must contain HostName and HttpRedirectCode") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should not be valid, RedirectRule must contain HostName and HttpRedirectCode") } func TestValidRequiredPrimitiveTypeListItemInList(t *testing.T) { @@ -227,7 +230,7 @@ func TestValidRequiredPrimitiveTypeListItemInList(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::List2::Bucket", "WebsiteConfiguration", properties) - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestLackOfNonRequiredNestedListItemProperty(t *testing.T) { @@ -245,7 +248,7 @@ func TestLackOfNonRequiredNestedListItemProperty(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::List3::Bucket", "LifecycleConfiguration", properties) - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestInvalidList(t *testing.T) { @@ -256,7 +259,7 @@ func TestInvalidList(t *testing.T) { resources["ExampleResource"] = createResourceWithOneProperty("AWS::List4::DBSubnetGroup", "SubnetIds", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestValidList(t *testing.T) { @@ -272,7 +275,7 @@ func TestValidList(t *testing.T) { } resources["ExampleResource"] = resource - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestValidIfMapInNestedPropertyIsMap(t *testing.T) { @@ -286,7 +289,7 @@ func TestValidIfMapInNestedPropertyIsMap(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::Map2::Thing", "AttributePayload", properties) - assert.True(t, validateResources(resources, &spec, &sink), "This resource should be valid") + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } func TestInvalidNonMapProperty(t *testing.T) { @@ -298,7 +301,7 @@ func TestInvalidNonMapProperty(t *testing.T) { } resources["ExampleResource"] = createResourceWithNestedProperties("AWS::Map2::Thing", "AttributePayload", properties) - assert.False(t, validateResources(resources, &spec, &sink), "This resource shouldn't be valid - Attributes should be a Map") + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource shouldn't be valid - Attributes should be a Map") } func createResourceWithNestedProperties(resourceType string, propertyName string, nestedPropertyValue map[string]interface{}) template.Resource { From cd840f028a4f27a2f56233b69789814506a88a67 Mon Sep 17 00:00:00 2001 From: jlampar Date: Mon, 26 Feb 2018 11:13:09 +0100 Subject: [PATCH 015/205] fix: omit unresolved goformation functions, prevent breaking the parser --- intrinsicsolver/fixFunctions.go | 29 ++++++--------- offlinevalidator/offlinevalidator.go | 55 +++++++++++++++------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/intrinsicsolver/fixFunctions.go b/intrinsicsolver/fixFunctions.go index 8672d9a..41f19dd 100644 --- a/intrinsicsolver/fixFunctions.go +++ b/intrinsicsolver/fixFunctions.go @@ -15,14 +15,15 @@ var mapNature = functions[5:] /* FixFunctions : takes []byte file and firstly converts all single quotation marks to double ones (anything between single ones is treated as the rune in GoLang), -then deconstructs file into lines, checks for intrinsic functions. The FixFunctions has modes: `multiline`, `elongate` and `correctlong`. +then deconstructs file into lines, checks for intrinsic functions. The FixFunctions has modes: `multiline`, `elongate`, `correctlong` and `temp`. Mode `multiline` looks for functions of a map nature where the function name is located in one line and it's body (map elements) are located in the following lines (if this would be not fixed an error would be thrown: `json: unsupported type: map[interface {}]interface {}`). The function changes the notation by putting function name in the next line with proper indentation. Mode `elongate` exchanges the short function names into their proper, long equivalent. Mode `correctlong` prepares the file for conversion into JSON. If the file is a YAML with every line being solicitously indented, there is no problem and the `elongate` mode is all we need. But if there is any mixed notation (e.g. indented maps along with one-line maps, functions in one line with the key), parsing must be preceded with some additional operations. -The result is saved to temporary file, then opened and returned as a []byte array. +Mode `temp` allows the user to save the result to a temporary file `.preprocessed.yml`. +The result is returned as a []byte array. */ func FixFunctions(template []byte, logger *logger.Logger, mode ...string) ([]byte, error) { var quotationProcessed, temporaryResult []string @@ -72,23 +73,17 @@ func FixFunctions(template []byte, logger *logger.Logger, mode ...string) ([]byt stringStream := strings.Join(temporaryResult, "\n") output := []byte(stringStream) - return output, nil - - // To investigate preprocessed template, uncomment the next lines: - /* - if err := writeLines(temporaryResult, ".preprocessed.yml"); err != nil { - logger.Error(err.Error()) - return nil, err - } - - preprocessedTemplate, err := ioutil.ReadFile(".preprocessed.yml") - if err != nil { - logger.Error(err.Error()) - return preprocessedTemplate, err + for _, m := range mode { + if m == "temp" { + if err := writeLines(temporaryResult, ".preprocessed.yml"); err != nil { + logger.Error(err.Error()) + return nil, err + } + logger.Info("Created temporary file of a preprocessed template `.preprocessed.yml`") } + } - return preprocessedTemplate, nil - */ + return output, nil } // Expands the function name to it's long form without a colon. For example - Fn::FindInMap. diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index 7d8f70d..99ffd38 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -235,7 +235,7 @@ func parseYAML(templateFile []byte, refTemplate template.Template, logger *logge return template, err } - preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong") + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong", "temp") if preprocessingError != nil { logger.Error(preprocessingError.Error()) } @@ -253,28 +253,16 @@ func obtainResources(goformationTemplate cloudformation.Template, perunTemplate perunResources := perunTemplate.Resources goformationResources := goformationTemplate.Resources - errDecode := mapstructure.Decode(goformationResources, &perunResources) - if errDecode != nil { - /* - Printing errDecode would log: + mapstructure.Decode(goformationResources, &perunResources) - ERROR error(s) decoding: - [template.Resource name] expected a map, got 'bool' - - whenever a value of a property would be a boolean value (e.g. evaluated by !Equals intrinsic function; or e.g. 'got string', 'got float' etc. in other options). - But after logging all the decoding errors, it would log if template is valid or not and eventually log the missing property as it should do - and the error doesn't stand as obstacle of validation. - */ - } - - for propertyName, propertyContent := range perunResources { // Searching through PROPERTIES + for propertyName, propertyContent := range perunResources { if propertyContent.Properties == nil { logger.Always("WARNING! " + propertyName + " <--- is nil.") } else { - for element, elementValue := range propertyContent.Properties { // Searching through PROPERTIES.ELEMENT + for element, elementValue := range propertyContent.Properties { if elementValue == nil { logger.Always("WARNING! " + propertyName + ": " + element + " <--- is nil.") - } else if elementMap, ok := elementValue.(map[string]interface{}); ok { // Searching through PROPERTIES.ELEMENT.ELEMENT when it is map + } else if elementMap, ok := elementValue.(map[string]interface{}); ok { for key, value := range elementMap { if value == nil { logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + " <--- is nil.") @@ -292,7 +280,7 @@ func obtainResources(goformationTemplate cloudformation.Template, perunTemplate } } } - } else if elementSlice, ok := elementValue.([]interface{}); ok { // Searching through PROPERTIES.ELEMENT.ELEMENT when is is slice + } else if elementSlice, ok := elementValue.([]interface{}); ok { for index, value := range elementSlice { if value == nil { logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "] <--- is nil.") @@ -337,6 +325,7 @@ func toStringList(resourceProperties map[string]interface{}, propertyName string if !ok { return nil } + list := make([]string, len(subproperties)) for index, value := range subproperties { if value != nil { @@ -354,9 +343,9 @@ func toMap(resourceProperties map[string]interface{}, propertyName string) map[s return subproperties } -// There is a possibility that a map[string]interface{} inside the template would have one of it's element's being an intrinsic function designed to output `key : value` pair. -// If this function would be unresolved, it would output of type interface{}. It would be an alien element in a map[string]interface{} surrounding. -// To fix this and alert that there is a missing element, we replace a lonely `nil` inside a map with a `MISSING: nil` pair. +// There is a possibility that a hash map inside the template would have one of it's element's being an intrinsic function designed to output `key : value` pair. +// If this function would be unresolved, it would output a standalone of type interface{}. It would be an alien element in a hash map. +// To prevent the parser from breaking, we wipe out the entire, expected hash map element. func nilNeutralize(template cloudformation.Template, logger *logger.Logger) (output cloudformation.Template, err error) { bytes, initErr := json.Marshal(template) if initErr != nil { @@ -367,19 +356,19 @@ func nilNeutralize(template cloudformation.Template, logger *logger.Logger) (out var info int var check1, check2, check3 string if strings.Contains(byteSlice, ",null,") { - check1 = strings.Replace(byteSlice, ",null,", ",{\"MISSING\":null},", -1) + check1 = strings.Replace(byteSlice, ",null,", ",", -1) info++ } else { check1 = byteSlice } if strings.Contains(check1, "[null,") { - check2 = strings.Replace(check1, "[null,", "[{\"MISSING\":null},", -1) + check2 = strings.Replace(check1, "[null,", "[", -1) info++ } else { check2 = check1 } if strings.Contains(check2, ",null]") { - check3 = strings.Replace(check2, ",null]", ",{\"MISSING\":null}]", -1) + check3 = strings.Replace(check2, ",null]", "]", -1) info++ } else { check3 = check2 @@ -392,8 +381,24 @@ func nilNeutralize(template cloudformation.Template, logger *logger.Logger) (out logger.Error(err.Error()) } + infoOpening, link, part, occurences, elements, a, t := "", "", "", "", "", "", "" if info > 0 { - logger.Info("There are intrinsic functions which would output `key : value` pair but are unresolved and are evaluated to . As this element of a template should be a hash table element, the is exchanged with a mock `MISSING : ` pair.") + if info == 1 { + elements = "element" + t = "this " + a = "a" + infoOpening = "is an intrinsic function " + link = "is" + part = "part" + } else { + elements = "elements" + t = "those " + occurences = strconv.Itoa(info) + infoOpening = "are " + occurences + " intrinsic functions " + link = "are" + part = "parts" + } + logger.Info("There " + infoOpening + "which would output `key : value` pair but " + link + " unresolved and " + link + " evaluated to . As " + t + elements + " of a template should be " + a + " hash table " + elements + ", " + t + "standalone " + link + " deleted completely. It is recommended to investigate " + t + part + " of a template manually.") } returnTemplate := *tempJSON From 895b8c0a884625d97b735f1dee3b99e183c3d037 Mon Sep 17 00:00:00 2001 From: jlampar Date: Mon, 26 Feb 2018 11:22:36 +0100 Subject: [PATCH 016/205] fix: omit unresolved goformation functions, prevent breaking the parser --- offlinevalidator/offlinevalidator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index 99ffd38..e9f896f 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -235,7 +235,7 @@ func parseYAML(templateFile []byte, refTemplate template.Template, logger *logge return template, err } - preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong", "temp") + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong") if preprocessingError != nil { logger.Error(preprocessingError.Error()) } From 7b5fedc3d034dd802e9d117fcade63655ea966b8 Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Fri, 9 Feb 2018 16:00:02 +0100 Subject: [PATCH 017/205] Stack Execution Progress #17 Added support for SNS and SQS notifications - remote sink creation and deletion. Untested Stack Execution Progress #17 Added setup/destroy remote sink for creation of required resources Stack Execution Progress #17 Added Table with Tablewriter Stack Execution Progress #17 Added Table Printing Stack Execution Progress #17 Added verification for current process stack messages and auto close Stack Execution Progress #17 Fixed Pull request issues Stack Execution Progress #17 Fixed Pull request issues --- README.md | 44 ++++- cliparser/cliparser.go | 26 ++- logger/logger.go | 8 + main.go | 11 ++ progress/parsewriter.go | 94 +++++++++ progress/progress.go | 424 ++++++++++++++++++++++++++++++++++++++++ stack/stack.go | 56 +++++- 7 files changed, 644 insertions(+), 19 deletions(-) create mode 100644 progress/parsewriter.go create mode 100644 progress/progress.go diff --git a/README.md b/README.md index 6f0e048..351404b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ With first command a default configuration file (`defaults/main.yaml`) will be c ### Commands +#### Validation To validate your template with AWS API (*online validation*), just type: ```bash @@ -50,6 +51,7 @@ To validate your template offline (*well*, almost offline :wink: - *AWS CloudFor ~ $ perun validate_offline ``` +#### Conversion To convert your template between JSON and YAML formats you have to type: ```bash @@ -58,6 +60,7 @@ To convert your template between JSON and YAML formats you have to type: ``` +#### Configuration To create your own configuration file use `configure` mode: ```bash @@ -65,18 +68,45 @@ To create your own configuration file use `configure` mode: ``` Then type path and name of new configuration file. +#### Stack Creation To create new stack you have to type: -``~ $ perun --mode=create-stack - --template= - --stack= -`` +```bash +~ $ perun create-stack +``` To destroy stack just type: -``~ $ perun --mode=delete-stack - --stack= -`` +```bash +~ $ perun delete-stack +``` + +You can use option ``--progress`` to show the stack creation/deletion progress in the console, but +note, that this requires setting up a remote sink. + +##### Remote sink + +To setup remote sink type: + +```bash +~ $ perun setup-remote-sink +``` + +This will create an sns topic and sqs queue with permissions for the sns topic to publish on the sqs +queue. Using above services may produce some cost: +According to the AWS SQS and SNS pricing: +- SNS: + - notifications to the SQS queue are free +- SQS: + - The first 1 million monthly requests are free. + - After that: 0.40$ per million requests after Free Tier (Monthly) + - Typical stack creation uses around a hundred requests + +To destroy remote sink just type: + +```bash +~ $ perun destroy-remote-sink +``` ### Configuration file diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index 316f674..c033694 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -31,6 +31,11 @@ var OfflineValidateMode = "validate_offline" var ConfigureMode = "configure" var CreateStackMode = "create-stack" var DestroyStackMode = "delete-stack" +var SetupSinkMode = "setup-remote-sink" +var DestroySinkMode = "destroy-remote-sink" + +const JSON = "json" +const YAML = "yaml" type CliArguments struct { Mode *string @@ -47,6 +52,7 @@ type CliArguments struct { Sandbox *bool Stack *string PrettyPrint *bool + Progress *bool } // Get and validate CLI arguments. Returns error if validation fails. @@ -63,6 +69,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { region = app.Flag("region", "An AWS region to use.").Short('r').String() sandbox = app.Flag("sandbox", "Do not use configuration files hierarchy.").Bool() configurationPath = app.Flag("config", "A path to the configuration file").Short('c').String() + showProgress = app.Flag("progress", "Show progress of stack creation. Option available only after setting up a remote sink").Bool() onlineValidate = app.Command(ValidateMode, "Online template Validation") onlineValidateTemplate = onlineValidate.Arg("template", "A path to the template file.").Required().String() @@ -77,12 +84,16 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { configure = app.Command(ConfigureMode, "Create your own configuration mode") - createStack = app.Command(CreateStackMode, "Creates a stack on aws") - createStackName = createStack.Arg("stack", "An AWS stack name.").Required().String() + createStack = app.Command(CreateStackMode, "Creates a stack on aws") + createStackName = createStack.Arg("stack", "An AWS stack name.").Required().String() createStackTemplate = createStack.Arg("template", "A path to the template file.").Required().String() deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") deleteStackName = deleteStack.Arg("stack", "An AWS stack name.").Required().String() + + setupSink = app.Command(SetupSinkMode, "Sets up resources required for progress report on stack events (SNS Topic, SQS Queue and SQS Queue Policy)") + + destroySink = app.Command(DestroySinkMode, "Destroys resources created with setup-remote-sink") ) app.HelpFlag.Short('h') app.Version(utilities.VersionStatus()) @@ -113,13 +124,21 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { // create Stack case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode - cliArguments.TemplatePath = createStackTemplate cliArguments.Stack = createStackName + cliArguments.TemplatePath = createStackTemplate // delete Stack case deleteStack.FullCommand(): cliArguments.Mode = &DestroyStackMode cliArguments.Stack = deleteStackName + + // set up remote sink + case setupSink.FullCommand(): + cliArguments.Mode = &SetupSinkMode + + // destroy remote sink + case destroySink.FullCommand(): + cliArguments.Mode = &DestroySinkMode } // OTHER FLAGS @@ -132,6 +151,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { cliArguments.Region = region cliArguments.Sandbox = sandbox cliArguments.ConfigurationPath = configurationPath + cliArguments.Progress = showProgress if *cliArguments.DurationForMFA < 0 { err = errors.New("You should specify value for duration of MFA token greater than zero") diff --git a/logger/logger.go b/logger/logger.go index ae8b5b0..4f46d9e 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -42,6 +42,7 @@ const ( DEBUG INFO ERROR + WARNING ) var verboseModes = [...]string{ @@ -49,6 +50,7 @@ var verboseModes = [...]string{ "DEBUG", " INFO", "ERROR", + "WARNING", } func (verbosity Verbosity) String() string { @@ -78,6 +80,11 @@ func (logger *Logger) Always(message string) { fmt.Println(message) } +// Log error. +func (logger *Logger) Warning(warning string) { + logger.log(WARNING, warning) +} + // Log error. func (logger *Logger) Error(err string) { logger.log(ERROR, err) @@ -169,6 +176,7 @@ func IsVerbosityValid(verbosity string) bool { "TRACE", "DEBUG", "INFO", + "WARNING", "ERROR": return true } diff --git a/main.go b/main.go index bfd83a1..5bd4be9 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "github.com/Appliscale/perun/converter" "github.com/Appliscale/perun/offlinevalidator" "github.com/Appliscale/perun/onlinevalidator" + "github.com/Appliscale/perun/progress" "github.com/Appliscale/perun/stack" "os" ) @@ -77,4 +78,14 @@ func main() { stack.DestroyStack(&context) os.Exit(0) } + + if *context.CliArguments.Mode == cliparser.SetupSinkMode { + progress.ConfigureRemoteSink(&context) + os.Exit(0) + } + + if *context.CliArguments.Mode == cliparser.DestroySinkMode { + progress.DestroyRemoteSink(&context) + os.Exit(0) + } } diff --git a/progress/parsewriter.go b/progress/parsewriter.go new file mode 100644 index 0000000..3947b4d --- /dev/null +++ b/progress/parsewriter.go @@ -0,0 +1,94 @@ +package progress + +import ( + "github.com/fatih/color" + "strings" +) + +const createCompleteStatus = "CREATE_COMPLETE" +const createInProgressStatus = "CREATE_IN_PROGRESS" +const createFailedStatus = "CREATE_FAILED" +const deleteCompleteStatus = "DELETE_COMPLETE" +const deleteFailedStatus = "DELETE_FAILED" +const deleteInProgressStatus = "DELETE_IN_PROGRESS" +const reviewInProgressStatus = "REVIEW_IN_PROGRESS" +const rollbackCompleteStatus = "ROLLBACK_COMPLETE" +const rollbackFailedStatus = "ROLLBACK_FAILED" +const rollbackInProgressStatus = "ROLLBACK_IN_PROGRESS" +const updateCompleteStatus = "UPDATE_COMPLETE" +const updateCompleteCleanupInProgressStatus = "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS" +const updateInProgressStatus = "UPDATE_IN_PROGRESS" +const updateRollbackCompleteStatus = "UPDATE_ROLLBACK_COMPLETE" +const updateRollbackCompleteCleanupInProgressStatus = "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS" +const updateRollbackFailedStatus = "UPDATE_ROLLBACK_FAILED" +const updateRollbackInProgressStatus = "UPDATE_ROLLBACK_IN_PROGRESS" + +type parseWriter struct { + linesPrinted int + bgRed func(a ...interface{}) string + fgRed func(a ...interface{}) string + fgOrange func(a ...interface{}) string + bgOrange func(a ...interface{}) string + grey func(a ...interface{}) string + bgGreen func(a ...interface{}) string + fgGreen func(a ...interface{}) string + cyan func(a ...interface{}) string + statusColorMap map[string]func(a ...interface{}) string +} + +func newParseWriter() (pw *parseWriter) { + pw = &parseWriter{} + pw.linesPrinted = 0 + pw.bgRed = color.New(color.BgHiRed).SprintFunc() + pw.fgRed = color.New(color.FgRed).SprintFunc() + pw.fgOrange = color.New(color.FgHiYellow).SprintFunc() + pw.bgOrange = color.New(color.BgHiYellow).SprintFunc() + pw.grey = color.New(color.FgHiWhite).SprintFunc() + pw.bgGreen = color.New(color.BgGreen).SprintFunc() + pw.fgGreen = color.New(color.FgHiGreen).SprintFunc() + pw.cyan = color.New(color.FgCyan).SprintFunc() + + pw.statusColorMap = map[string]func(a ...interface{}) string{ + createFailedStatus: pw.bgRed, + rollbackFailedStatus: pw.bgRed, + rollbackCompleteStatus: pw.fgRed, + updateRollbackCompleteStatus: pw.fgRed, + updateRollbackInProgressStatus: pw.fgRed, + rollbackInProgressStatus: pw.fgRed, + deleteFailedStatus: pw.bgRed, + updateRollbackFailedStatus: pw.bgRed, + deleteCompleteStatus: pw.grey, + createInProgressStatus: pw.fgOrange, + updateRollbackCompleteCleanupInProgressStatus: pw.bgOrange, + deleteInProgressStatus: pw.fgOrange, + updateCompleteCleanupInProgressStatus: pw.fgOrange, + updateInProgressStatus: pw.fgOrange, + createCompleteStatus: pw.bgGreen, + updateCompleteStatus: pw.bgGreen, + reviewInProgressStatus: pw.cyan, + } + return +} + +func (pw *parseWriter) Write(p []byte) (n int, err error) { + var newString = pw.colorStatuses(string(p)) + print(newString) + pw.linesPrinted += strings.Count(newString, "\n") - strings.Count(newString, "\033[A") + return len(p), nil +} +func (pw *parseWriter) colorStatuses(s string) string { + for status, colorizeFun := range pw.statusColorMap { + if strings.Contains(s, status) { + s = strings.Replace(s, status, colorizeFun(status), -1) + } + } + return s +} + +func (pw *parseWriter) returnWritten() { + for i := 0; i < pw.linesPrinted; i++ { + print("\033[A") + } + pw.linesPrinted = 0 + return +} diff --git a/progress/progress.go b/progress/progress.go new file mode 100644 index 0000000..b16590b --- /dev/null +++ b/progress/progress.go @@ -0,0 +1,424 @@ +package progress + +import ( + "encoding/json" + "errors" + "github.com/Appliscale/perun/context" + "github.com/Appliscale/perun/mysession" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sns" + "github.com/aws/aws-sdk-go/service/sqs" + "github.com/olekukonko/tablewriter" + "os/user" + "strings" + "time" +) + +type Connection struct { + context *context.Context + session *session.Session + + SqsClient *sqs.SQS + sqsQueueOutput *sqs.CreateQueueOutput + sqsQueueAttributes *sqs.GetQueueAttributesOutput + + snsClient *sns.SNS + TopicArn *string +} + +var sinkName = "perun-sink-" + +const awsTimestampLayout = "2006-01-02T15:04:05.000Z" + +// Configure AWS Resources needed for pregress monitoring +func ConfigureRemoteSink(context *context.Context) (err error) { + conn := initRemoteConnection(context) + snsTopicExists, sqsQueueExists, err := conn.verifyRemoteSinkConfigured() + + if snsTopicExists && sqsQueueExists { + context.Logger.Info("Remote sink has already been configured") + return + } else { + shouldSNSTopicBeRemoved := snsTopicExists && !sqsQueueExists + err = conn.deleteRemainingSinkResources(shouldSNSTopicBeRemoved, false) + if err != nil { + context.Logger.Error("error deleting up remote sink: " + err.Error()) + return + } + + if !sqsQueueExists { + err = conn.setUpSQSQueue() + if err != nil { + context.Logger.Error("Error creating sqs queue: " + err.Error()) + return + } + } + + if shouldSNSTopicBeRemoved || !snsTopicExists { // SNS Topic has been removed or does not exist + err = conn.setUpSNSNotification() + if err != nil { + context.Logger.Error("Error creating sqs queue: " + err.Error()) + return + } + } + + if err == nil { + context.Logger.Info("Remote sink configuration successful") + context.Logger.Info("It's configuration may take up to a minute, wait before calling 'create-stack' with flag --progress") + } + return + } +} + +// Remove all AWS Resources created for stack monitoring +func DestroyRemoteSink(context *context.Context) (conn Connection, err error) { + conn = initRemoteConnection(context) + snsTopicExists, sqsQueueExists, err := conn.verifyRemoteSinkConfigured() + if err != nil { + context.Logger.Error("error verifying: " + err.Error()) + } + + if !(snsTopicExists && sqsQueueExists) { + err = errors.New("remote sink has not been configured or has already been deleted") + return + } else { + err = conn.deleteRemainingSinkResources(snsTopicExists, sqsQueueExists) + if err != nil { + return + } + context.Logger.Info("Remote sink deconstruction successful.") + return + } +} + +// Get configuration of created AWS Resources +func GetRemoteSink(context *context.Context, session *session.Session) (conn Connection, err error) { + conn = initMessageService(context, session) + snsTopicExists, sqsQueueExists, err := conn.verifyRemoteSinkConfigured() + if !(snsTopicExists && sqsQueueExists) { + err = errors.New("remote sink has not been configured, run 'perun setup-remote-sink' first. If You done it already, wait for aws sink configuration") + return + } + return +} + +func initRemoteConnection(context *context.Context) Connection { + currentSession := initSession(context) + return initMessageService(context, currentSession) + +} +func initMessageService(context *context.Context, currentSession *session.Session) (conn Connection) { + currentUser, userError := user.Current() + if userError != nil { + context.Logger.Error("error reading currentUser") + } + sinkName += currentUser.Username + "-" + currentUser.Uid + conn.session = currentSession + conn.context = context + return +} + +func initSession(context *context.Context) *session.Session { + tokenError := mysession.UpdateSessionToken(context.Config.DefaultProfile, context.Config.DefaultRegion, context.Config.DefaultDurationForMFA, context) + if tokenError != nil { + context.Logger.Error(tokenError.Error()) + } + currentSession, createSessionError := mysession.CreateSession(context, context.Config.DefaultProfile, &context.Config.DefaultRegion) + if createSessionError != nil { + context.Logger.Error(createSessionError.Error()) + } + return currentSession +} + +func (conn *Connection) verifyRemoteSinkConfigured() (snsTopicExists bool, sqsQueueExists bool, err error) { + snsTopicExists, err = conn.getSnsTopicAttributes() + if err != nil { + conn.context.Logger.Error("Error getting sns topic configuration: " + err.Error()) + } + sqsQueueExists, err = conn.getSqsQueueAttributes() + if err != nil { + conn.context.Logger.Error("Error getting sqs queue configuration: " + err.Error()) + } + return +} + +type Message struct { + Type string + MessageId string + TopicArn string + Subject string + Message string + Timestamp string + SignatureVersion string + Signature string + SigningCertURL string + UnsubscribeURL string +} + +// Monitor queue, that delivers messages sent by cloud formation stack progress +func (conn *Connection) MonitorQueue() { + waitTimeSeconds := int64(3) + receiveMessageInput := sqs.ReceiveMessageInput{ + QueueUrl: conn.sqsQueueOutput.QueueUrl, + WaitTimeSeconds: &waitTimeSeconds, + } + + pw, table := initTableWriter() + + tolerance, err := time.ParseDuration("1s") + if err != nil { + conn.context.Logger.Error(err.Error()) + } + startReadingMessagesTime := time.Now().Add(-tolerance) + + receivedAllMessages := true + for receivedAllMessages { + receivedMessages, err := conn.SqsClient.ReceiveMessage(&receiveMessageInput) + if err != nil { + conn.context.Logger.Error("Error reading messages: " + err.Error()) + } + for e := range receivedMessages.Messages { + v := Message{} + jsonBlob := []byte(*receivedMessages.Messages[e].Body) + err = json.Unmarshal(jsonBlob, &v) + if err != nil { + conn.context.Logger.Error("error reading json message" + err.Error()) + } + + // DELETE READ MESSAGE (to prevent reading the same message multiple times) + conn.SqsClient.DeleteMessage(&sqs.DeleteMessageInput{ + QueueUrl: conn.sqsQueueOutput.QueueUrl, + ReceiptHandle: receivedMessages.Messages[e].ReceiptHandle, + }) + + // Parse property message + splittedMessage := strings.FieldsFunc(v.Message, func(r rune) bool { return r == '\n' }) + messageMap := map[string]string{} + for messageNum := range splittedMessage { + messages := strings.FieldsFunc(splittedMessage[messageNum], func(r rune) bool { return r == '=' }) + messageMap[messages[0]] = messages[1] + } + // Parse timestamp of message + messageArrivedTime, err := time.Parse(awsTimestampLayout, v.Timestamp) + if err != nil { + conn.context.Logger.Error(err.Error()) + } + + if startReadingMessagesTime.Before(messageArrivedTime) { + table.Append([]string{v.Timestamp, messageMap["ResourceStatus"], messageMap["ResourceType"], messageMap["LogicalResourceId"], messageMap["ResourceStatusReason"]}) + pw.returnWritten() + table.Render() + } + // Check if the message has been the last one (status COMPLETE for current stack resource) + if strings.Contains(messageMap["LogicalResourceId"], *conn.context.CliArguments.Stack) && + strings.Contains(messageMap["ResourceStatus"], "COMPLETE") { + receivedAllMessages = false + } + } + } +} +func initTableWriter() (*parseWriter, *tablewriter.Table) { + pw := newParseWriter() + table := tablewriter.NewWriter(pw) + table.SetHeader([]string{"Time", "Status", "Type", "LogicalID", "Status Reason"}) + table.SetBorder(false) + // Set Border to false + table.SetColumnColor(tablewriter.Colors{tablewriter.FgWhiteColor}, + tablewriter.Colors{tablewriter.Bold}, + tablewriter.Colors{tablewriter.FgWhiteColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}, + tablewriter.Colors{tablewriter.FgWhiteColor}) + return pw, table +} + +func (conn *Connection) setUpSNSNotification() (err error) { + //CREATE SNS TOPIC + conn.snsClient = sns.New(conn.session) + topicInput := sns.CreateTopicInput{ + Name: &sinkName, + } + topicOutput, _ := conn.snsClient.CreateTopic(&topicInput) + conn.TopicArn = topicOutput.TopicArn + + //SET UP POLICY + err = conn.setUpSqsPolicy() + + protocolSQS := "sqs" + subscribeInput := sns.SubscribeInput{ + Endpoint: conn.sqsQueueAttributes.Attributes[sqs.QueueAttributeNameQueueArn], + Protocol: &protocolSQS, + TopicArn: conn.TopicArn, + } + conn.snsClient.Subscribe(&subscribeInput) + + conn.context.Logger.Info("Set up SNS Notification topic: " + sinkName) + return +} +func (conn *Connection) setUpSQSQueue() (err error) { + conn.SqsClient = sqs.New(conn.session) + + sixtySec := "60" + sqsInput := sqs.CreateQueueInput{ + QueueName: &sinkName, + Attributes: map[string]*string{ + "MessageRetentionPeriod": &sixtySec, + }, + } + conn.sqsQueueOutput, err = conn.SqsClient.CreateQueue(&sqsInput) + if err != nil { + return + } + + arnAttribute := sqs.QueueAttributeNameAll + queueAttributesInput := sqs.GetQueueAttributesInput{ + AttributeNames: []*string{&arnAttribute}, + QueueUrl: conn.sqsQueueOutput.QueueUrl, + } + conn.sqsQueueAttributes, err = conn.SqsClient.GetQueueAttributes(&queueAttributesInput) + if err != nil { + return + } + conn.context.Logger.Info("Set up SQS Notification Queue: " + sinkName) + return +} + +func (conn *Connection) setUpSqsPolicy() (err error) { + jsonStringPolicy, err := conn.createJsonPolicy() + if err != nil { + conn.context.Logger.Error("error creating json: " + err.Error()) + } + + queueAttributes := sqs.SetQueueAttributesInput{ + QueueUrl: conn.sqsQueueOutput.QueueUrl, + Attributes: map[string]*string{ + sqs.QueueAttributeNamePolicy: &jsonStringPolicy, + }, + } + conn.SqsClient.SetQueueAttributes(&queueAttributes) + + conn.context.Logger.Info("Created SQS access policy for SNS Topic: " + sinkName) + return +} + +type PolicyDocument struct { + Version string + Statement []StatementEntry +} +type StatementEntry struct { + Sid string + Effect string + Action []string + Resource string + Condition Condition + Principal string +} +type Condition struct { + StringEquals map[string]string +} + +func (conn *Connection) createJsonPolicy() (jsonStringPolicy string, err error) { + policy := PolicyDocument{ + Version: "2012-10-17", + Statement: []StatementEntry{ + { + Effect: "Allow", + Action: []string{ + "SQS:*", + }, + Resource: *conn.sqsQueueAttributes.Attributes[sqs.QueueAttributeNameQueueArn], + Condition: Condition{ + StringEquals: map[string]string{"aws:SourceArn": *conn.TopicArn}, + }, + Principal: "*", + }, + }, + } + + jsonPolicy, err := json.Marshal(policy) + jsonStringPolicy = string(jsonPolicy) + return +} +func (conn *Connection) getSnsTopicAttributes() (topicExists bool, err error) { + conn.snsClient = sns.New(conn.session) + + topicExists = false + listTopicsInput := sns.ListTopicsInput{} + err = conn.snsClient.ListTopicsPages(&listTopicsInput, + func(output *sns.ListTopicsOutput, lastPage bool) bool { + for topicNum := range output.Topics { + if strings.Contains(*output.Topics[topicNum].TopicArn, sinkName) { + topicExists = true + conn.TopicArn = output.Topics[topicNum].TopicArn + return false + } + } + return true + }) + return +} +func (conn *Connection) getSqsQueueAttributes() (queueExists bool, err error) { + conn.SqsClient = sqs.New(conn.session) + + queueExists = false + listQueuesInput := sqs.ListQueuesInput{ + QueueNamePrefix: &sinkName, + } + + listQueuesOutput, err := conn.SqsClient.ListQueues(&listQueuesInput) + + for queueNum := range listQueuesOutput.QueueUrls { + if strings.Contains(*listQueuesOutput.QueueUrls[queueNum], sinkName) { + queueExists = true + conn.sqsQueueOutput = &sqs.CreateQueueOutput{ + QueueUrl: listQueuesOutput.QueueUrls[queueNum], + } + + arnAttribute := sqs.QueueAttributeNameQueueArn + queueAttributesInput := sqs.GetQueueAttributesInput{ + AttributeNames: []*string{&arnAttribute}, + QueueUrl: conn.sqsQueueOutput.QueueUrl, + } + conn.sqsQueueAttributes, err = conn.SqsClient.GetQueueAttributes(&queueAttributesInput) + if err != nil { + return + } + return + } + } + return +} + +func (conn *Connection) deleteSnsTopic() (err error) { + deleteTopicInput := sns.DeleteTopicInput{ + TopicArn: conn.TopicArn, + } + _, err = conn.snsClient.DeleteTopic(&deleteTopicInput) + return +} +func (conn *Connection) deleteSqsQueue() (err error) { + deleteQueueInput := sqs.DeleteQueueInput{ + QueueUrl: conn.sqsQueueOutput.QueueUrl, + } + _, err = conn.SqsClient.DeleteQueue(&deleteQueueInput) + return +} + +func (conn *Connection) deleteRemainingSinkResources(deleteSnsTopic bool, deleteSqsQueue bool) (err error) { + if deleteSnsTopic { + err = conn.deleteSnsTopic() + if err != nil { + conn.context.Logger.Error("Error deleting sns Topic: " + err.Error()) + return + } + conn.context.Logger.Info("Deleting SNS Topic") + } + if deleteSqsQueue { + err = conn.deleteSqsQueue() + if err != nil { + conn.context.Logger.Error("Error deleting sqs Queue: " + err.Error()) + return + } + conn.context.Logger.Info("Deleting SQS Queue") + } + return +} diff --git a/stack/stack.go b/stack/stack.go index 0f49f2c..6fa4be2 100644 --- a/stack/stack.go +++ b/stack/stack.go @@ -3,13 +3,15 @@ package stack import ( "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/mysession" + //"github.com/Appliscale/perun/notificationservice" + "github.com/Appliscale/perun/progress" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "io/ioutil" ) // This function gets template and name of stack. It creates "CreateStackInput" structure. -func createStackInput(context *context.Context, template *string, stackName *string) cloudformation.CreateStackInput { +func createStackInput(template *string, stackName *string) cloudformation.CreateStackInput { templateStruct := cloudformation.CreateStackInput{ TemplateBody: template, StackName: stackName, @@ -32,35 +34,71 @@ func getTemplateFromFile(context *context.Context) (string, string) { } // This function uses CreateStackInput variable to create Stack. -func createStack(templateStruct cloudformation.CreateStackInput, session *session.Session) { +func createStack(context *context.Context, templateStruct cloudformation.CreateStackInput, session *session.Session) { api := cloudformation.New(session) - api.CreateStack(&templateStruct) + _, err := api.CreateStack(&templateStruct) + if err != nil { + context.Logger.Error("Error creating stack: " + err.Error()) + } } // This function uses all functions above and session to create Stack. func NewStack(context *context.Context) { template, stackName := getTemplateFromFile(context) - templateStruct := createStackInput(context, &template, &stackName) + templateStruct := createStackInput(&template, &stackName) + tokenError := mysession.UpdateSessionToken(context.Config.DefaultProfile, context.Config.DefaultRegion, context.Config.DefaultDurationForMFA, context) if tokenError != nil { context.Logger.Error(tokenError.Error()) } - session, createSessionError := mysession.CreateSession(context, context.Config.DefaultProfile, &context.Config.DefaultRegion) + currentSession, createSessionError := mysession.CreateSession(context, context.Config.DefaultProfile, &context.Config.DefaultRegion) if createSessionError != nil { context.Logger.Error(createSessionError.Error()) } - createStack(templateStruct, session) + + if *context.CliArguments.Progress { + conn, err := progress.GetRemoteSink(context, currentSession) + if err != nil { + context.Logger.Error("Error getting remote sink configuration: " + err.Error()) + return + } + templateStruct.NotificationARNs = []*string{conn.TopicArn} + createStack(context, templateStruct, currentSession) + conn.MonitorQueue() + } else { + createStack(context, templateStruct, currentSession) + } + } // This function bases on "DeleteStackInput" structure and destroys stack. It uses "StackName" to choose which stack will be destroy. Before that it creates session. func DestroyStack(context *context.Context) { delStackInput := deleteStackInput(context) - session, sessionError := mysession.CreateSession(context, context.Config.DefaultProfile, &context.Config.DefaultRegion) + tokenError := mysession.UpdateSessionToken(context.Config.DefaultProfile, context.Config.DefaultRegion, context.Config.DefaultDurationForMFA, context) + if tokenError != nil { + context.Logger.Error(tokenError.Error()) + } + currentSession, sessionError := mysession.CreateSession(context, context.Config.DefaultProfile, &context.Config.DefaultRegion) if sessionError != nil { context.Logger.Error(sessionError.Error()) } - api := cloudformation.New(session) - api.DeleteStack(&delStackInput) + api := cloudformation.New(currentSession) + + var err error = nil + if *context.CliArguments.Progress { + conn, err := progress.GetRemoteSink(context, currentSession) + if err != nil { + context.Logger.Error("Error getting remote sink configuration: " + err.Error()) + return + } + _, err = api.DeleteStack(&delStackInput) + conn.MonitorQueue() + } else { + _, err = api.DeleteStack(&delStackInput) + } + if err != nil { + context.Logger.Error(err.Error()) + } } // This function gets "StackName" from Stack in CliArguments and creates "DeleteStackInput" structure. From f0798b2328aaafda79fec8005b4086ba60407352 Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Wed, 28 Feb 2018 13:11:34 +0100 Subject: [PATCH 018/205] Named Mode Args #84 Added support for specifying mode arguments via flags --- cliparser/cliparser.go | 91 ++++++++++++++++++++++++++++++++---------- progress/progress.go | 2 +- stack/stack.go | 20 ++++++---- 3 files changed, 85 insertions(+), 28 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index c033694..b8f6628 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -71,25 +71,32 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { configurationPath = app.Flag("config", "A path to the configuration file").Short('c').String() showProgress = app.Flag("progress", "Show progress of stack creation. Option available only after setting up a remote sink").Bool() - onlineValidate = app.Command(ValidateMode, "Online template Validation") - onlineValidateTemplate = onlineValidate.Arg("template", "A path to the template file.").Required().String() + onlineValidate = app.Command(ValidateMode, "Online template Validation") + onlineValidateTemplate = onlineValidate.Arg("template", "A path to the template file.").String() + onlineValidateImpTemplate = onlineValidate.Flag("template", "A path to the template file.").String() - offlineValidate = app.Command(OfflineValidateMode, "Offline Template Validation") - offlineValidateTemplate = offlineValidate.Arg("template", "A path to the template file.").Required().String() + offlineValidate = app.Command(OfflineValidateMode, "Offline Template Validation") + offlineValidateTemplate = offlineValidate.Arg("template", "A path to the template file.").String() + offlineValidateImpTemplate = offlineValidate.Flag("template", "A path to the template file.").String() - convert = app.Command(ConvertMode, "Convertion between JSON and YAML of template files") - convertTemplate = convert.Arg("template", "A path to the template file.").Required().String() - convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").Required().String() - prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() + convert = app.Command(ConvertMode, "Convertion between JSON and YAML of template files") + convertTemplate = convert.Arg("template", "A path to the template file.").String() + convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").String() + convertImpTemplate = convert.Flag("from", "A path to the template file.").String() + convertImpOutputFile = convert.Flag("to", "A path where converted file will be saved.").String() + prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() configure = app.Command(ConfigureMode, "Create your own configuration mode") - createStack = app.Command(CreateStackMode, "Creates a stack on aws") - createStackName = createStack.Arg("stack", "An AWS stack name.").Required().String() - createStackTemplate = createStack.Arg("template", "A path to the template file.").Required().String() + createStack = app.Command(CreateStackMode, "Creates a stack on aws") + createStackName = createStack.Arg("stack", "An AWS stack name.").String() + createStackTemplate = createStack.Arg("template", "A path to the template file.").String() + createStackImpName = createStack.Flag("stack", "Sn AWS stack name.").String() + createStackImpTemplate = createStack.Flag("template", "A path to the template file.").String() - deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") - deleteStackName = deleteStack.Arg("stack", "An AWS stack name.").Required().String() + deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") + deleteStackName = deleteStack.Arg("stack", "An AWS stack name.").String() + deleteStackImpName = deleteStack.Flag("stack", "An AWS stack name.").String() setupSink = app.Command(SetupSinkMode, "Sets up resources required for progress report on stack events (SNS Topic, SQS Queue and SQS Queue Policy)") @@ -103,20 +110,46 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { //online validate case onlineValidate.FullCommand(): cliArguments.Mode = &ValidateMode - cliArguments.TemplatePath = onlineValidateTemplate + if len(*onlineValidateTemplate) > 0 { + cliArguments.TemplatePath = onlineValidateTemplate + } else if len(*onlineValidateImpTemplate) > 0 { + cliArguments.TemplatePath = onlineValidateImpTemplate + } else { + err = errors.New("You have to specify the template, try --help") + return + } // offline validation case offlineValidate.FullCommand(): cliArguments.Mode = &OfflineValidateMode - cliArguments.TemplatePath = offlineValidateTemplate + if len(*offlineValidateTemplate) > 0 { + cliArguments.TemplatePath = offlineValidateTemplate + } else if len(*offlineValidateImpTemplate) > 0 { + cliArguments.TemplatePath = offlineValidateImpTemplate + } else { + err = errors.New("You have to specify the template, try --help") + return + } // convert case convert.FullCommand(): cliArguments.Mode = &ConvertMode - cliArguments.TemplatePath = convertTemplate - cliArguments.OutputFilePath = convertOutputFile cliArguments.PrettyPrint = prettyPrint + if len(*convertImpOutputFile) > 0 && len(*convertImpTemplate) > 0 { + cliArguments.TemplatePath = convertImpTemplate + cliArguments.OutputFilePath = convertImpOutputFile + } else if len(*convertOutputFile) > 0 && len(*convertTemplate) > 0 { + cliArguments.TemplatePath = convertTemplate + cliArguments.OutputFilePath = convertOutputFile + } else if len(*convertTemplate) > 0 && len(*convertImpOutputFile) > 0 { + cliArguments.TemplatePath = convertTemplate + cliArguments.OutputFilePath = convertImpOutputFile + } else { + err = errors.New("You have to specify the template and the output file, try --help") + return + } + // configure case configure.FullCommand(): cliArguments.Mode = &ConfigureMode @@ -124,13 +157,31 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { // create Stack case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode - cliArguments.Stack = createStackName - cliArguments.TemplatePath = createStackTemplate + if len(*createStackImpTemplate) > 0 && len(*createStackImpName) > 0 { + cliArguments.Stack = createStackImpName + cliArguments.TemplatePath = createStackImpTemplate + } else if len(*createStackName) > 0 && len(*createStackTemplate) > 0 { + cliArguments.Stack = createStackName + cliArguments.TemplatePath = createStackTemplate + } else if len(*createStackName) > 0 && len(*createStackImpTemplate) > 0 { + cliArguments.Stack = createStackName + cliArguments.TemplatePath = createStackImpTemplate + } else { + err = errors.New("You have to specify stack name and template file, try --help") + return + } // delete Stack case deleteStack.FullCommand(): cliArguments.Mode = &DestroyStackMode - cliArguments.Stack = deleteStackName + if len(*deleteStackName) > 0 { + cliArguments.Stack = deleteStackName + } else if len(*deleteStackImpName) > 0 { + cliArguments.Stack = deleteStackImpName + } else { + err = errors.New("You have to specify the stack name, try --help") + return + } // set up remote sink case setupSink.FullCommand(): diff --git a/progress/progress.go b/progress/progress.go index b16590b..db4a8b1 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -64,7 +64,7 @@ func ConfigureRemoteSink(context *context.Context) (err error) { if err == nil { context.Logger.Info("Remote sink configuration successful") - context.Logger.Info("It's configuration may take up to a minute, wait before calling 'create-stack' with flag --progress") + context.Logger.Warning("It's configuration may take up to a minute, wait before calling 'create-stack' with flag --progress") } return } diff --git a/stack/stack.go b/stack/stack.go index 6fa4be2..69833bd 100644 --- a/stack/stack.go +++ b/stack/stack.go @@ -34,12 +34,10 @@ func getTemplateFromFile(context *context.Context) (string, string) { } // This function uses CreateStackInput variable to create Stack. -func createStack(context *context.Context, templateStruct cloudformation.CreateStackInput, session *session.Session) { +func createStack(context *context.Context, templateStruct cloudformation.CreateStackInput, session *session.Session) (err error) { api := cloudformation.New(session) - _, err := api.CreateStack(&templateStruct) - if err != nil { - context.Logger.Error("Error creating stack: " + err.Error()) - } + _, err = api.CreateStack(&templateStruct) + return } // This function uses all functions above and session to create Stack. @@ -63,10 +61,18 @@ func NewStack(context *context.Context) { return } templateStruct.NotificationARNs = []*string{conn.TopicArn} - createStack(context, templateStruct, currentSession) + err = createStack(context, templateStruct, currentSession) + if err != nil { + context.Logger.Error("Error creating stack: " + err.Error()) + return + } conn.MonitorQueue() } else { - createStack(context, templateStruct, currentSession) + err := createStack(context, templateStruct, currentSession) + if err != nil { + context.Logger.Error("Error creating stack: " + err.Error()) + return + } } } From d5f6e4cc5f96748dc152bd1e51dcf1fe8af1a769 Mon Sep 17 00:00:00 2001 From: mpolcik Date: Tue, 13 Feb 2018 10:54:45 +0100 Subject: [PATCH 019/205] 54 fix map validation in top level property --- offlinevalidator/offlinevalidator.go | 22 ++++++------ offlinevalidator/offlinevalidator_test.go | 34 ++++++++++++++++++- .../test_resources/test_specification.json | 9 +++++ 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index e9f896f..44848ba 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -137,7 +137,7 @@ func validateProperties( } else if propertyValue.Type == "List" { checkListProperties(specification, resourceValue.Properties, resourceValue.Type, propertyName, propertyValue.ItemType, resourceValidation) } else if propertyValue.Type == "Map" { - checkMapProperties(resourceValue.Properties, resourceValidation) + checkMapProperties(resourceValue.Properties, propertyName, resourceValidation) } } } @@ -166,7 +166,7 @@ func checkListProperties( if subpropertyValue.IsSubproperty() { checkNestedProperties(spec, listItem, resourceValueType, subpropertyName, subpropertyValue.Type, resourceValidation) } else if subpropertyValue.Type == "Map" { - checkMapProperties(listItem, resourceValidation) + checkMapProperties(listItem, propertyName, resourceValidation) } } } @@ -181,7 +181,7 @@ func checkNestedProperties( resourceValidation *logger.ResourceValidation) { if propertySpec, hasSpec := spec.PropertyTypes[resourceValueType+"."+propertyType]; hasSpec { - resourceSubproperties := toMap(resourceProperties, propertyName) + resourceSubproperties, _ := toMap(resourceProperties, propertyName) for subpropertyName, subpropertyValue := range propertySpec.Properties { if _, isPresent := resourceSubproperties[subpropertyName]; !isPresent { if subpropertyValue.Required { @@ -193,7 +193,7 @@ func checkNestedProperties( } else if subpropertyValue.Type == "List" { checkListProperties(spec, resourceSubproperties, resourceValueType, subpropertyName, subpropertyValue.ItemType, resourceValidation) } else if subpropertyValue.Type == "Map" { - checkMapProperties(resourceSubproperties, resourceValidation) + checkMapProperties(resourceSubproperties, subpropertyName, resourceValidation) } } } @@ -202,12 +202,12 @@ func checkNestedProperties( func checkMapProperties( resourceProperties map[string]interface{}, + propertyName string, resourceValidation *logger.ResourceValidation) { - for subpropertyName, subpropertyValue := range resourceProperties { - if reflect.TypeOf(subpropertyValue).Kind() != reflect.Map { - resourceValidation.AddValidationError(subpropertyName + " must be a Map") - } + _, err := toMap(resourceProperties, propertyName) + if err != nil { + resourceValidation.AddValidationError(err.Error()) } } @@ -335,12 +335,12 @@ func toStringList(resourceProperties map[string]interface{}, propertyName string return list } -func toMap(resourceProperties map[string]interface{}, propertyName string) map[string]interface{} { +func toMap(resourceProperties map[string]interface{}, propertyName string) (map[string]interface{}, error) { subproperties, ok := resourceProperties[propertyName].(map[string]interface{}) if !ok { - return map[string]interface{}{} + return nil, errors.New(propertyName + " must be a Map") } - return subproperties + return subproperties, nil } // There is a possibility that a hash map inside the template would have one of it's element's being an intrinsic function designed to output `key : value` pair. diff --git a/offlinevalidator/offlinevalidator_test.go b/offlinevalidator/offlinevalidator_test.go index d2fd5d9..8309665 100644 --- a/offlinevalidator/offlinevalidator_test.go +++ b/offlinevalidator/offlinevalidator_test.go @@ -292,7 +292,7 @@ func TestValidIfMapInNestedPropertyIsMap(t *testing.T) { assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") } -func TestInvalidNonMapProperty(t *testing.T) { +func TestInvalidNestedNonMapProperty(t *testing.T) { sink = logger.Logger{} resources := make(map[string]template.Resource) @@ -304,6 +304,38 @@ func TestInvalidNonMapProperty(t *testing.T) { assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource shouldn't be valid - Attributes should be a Map") } +func TestValidMapProperty(t *testing.T) { + sink = logger.Logger{} + + resources := make(map[string]template.Resource) + resource := template.Resource{} + resource.Type = "AWS::Map3::DBParameterGroup" + resource.Properties = make(map[string]interface{}) + resource.Properties["Parameters"] = map[string]interface{}{ + "general_log": 1, + "long_query_time": 10, + "slow_query_log": 1, + } + resource.Properties["Family"] = "mysql5.6" + resources["ExampleResource"] = resource + + assert.True(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") +} + +func TestInvalidMapProperty(t *testing.T) { + sink = logger.Logger{} + + resources := make(map[string]template.Resource) + resource := template.Resource{} + resource.Type = "AWS::Map3::DBParameterGroup" + resource.Properties = make(map[string]interface{}) + resource.Properties["Parameters"] = "DummyValue" + resource.Properties["Family"] = "mysql5.6" + resources["ExampleResource"] = resource + + assert.False(t, validateResources(resources, &spec, &sink, deadProp, deadRes), "This resource should be valid") +} + func createResourceWithNestedProperties(resourceType string, propertyName string, nestedPropertyValue map[string]interface{}) template.Resource { resource := template.Resource{} diff --git a/offlinevalidator/test_resources/test_specification.json b/offlinevalidator/test_resources/test_specification.json index 7d6f389..57ebbc5 100644 --- a/offlinevalidator/test_resources/test_specification.json +++ b/offlinevalidator/test_resources/test_specification.json @@ -243,6 +243,15 @@ "Type": "AttributePayload" } } + }, + "AWS::Map3::DBParameterGroup": { + "Properties":{ + "Parameters": { + "PrimitiveItemType": "String", + "Required": false, + "Type": "Map" + } + } } }, "ResourceSpecificationVersion": "1.2.3" From 1b34c1ac744460e4cdb5ca816377e7964adc8ed4 Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 1 Mar 2018 11:03:47 +0100 Subject: [PATCH 020/205] json parser now detects syntax errors and writes their position in the malformed file --- offlinevalidator/offlinevalidator.go | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index e9f896f..659e1ac 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -215,6 +215,12 @@ func parseJSON(templateFile []byte, refTemplate template.Template, logger *logge err = json.Unmarshal(templateFile, &refTemplate) if err != nil { + syntaxError, ok := err.(*json.SyntaxError) + if ok { + offset := int(syntaxError.Offset) + line, character := lineAndCharacter(string(templateFile), offset) + logger.Error("Syntax error at line " + strconv.Itoa(line) + ", column " + strconv.Itoa(character)) + } return template, err } @@ -438,3 +444,28 @@ func sliceContains(slice []string, match string) bool { } return false } + +func lineAndCharacter(input string, offset int) (line int, character int) { + lf := rune(0x0A) + + if offset > len(input) || offset < 0 { + return 0, 0 + } + + line = 1 + + for i, b := range input { + if b == lf { + if i < offset { + line++ + character = 0 + } + } else { + character++ + } + if i == offset { + break + } + } + return line, character +} From 75ccc8c75cc35081228d6a5efb3f18d90d305989 Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 1 Mar 2018 11:47:00 +0100 Subject: [PATCH 021/205] handling also the json.UnmarshalTypeError --- offlinevalidator/offlinevalidator.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index 659e1ac..4d434e5 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -215,11 +215,14 @@ func parseJSON(templateFile []byte, refTemplate template.Template, logger *logge err = json.Unmarshal(templateFile, &refTemplate) if err != nil { - syntaxError, ok := err.(*json.SyntaxError) - if ok { - offset := int(syntaxError.Offset) - line, character := lineAndCharacter(string(templateFile), offset) + if syntaxError, isSyntaxError := err.(*json.SyntaxError); isSyntaxError { + syntaxOffset := int(syntaxError.Offset) + line, character := lineAndCharacter(string(templateFile), syntaxOffset) logger.Error("Syntax error at line " + strconv.Itoa(line) + ", column " + strconv.Itoa(character)) + } else if typeError, isTypeError := err.(*json.UnmarshalTypeError); isTypeError { + typeOffset := int(typeError.Offset) + line, character := lineAndCharacter(string(templateFile), typeOffset) + logger.Error("Type error at line " + strconv.Itoa(line) + ", column " + strconv.Itoa(character)) } return template, err } From 767cc005d6ff2c6ce4237cd7476c83a20f1e37a2 Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 1 Mar 2018 12:24:01 +0100 Subject: [PATCH 022/205] updated dependencies reference file --- DEPENDENCIES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 92c2b83..85d44fd 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -2,7 +2,7 @@ ## MIT -- https://github.com/alecthomas/kingpin +- gopkg.in/alecthomas/kingpin.v2 - Command-line parser. It is type-safe and allows to have short version of commands (e.g. `--config`, `-c`). - https://github.com/ghodss/yaml - *Go* lacks of YAML support out of the box, so we need this one. @@ -19,3 +19,5 @@ - AWS API. - https://github.com/go-ini/ini - For handling AWS credential files. +- https://github.com/awslabs/goformation + - Library for working with AWS CloudFormation templates (capable of resolving the intrinsic functions). \ No newline at end of file From 8aed54330e39fed998d1c730e5a0b12188d63843 Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 1 Mar 2018 12:32:30 +0100 Subject: [PATCH 023/205] added https:// --- DEPENDENCIES.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 85d44fd..7cddc31 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -2,7 +2,7 @@ ## MIT -- gopkg.in/alecthomas/kingpin.v2 +- https://gopkg.in/alecthomas/kingpin.v2 - Command-line parser. It is type-safe and allows to have short version of commands (e.g. `--config`, `-c`). - https://github.com/ghodss/yaml - *Go* lacks of YAML support out of the box, so we need this one. @@ -12,6 +12,7 @@ - Library for decoding generic map to go structures. We need this one for type-aware validators implementation. - https://github.com/stretchr/testify - Test framework and assertions. + ## Apache 2.0 @@ -19,5 +20,3 @@ - AWS API. - https://github.com/go-ini/ini - For handling AWS credential files. -- https://github.com/awslabs/goformation - - Library for working with AWS CloudFormation templates (capable of resolving the intrinsic functions). \ No newline at end of file From 8de29480c3cac0bb5b7d10b5743d84c99c12cd30 Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 1 Mar 2018 12:34:10 +0100 Subject: [PATCH 024/205] added https:// --- DEPENDENCIES.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 7cddc31..cc32952 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -2,7 +2,7 @@ ## MIT -- https://gopkg.in/alecthomas/kingpin.v2 +- https://github.com/alecthomas/kingpin - Command-line parser. It is type-safe and allows to have short version of commands (e.g. `--config`, `-c`). - https://github.com/ghodss/yaml - *Go* lacks of YAML support out of the box, so we need this one. @@ -12,7 +12,6 @@ - Library for decoding generic map to go structures. We need this one for type-aware validators implementation. - https://github.com/stretchr/testify - Test framework and assertions. - ## Apache 2.0 @@ -20,3 +19,5 @@ - AWS API. - https://github.com/go-ini/ini - For handling AWS credential files. +- https://github.com/awslabs/goformation + - Library for working with AWS CloudFormation templates (capable of resolving the intrinsic functions). \ No newline at end of file From 32c6babdb837cb4f4b07903ef9ec52a21b677001 Mon Sep 17 00:00:00 2001 From: mpolcik Date: Tue, 6 Mar 2018 11:01:31 +0100 Subject: [PATCH 025/205] update readme for create/delete stack --- README.md | 11 ++++------- cliparser/cliparser.go | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6f0e048..3c51495 100644 --- a/README.md +++ b/README.md @@ -67,15 +67,12 @@ Then type path and name of new configuration file. To create new stack you have to type: -``~ $ perun --mode=create-stack - --template= - --stack= +``~ $ perun create-stack +``~ $ perun delete-stack `` ### Configuration file @@ -123,7 +120,7 @@ mfa_serial = ### Working with stacks -Perun allows to create and destroy stacks. +Perun allows to create and destroy stacks. To create stack it uses your template. It can be JSON or YAML format. @@ -139,7 +136,7 @@ Example JSON template which describe S3 Bucket: } ``` -If you want to destroy stack just type its name. +If you want to destroy stack just type its name. Before you create stack you should validate it with perun :wink:. ## License diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index 316f674..67d4499 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -77,8 +77,8 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { configure = app.Command(ConfigureMode, "Create your own configuration mode") - createStack = app.Command(CreateStackMode, "Creates a stack on aws") - createStackName = createStack.Arg("stack", "An AWS stack name.").Required().String() + createStack = app.Command(CreateStackMode, "Creates a stack on aws") + createStackName = createStack.Arg("stack", "An AWS stack name.").Required().String() createStackTemplate = createStack.Arg("template", "A path to the template file.").Required().String() deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") From 59d333ee896de8896c0dfef6cb4ae40301e7cddf Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Wed, 7 Mar 2018 17:42:51 +0100 Subject: [PATCH 026/205] Stack Parameters #16 Added support for creating parameter files and passing parameters during stack creation --- cliparser/cliparser.go | 30 ++++- main.go | 6 + offlinevalidator/offlinevalidator.go | 8 +- parameters/parameters.go | 193 +++++++++++++++++++++++++++ stack/stack.go | 18 ++- 5 files changed, 243 insertions(+), 12 deletions(-) create mode 100644 parameters/parameters.go diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index b8f6628..f7cdd7b 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -33,6 +33,7 @@ var CreateStackMode = "create-stack" var DestroyStackMode = "delete-stack" var SetupSinkMode = "setup-remote-sink" var DestroySinkMode = "destroy-remote-sink" +var CreateParametersMode = "create-parameters" const JSON = "json" const YAML = "yaml" @@ -40,6 +41,7 @@ const YAML = "yaml" type CliArguments struct { Mode *string TemplatePath *string + Parameters *map[string]string OutputFilePath *string ConfigurationPath *string Quiet *bool @@ -84,7 +86,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { convertOutputFile = convert.Arg("output", "A path where converted file will be saved.").String() convertImpTemplate = convert.Flag("from", "A path to the template file.").String() convertImpOutputFile = convert.Flag("to", "A path where converted file will be saved.").String() - prettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() + convertPrettyPrint = convert.Flag("pretty-print", "Pretty printing JSON").Bool() configure = app.Command(ConfigureMode, "Create your own configuration mode") @@ -93,6 +95,8 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { createStackTemplate = createStack.Arg("template", "A path to the template file.").String() createStackImpName = createStack.Flag("stack", "Sn AWS stack name.").String() createStackImpTemplate = createStack.Flag("template", "A path to the template file.").String() + createStackParams = createStack.Flag("parameter", "list of parameters").StringMap() + //createStackParametersFile = createStack.Flag("parametersFile", "filename with parameters") deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") deleteStackName = deleteStack.Arg("stack", "An AWS stack name.").String() @@ -101,6 +105,13 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { setupSink = app.Command(SetupSinkMode, "Sets up resources required for progress report on stack events (SNS Topic, SQS Queue and SQS Queue Policy)") destroySink = app.Command(DestroySinkMode, "Destroys resources created with setup-remote-sink") + + createParameters = app.Command(CreateParametersMode, "Creates a JSON parameters configuration suitable for give cloud formation file") + createParametersTemplate = createParameters.Arg("template", "A path to the template file.").String() + createParametersImpTemplate = createParameters.Flag("template", "A path to the template file.").String() + createParametersParamsOutputFile = createParameters.Flag("output", "A path to file where parameters will be saved.").String() + createParametersParams = createParameters.Flag("parameter", "list of parameters").StringMap() + createParametersPrettyPrint = createParameters.Flag("pretty-print", "Pretty printing JSON").Bool() ) app.HelpFlag.Short('h') app.Version(utilities.VersionStatus()) @@ -134,7 +145,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { // convert case convert.FullCommand(): cliArguments.Mode = &ConvertMode - cliArguments.PrettyPrint = prettyPrint + cliArguments.PrettyPrint = convertPrettyPrint if len(*convertImpOutputFile) > 0 && len(*convertImpTemplate) > 0 { cliArguments.TemplatePath = convertImpTemplate @@ -157,6 +168,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { // create Stack case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode + cliArguments.Parameters = createStackParams if len(*createStackImpTemplate) > 0 && len(*createStackImpName) > 0 { cliArguments.Stack = createStackImpName cliArguments.TemplatePath = createStackImpTemplate @@ -183,6 +195,20 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { return } + // create Parameters + case createParameters.FullCommand(): + cliArguments.Mode = &CreateParametersMode + if len(*createParametersTemplate) > 0 { + cliArguments.TemplatePath = createParametersTemplate + } else if len(*createParametersImpTemplate) > 0 { + cliArguments.TemplatePath = createParametersImpTemplate + } else { + err = errors.New("You have to specify the cloud formation template, try --help") + } + cliArguments.OutputFilePath = createParametersParamsOutputFile + cliArguments.Parameters = createParametersParams + cliArguments.PrettyPrint = createParametersPrettyPrint + // set up remote sink case setupSink.FullCommand(): cliArguments.Mode = &SetupSinkMode diff --git a/main.go b/main.go index 5bd4be9..bd5c64e 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "github.com/Appliscale/perun/converter" "github.com/Appliscale/perun/offlinevalidator" "github.com/Appliscale/perun/onlinevalidator" + "github.com/Appliscale/perun/parameters" "github.com/Appliscale/perun/progress" "github.com/Appliscale/perun/stack" "os" @@ -88,4 +89,9 @@ func main() { progress.DestroyRemoteSink(&context) os.Exit(0) } + + if *context.CliArguments.Mode == cliparser.CreateParametersMode { + parameters.ConfigureParameters(&context) + os.Exit(0) + } } diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index e9f896f..019db73 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -75,9 +75,9 @@ func Validate(context *context.Context) bool { templateFileExtension := path.Ext(*context.CliArguments.TemplatePath) if templateFileExtension == ".json" { - goFormationTemplate, err = parseJSON(rawTemplate, perunTemplate, context.Logger) + goFormationTemplate, err = ParseJSON(rawTemplate, perunTemplate, context.Logger) } else if templateFileExtension == ".yaml" || templateFileExtension == ".yml" { - goFormationTemplate, err = parseYAML(rawTemplate, perunTemplate, context.Logger) + goFormationTemplate, err = ParseYAML(rawTemplate, perunTemplate, context.Logger) } else { err = errors.New("Invalid template file format.") } @@ -211,7 +211,7 @@ func checkMapProperties( } } -func parseJSON(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { +func ParseJSON(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { err = json.Unmarshal(templateFile, &refTemplate) if err != nil { @@ -228,7 +228,7 @@ func parseJSON(templateFile []byte, refTemplate template.Template, logger *logge return returnTemplate, nil } -func parseYAML(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { +func ParseYAML(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { err = yaml.Unmarshal(templateFile, &refTemplate) if err != nil { diff --git a/parameters/parameters.go b/parameters/parameters.go new file mode 100644 index 0000000..f2e98e8 --- /dev/null +++ b/parameters/parameters.go @@ -0,0 +1,193 @@ +// Copyright 2017 Appliscale +// +// Maintainers and contributors are listed in README file inside repository. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package parameters provides tools for interactive creation of parameters file for aws +// cloud formation. + +package parameters + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/Appliscale/perun/context" + "github.com/Appliscale/perun/offlinevalidator" + "github.com/Appliscale/perun/offlinevalidator/template" + cloudformation2 "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/awslabs/goformation/cloudformation" + "io/ioutil" + "os" + "path" + "regexp" + "strings" +) + +type Parameter struct { + ParameterKey string + ParameterValue string +} + +func GetJSONParameters(context *context.Context) (resultString []byte, err error) { + var parameters []*Parameter + parameters, err = GetParameters(context) + if err != nil { + context.Logger.Error(err.Error()) + return + } + + if *context.CliArguments.PrettyPrint { + resultString, err = json.MarshalIndent(parameters, "", " ") + } else { + resultString, err = json.Marshal(parameters) + } + return +} + +func ConfigureParameters(context *context.Context) error { + resultString, err := GetJSONParameters(context) + if err != nil { + return err + } + if *context.CliArguments.OutputFilePath != "" { + context.Logger.Info("Writing parameters configuration to file: " + *context.CliArguments.OutputFilePath) + + _, err = os.Stat(*context.CliArguments.OutputFilePath + ".json") + if err == nil { + context.Logger.Warning("File " + *context.CliArguments.OutputFilePath + ".json would be overriten by this action. Do you want to continue? [Y/N]") + var ans string + for ans != "n" && ans != "y" { + fmt.Scanf("%s", &ans) + ans = strings.ToLower(ans) + } + if ans == "n" { + context.Logger.Info("Aborting..") + return errors.New("user aborted") + } + } + err = ioutil.WriteFile(*context.CliArguments.OutputFilePath+".json", resultString, 0666) + if err != nil { + context.Logger.Error(err.Error()) + } + } else { + println(string(resultString)) + } + return nil +} + +func GetAwsParameters(context *context.Context) (parameters []*cloudformation2.Parameter, err error) { + var params []*Parameter + params, err = GetParameters(context) + if err != nil { + return + } + for paramnum := range params { + parameters = append(parameters, + &cloudformation2.Parameter{ + ParameterValue: ¶ms[paramnum].ParameterValue, + ParameterKey: ¶ms[paramnum].ParameterKey}) + } + return +} + +func GetParameters(context *context.Context) (parameters []*Parameter, err error) { + templateFile, err := parseTemplate(context) + if err != nil { + context.Logger.Error(err.Error()) + return nil, err + } + for parameterName, parameterSpec := range templateFile.Parameters { + var parameterValid bool + var parameterValue string + if context.CliArguments.Parameters != nil { + var exists bool + parameterValue, exists = (*context.CliArguments.Parameters)[parameterName] + if exists { + parameterValid, err = checkParameterValid(parameterName, parameterSpec.(map[string]interface{}), parameterValue, context) + } + } else { + parameterValid = false + } + for !parameterValid { + print(parameterName, ": ") + fmt.Scanf("%s", ¶meterValue) + parameterValid, err = checkParameterValid(parameterName, parameterSpec.(map[string]interface{}), parameterValue, context) + if err != nil { + context.Logger.Error(err.Error()) + return + } + } + parameters = append(parameters, &Parameter{ParameterKey: parameterName, ParameterValue: parameterValue}) + } + return +} + +func checkParameterValid(parameterName string, parameterArgument map[string]interface{}, parameterValue string, context *context.Context) (bool, error) { + if parameterArgument["AllowedValues"] != nil { + allowedValues := getAllowedValues(parameterArgument) + if !sliceContains(parameterValue, allowedValues) { + context.Logger.Error("Value '" + parameterValue + "' is not allowed for Parameter " + parameterName + ". Value must be one of following: [" + strings.Join(allowedValues, ", ") + "]") + return false, nil + } + } + + if parameterArgument["AllowedPattern"] != nil { + allowedPattern := parameterArgument["AllowedPattern"].(string) + matches, err := regexp.Match(allowedPattern, []byte(parameterValue)) + if err != nil { + return false, err + } + if !matches { + context.Logger.Error("Value '" + parameterValue + "' does not match the required pattern for Parameter " + parameterName) + return false, nil + } + } + return true, nil +} +func getAllowedValues(parameterArgument map[string]interface{}) (res []string) { + list := parameterArgument["AllowedValues"].([]interface{}) + for _, val := range list { + res = append(res, val.(string)) + } + return +} + +func sliceContains(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +func parseTemplate(context *context.Context) (res cloudformation.Template, err error) { + rawTemplate, err := ioutil.ReadFile(*context.CliArguments.TemplatePath) + if err != nil { + return + } + + myTemplate := template.Template{} + + templateFileExtension := path.Ext(*context.CliArguments.TemplatePath) + if templateFileExtension == ".json" { + res, err = offlinevalidator.ParseJSON(rawTemplate, myTemplate, context.Logger) + } else if templateFileExtension == ".yaml" || templateFileExtension == ".yml" { + res, err = offlinevalidator.ParseYAML(rawTemplate, myTemplate, context.Logger) + } else { + err = errors.New("Invalid template file format.") + } + return +} diff --git a/stack/stack.go b/stack/stack.go index 69833bd..cfef4f6 100644 --- a/stack/stack.go +++ b/stack/stack.go @@ -3,7 +3,7 @@ package stack import ( "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/mysession" - //"github.com/Appliscale/perun/notificationservice" + "github.com/Appliscale/perun/parameters" "github.com/Appliscale/perun/progress" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -11,8 +11,14 @@ import ( ) // This function gets template and name of stack. It creates "CreateStackInput" structure. -func createStackInput(template *string, stackName *string) cloudformation.CreateStackInput { +func createStackInput(template *string, stackName *string, context *context.Context) cloudformation.CreateStackInput { + params, err := parameters.GetAwsParameters(context) + if err != nil { + context.Logger.Error(err.Error()) + } + templateStruct := cloudformation.CreateStackInput{ + Parameters: params, TemplateBody: template, StackName: stackName, } @@ -34,7 +40,7 @@ func getTemplateFromFile(context *context.Context) (string, string) { } // This function uses CreateStackInput variable to create Stack. -func createStack(context *context.Context, templateStruct cloudformation.CreateStackInput, session *session.Session) (err error) { +func createStack(templateStruct cloudformation.CreateStackInput, session *session.Session) (err error) { api := cloudformation.New(session) _, err = api.CreateStack(&templateStruct) return @@ -43,7 +49,7 @@ func createStack(context *context.Context, templateStruct cloudformation.CreateS // This function uses all functions above and session to create Stack. func NewStack(context *context.Context) { template, stackName := getTemplateFromFile(context) - templateStruct := createStackInput(&template, &stackName) + templateStruct := createStackInput(&template, &stackName, context) tokenError := mysession.UpdateSessionToken(context.Config.DefaultProfile, context.Config.DefaultRegion, context.Config.DefaultDurationForMFA, context) if tokenError != nil { @@ -61,14 +67,14 @@ func NewStack(context *context.Context) { return } templateStruct.NotificationARNs = []*string{conn.TopicArn} - err = createStack(context, templateStruct, currentSession) + err = createStack(templateStruct, currentSession) if err != nil { context.Logger.Error("Error creating stack: " + err.Error()) return } conn.MonitorQueue() } else { - err := createStack(context, templateStruct, currentSession) + err := createStack(templateStruct, currentSession) if err != nil { context.Logger.Error("Error creating stack: " + err.Error()) return From 28e66798b4e56e98508f3f08243fb3b62099dc0c Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Wed, 7 Mar 2018 18:26:31 +0100 Subject: [PATCH 027/205] Stack Parameters #16 Added support for getting parameters from file --- cliparser/cliparser.go | 16 +++++++++------- parameters/parameters.go | 10 +++++++--- stack/stack.go | 23 ++++++++++++++++++++++- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/cliparser/cliparser.go b/cliparser/cliparser.go index f7cdd7b..134b68c 100644 --- a/cliparser/cliparser.go +++ b/cliparser/cliparser.go @@ -55,6 +55,7 @@ type CliArguments struct { Stack *string PrettyPrint *bool Progress *bool + ParametersFile *string } // Get and validate CLI arguments. Returns error if validation fails. @@ -90,13 +91,13 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { configure = app.Command(ConfigureMode, "Create your own configuration mode") - createStack = app.Command(CreateStackMode, "Creates a stack on aws") - createStackName = createStack.Arg("stack", "An AWS stack name.").String() - createStackTemplate = createStack.Arg("template", "A path to the template file.").String() - createStackImpName = createStack.Flag("stack", "Sn AWS stack name.").String() - createStackImpTemplate = createStack.Flag("template", "A path to the template file.").String() - createStackParams = createStack.Flag("parameter", "list of parameters").StringMap() - //createStackParametersFile = createStack.Flag("parametersFile", "filename with parameters") + createStack = app.Command(CreateStackMode, "Creates a stack on aws") + createStackName = createStack.Arg("stack", "An AWS stack name.").String() + createStackTemplate = createStack.Arg("template", "A path to the template file.").String() + createStackImpName = createStack.Flag("stack", "Sn AWS stack name.").String() + createStackImpTemplate = createStack.Flag("template", "A path to the template file.").String() + createStackParams = createStack.Flag("parameter", "list of parameters").StringMap() + createStackParametersFile = createStack.Flag("parameters-file", "filename with parameters").String() deleteStack = app.Command(DestroyStackMode, "Deletes a stack on aws") deleteStackName = deleteStack.Arg("stack", "An AWS stack name.").String() @@ -169,6 +170,7 @@ func ParseCliArguments(args []string) (cliArguments CliArguments, err error) { case createStack.FullCommand(): cliArguments.Mode = &CreateStackMode cliArguments.Parameters = createStackParams + cliArguments.ParametersFile = createStackParametersFile if len(*createStackImpTemplate) > 0 && len(*createStackImpName) > 0 { cliArguments.Stack = createStackImpName cliArguments.TemplatePath = createStackImpTemplate diff --git a/parameters/parameters.go b/parameters/parameters.go index f2e98e8..ab45881 100644 --- a/parameters/parameters.go +++ b/parameters/parameters.go @@ -64,9 +64,9 @@ func ConfigureParameters(context *context.Context) error { if *context.CliArguments.OutputFilePath != "" { context.Logger.Info("Writing parameters configuration to file: " + *context.CliArguments.OutputFilePath) - _, err = os.Stat(*context.CliArguments.OutputFilePath + ".json") + _, err = os.Stat(*context.CliArguments.OutputFilePath) if err == nil { - context.Logger.Warning("File " + *context.CliArguments.OutputFilePath + ".json would be overriten by this action. Do you want to continue? [Y/N]") + context.Logger.Warning("File " + *context.CliArguments.OutputFilePath + " would be overriten by this action. Do you want to continue? [Y/N]") var ans string for ans != "n" && ans != "y" { fmt.Scanf("%s", &ans) @@ -77,7 +77,7 @@ func ConfigureParameters(context *context.Context) error { return errors.New("user aborted") } } - err = ioutil.WriteFile(*context.CliArguments.OutputFilePath+".json", resultString, 0666) + err = ioutil.WriteFile(*context.CliArguments.OutputFilePath, resultString, 0666) if err != nil { context.Logger.Error(err.Error()) } @@ -93,6 +93,10 @@ func GetAwsParameters(context *context.Context) (parameters []*cloudformation2.P if err != nil { return } + parameters = ParseParameterToAwsCompatible(params) + return +} +func ParseParameterToAwsCompatible(params []*Parameter) (parameters []*cloudformation2.Parameter) { for paramnum := range params { parameters = append(parameters, &cloudformation2.Parameter{ diff --git a/stack/stack.go b/stack/stack.go index cfef4f6..e696e00 100644 --- a/stack/stack.go +++ b/stack/stack.go @@ -1,6 +1,7 @@ package stack import ( + "encoding/json" "github.com/Appliscale/perun/context" "github.com/Appliscale/perun/mysession" "github.com/Appliscale/perun/parameters" @@ -12,7 +13,7 @@ import ( // This function gets template and name of stack. It creates "CreateStackInput" structure. func createStackInput(template *string, stackName *string, context *context.Context) cloudformation.CreateStackInput { - params, err := parameters.GetAwsParameters(context) + params, err := getParameters(context) if err != nil { context.Logger.Error(err.Error()) } @@ -121,3 +122,23 @@ func deleteStackInput(context *context.Context) cloudformation.DeleteStackInput } return templateStruct } + +// Get the parameters - if parameters file provided - from file, else - interactively from user +func getParameters(context *context.Context) (params []*cloudformation.Parameter, err error) { + if *context.CliArguments.ParametersFile == "" { + params, err = parameters.GetAwsParameters(context) + } else { + var parametersData []byte + var readParameters []*parameters.Parameter + parametersData, err = ioutil.ReadFile(*context.CliArguments.ParametersFile) + if err != nil { + return + } + err = json.Unmarshal(parametersData, &readParameters) + if err != nil { + return + } + params = parameters.ParseParameterToAwsCompatible(readParameters) + } + return +} From 411bf024235735c46d864fdfbf1ac637545e54cf Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Thu, 8 Mar 2018 11:34:52 +0100 Subject: [PATCH 028/205] Stack Parameters #16 Extracted some common functions --- converter/converter.go | 12 ++--- helpers/formathelpers.go | 66 ++++++++++++++++++++++++++ helpers/listhelpers.go | 10 ++++ offlinevalidator/offlinevalidator.go | 69 ++++------------------------ parameters/parameters.go | 41 ++++++----------- 5 files changed, 104 insertions(+), 94 deletions(-) create mode 100644 helpers/formathelpers.go create mode 100644 helpers/listhelpers.go diff --git a/converter/converter.go b/converter/converter.go index b8ba90a..57c981f 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -19,9 +19,9 @@ package converter import ( - "encoding/json" "errors" "github.com/Appliscale/perun/context" + "github.com/Appliscale/perun/helpers" "github.com/Appliscale/perun/intrinsicsolver" "github.com/Appliscale/perun/logger" "github.com/asaskevich/govalidator" @@ -79,21 +79,21 @@ func jsonToYaml(jsonTemplate []byte) ([]byte, error) { return nil, errors.New("This is not a valid JSON file") } - yamlTemplate, error := yaml.JSONToYAML(jsonTemplate) + yamlTemplate, err := yaml.JSONToYAML(jsonTemplate) - return yamlTemplate, error + return yamlTemplate, err } func yamlToJson(yamlTemplate []byte) ([]byte, error) { - jsonTemplate, error := yaml.YAMLToJSON(yamlTemplate) - return jsonTemplate, error + jsonTemplate, err := yaml.YAMLToJSON(yamlTemplate) + return jsonTemplate, err } func yamlToPrettyJson(yamlTemplate []byte) ([]byte, error) { var YAMLObj interface{} templateError := yaml.Unmarshal(yamlTemplate, &YAMLObj) - jsonTemplate, templateError := json.MarshalIndent(YAMLObj, "", " ") + jsonTemplate, templateError := helpers.PrettyPrintJSON(YAMLObj) return jsonTemplate, templateError diff --git a/helpers/formathelpers.go b/helpers/formathelpers.go new file mode 100644 index 0000000..1adec35 --- /dev/null +++ b/helpers/formathelpers.go @@ -0,0 +1,66 @@ +package helpers + +import ( + "encoding/json" + "errors" + "github.com/Appliscale/perun/intrinsicsolver" + "github.com/Appliscale/perun/logger" + "github.com/Appliscale/perun/offlinevalidator/template" + "github.com/awslabs/goformation" + "github.com/awslabs/goformation/cloudformation" + "github.com/ghodss/yaml" + "path" +) + +func GetParser(filename string) (func([]byte, template.Template, *logger.Logger) (cloudformation.Template, error), error) { + templateFileExtension := path.Ext(filename) + if templateFileExtension == ".json" { + return ParseJSON, nil + } else if templateFileExtension == ".yaml" || templateFileExtension == ".yml" { + return ParseYAML, nil + } else { + return nil, errors.New("Invalid template file format.") + } +} + +func ParseJSON(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { + + err = json.Unmarshal(templateFile, &refTemplate) + if err != nil { + return template, err + } + + tempJSON, err := goformation.ParseJSON(templateFile) + if err != nil { + logger.Error(err.Error()) + } + + returnTemplate := *tempJSON + + return returnTemplate, nil +} + +func ParseYAML(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { + + err = yaml.Unmarshal(templateFile, &refTemplate) + if err != nil { + return template, err + } + + preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong") + if preprocessingError != nil { + logger.Error(preprocessingError.Error()) + } + tempYAML, err := goformation.ParseYAML(preprocessed) + if err != nil { + logger.Error(err.Error()) + } + + returnTemplate := *tempYAML + + return returnTemplate, nil +} + +func PrettyPrintJSON(toPrint interface{}) ([]byte, error) { + return json.MarshalIndent(toPrint, "", " ") +} diff --git a/helpers/listhelpers.go b/helpers/listhelpers.go new file mode 100644 index 0000000..61e7b52 --- /dev/null +++ b/helpers/listhelpers.go @@ -0,0 +1,10 @@ +package helpers + +func SliceContains(list []string, a string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index 019db73..0b9e4b8 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -20,22 +20,19 @@ package offlinevalidator import ( "encoding/json" - "errors" "io/ioutil" - "path" "reflect" "strconv" "strings" "github.com/Appliscale/perun/context" - "github.com/Appliscale/perun/intrinsicsolver" + "github.com/Appliscale/perun/helpers" "github.com/Appliscale/perun/logger" "github.com/Appliscale/perun/offlinevalidator/template" "github.com/Appliscale/perun/offlinevalidator/validators" "github.com/Appliscale/perun/specification" "github.com/awslabs/goformation" "github.com/awslabs/goformation/cloudformation" - "github.com/ghodss/yaml" "github.com/mitchellh/mapstructure" ) @@ -73,15 +70,12 @@ func Validate(context *context.Context) bool { var perunTemplate template.Template var goFormationTemplate cloudformation.Template - templateFileExtension := path.Ext(*context.CliArguments.TemplatePath) - if templateFileExtension == ".json" { - goFormationTemplate, err = ParseJSON(rawTemplate, perunTemplate, context.Logger) - } else if templateFileExtension == ".yaml" || templateFileExtension == ".yml" { - goFormationTemplate, err = ParseYAML(rawTemplate, perunTemplate, context.Logger) - } else { - err = errors.New("Invalid template file format.") + parser, err := helpers.GetParser(*context.CliArguments.TemplatePath) + if err != nil { + context.Logger.Error(err.Error()) + return false } - + goFormationTemplate, err = parser(rawTemplate, perunTemplate, context.Logger) if err != nil { context.Logger.Error(err.Error()) return false @@ -99,12 +93,12 @@ func Validate(context *context.Context) bool { func validateResources(resources map[string]template.Resource, specification *specification.Specification, sink *logger.Logger, deadProp []string, deadRes []string) bool { for resourceName, resourceValue := range resources { - if deadResource := sliceContains(deadRes, resourceName); !deadResource { + if deadResource := helpers.SliceContains(deadRes, resourceName); !deadResource { resourceValidation := sink.AddResourceForValidation(resourceName) if resourceSpecification, ok := specification.ResourceTypes[resourceValue.Type]; ok { for propertyName, propertyValue := range resourceSpecification.Properties { - if deadProperty := sliceContains(deadProp, propertyName); !deadProperty { + if deadProperty := helpers.SliceContains(deadProp, propertyName); !deadProperty { validateProperties(specification, resourceValue, propertyName, propertyValue, resourceValidation) } } @@ -211,44 +205,6 @@ func checkMapProperties( } } -func ParseJSON(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { - - err = json.Unmarshal(templateFile, &refTemplate) - if err != nil { - return template, err - } - - tempJSON, err := goformation.ParseJSON(templateFile) - if err != nil { - logger.Error(err.Error()) - } - - returnTemplate := *tempJSON - - return returnTemplate, nil -} - -func ParseYAML(templateFile []byte, refTemplate template.Template, logger *logger.Logger) (template cloudformation.Template, err error) { - - err = yaml.Unmarshal(templateFile, &refTemplate) - if err != nil { - return template, err - } - - preprocessed, preprocessingError := intrinsicsolver.FixFunctions(templateFile, logger, "multiline", "elongate", "correctlong") - if preprocessingError != nil { - logger.Error(preprocessingError.Error()) - } - tempYAML, err := goformation.ParseYAML(preprocessed) - if err != nil { - logger.Error(err.Error()) - } - - returnTemplate := *tempYAML - - return returnTemplate, nil -} - func obtainResources(goformationTemplate cloudformation.Template, perunTemplate template.Template, logger *logger.Logger) map[string]template.Resource { perunResources := perunTemplate.Resources goformationResources := goformationTemplate.Resources @@ -429,12 +385,3 @@ func getNilResources(resources map[string]template.Resource) []string { } return list } - -func sliceContains(slice []string, match string) bool { - for _, s := range slice { - if s == match { - return true - } - } - return false -} diff --git a/parameters/parameters.go b/parameters/parameters.go index ab45881..6d54769 100644 --- a/parameters/parameters.go +++ b/parameters/parameters.go @@ -24,13 +24,12 @@ import ( "errors" "fmt" "github.com/Appliscale/perun/context" - "github.com/Appliscale/perun/offlinevalidator" + "github.com/Appliscale/perun/helpers" "github.com/Appliscale/perun/offlinevalidator/template" cloudformation2 "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/awslabs/goformation/cloudformation" "io/ioutil" "os" - "path" "regexp" "strings" ) @@ -49,7 +48,7 @@ func GetJSONParameters(context *context.Context) (resultString []byte, err error } if *context.CliArguments.PrettyPrint { - resultString, err = json.MarshalIndent(parameters, "", " ") + resultString, err = helpers.PrettyPrintJSON(parameters) } else { resultString, err = json.Marshal(parameters) } @@ -67,12 +66,12 @@ func ConfigureParameters(context *context.Context) error { _, err = os.Stat(*context.CliArguments.OutputFilePath) if err == nil { context.Logger.Warning("File " + *context.CliArguments.OutputFilePath + " would be overriten by this action. Do you want to continue? [Y/N]") - var ans string - for ans != "n" && ans != "y" { - fmt.Scanf("%s", &ans) - ans = strings.ToLower(ans) + var answer string + for answer != "n" && answer != "y" { + fmt.Scanf("%s", &answer) + answer = strings.ToLower(answer) } - if ans == "n" { + if answer == "n" { context.Logger.Info("Aborting..") return errors.New("user aborted") } @@ -96,6 +95,7 @@ func GetAwsParameters(context *context.Context) (parameters []*cloudformation2.P parameters = ParseParameterToAwsCompatible(params) return } + func ParseParameterToAwsCompatible(params []*Parameter) (parameters []*cloudformation2.Parameter) { for paramnum := range params { parameters = append(parameters, @@ -141,7 +141,7 @@ func GetParameters(context *context.Context) (parameters []*Parameter, err error func checkParameterValid(parameterName string, parameterArgument map[string]interface{}, parameterValue string, context *context.Context) (bool, error) { if parameterArgument["AllowedValues"] != nil { allowedValues := getAllowedValues(parameterArgument) - if !sliceContains(parameterValue, allowedValues) { + if !helpers.SliceContains(allowedValues, parameterValue) { context.Logger.Error("Value '" + parameterValue + "' is not allowed for Parameter " + parameterName + ". Value must be one of following: [" + strings.Join(allowedValues, ", ") + "]") return false, nil } @@ -160,6 +160,7 @@ func checkParameterValid(parameterName string, parameterArgument map[string]inte } return true, nil } + func getAllowedValues(parameterArgument map[string]interface{}) (res []string) { list := parameterArgument["AllowedValues"].([]interface{}) for _, val := range list { @@ -168,30 +169,16 @@ func getAllowedValues(parameterArgument map[string]interface{}) (res []string) { return } -func sliceContains(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - func parseTemplate(context *context.Context) (res cloudformation.Template, err error) { rawTemplate, err := ioutil.ReadFile(*context.CliArguments.TemplatePath) if err != nil { return } - myTemplate := template.Template{} - - templateFileExtension := path.Ext(*context.CliArguments.TemplatePath) - if templateFileExtension == ".json" { - res, err = offlinevalidator.ParseJSON(rawTemplate, myTemplate, context.Logger) - } else if templateFileExtension == ".yaml" || templateFileExtension == ".yml" { - res, err = offlinevalidator.ParseYAML(rawTemplate, myTemplate, context.Logger) - } else { - err = errors.New("Invalid template file format.") + parser, err := helpers.GetParser(*context.CliArguments.TemplatePath) + if err != nil { + return } + res, err = parser(rawTemplate, myTemplate, context.Logger) return } From 56647680ef3e8dd0dc87ab0253fe1b2838c09d7c Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 8 Mar 2018 11:41:21 +0100 Subject: [PATCH 029/205] simplifying the obtainResources function --- logger/logger.go | 8 ++ offlinevalidator/offlinevalidator.go | 167 ++++++++++++++++++++------- 2 files changed, 135 insertions(+), 40 deletions(-) diff --git a/logger/logger.go b/logger/logger.go index ae8b5b0..4f46d9e 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -42,6 +42,7 @@ const ( DEBUG INFO ERROR + WARNING ) var verboseModes = [...]string{ @@ -49,6 +50,7 @@ var verboseModes = [...]string{ "DEBUG", " INFO", "ERROR", + "WARNING", } func (verbosity Verbosity) String() string { @@ -78,6 +80,11 @@ func (logger *Logger) Always(message string) { fmt.Println(message) } +// Log error. +func (logger *Logger) Warning(warning string) { + logger.log(WARNING, warning) +} + // Log error. func (logger *Logger) Error(err string) { logger.log(ERROR, err) @@ -169,6 +176,7 @@ func IsVerbosityValid(verbosity string) bool { "TRACE", "DEBUG", "INFO", + "WARNING", "ERROR": return true } diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index fde480b..1963cee 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -264,50 +264,16 @@ func obtainResources(goformationTemplate cloudformation.Template, perunTemplate mapstructure.Decode(goformationResources, &perunResources) + //spew.Dump(perunResources) + for propertyName, propertyContent := range perunResources { if propertyContent.Properties == nil { - logger.Always("WARNING! " + propertyName + " <--- is nil.") + logger.Warning(propertyName + " <--- is nil.") } else { for element, elementValue := range propertyContent.Properties { - if elementValue == nil { - logger.Always("WARNING! " + propertyName + ": " + element + " <--- is nil.") - } else if elementMap, ok := elementValue.(map[string]interface{}); ok { - for key, value := range elementMap { - if value == nil { - logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + " <--- is nil.") - } else if elementOfElement, ok := value.(map[string]interface{}); ok { - for subKey, subValue := range elementOfElement { - if subValue == nil { - logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + ": " + subKey + " <--- is nil.") - } - } - } else if sliceOfElement, ok := value.([]interface{}); ok { - for indexKey, indexValue := range sliceOfElement { - if indexValue == nil { - logger.Always("WARNING! " + propertyName + ": " + element + ": " + key + "[" + strconv.Itoa(indexKey) + "] <--- is nil.") - } - } - } - } - } else if elementSlice, ok := elementValue.([]interface{}); ok { - for index, value := range elementSlice { - if value == nil { - logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "] <--- is nil.") - } else if elementOfElement, ok := value.(map[string]interface{}); ok { - for subKey, subValue := range elementOfElement { - if subValue == nil { - logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "]: " + subKey + " <--- is nil.") - } - } - } else if sliceOfElement, ok := value.([]interface{}); ok { - for indexKey, indexValue := range sliceOfElement { - if indexValue == nil { - logger.Always("WARNING! " + propertyName + ": " + element + "[" + strconv.Itoa(index) + "][" + strconv.Itoa(indexKey) + "] <--- is nil.") - } - } - } - } - } + initPath := []interface{}{element} + var discarded interface{} + checkWhereIsNil(element, elementValue, propertyName, logger, initPath, &discarded) } } } @@ -448,6 +414,83 @@ func sliceContains(slice []string, match string) bool { return false } +func mapContainsNil(mp map[string]interface{}) bool { + for _, m := range mp { + if m == nil { + return true + } + } + return false +} + +func sliceContainsNil(slice []interface{}) bool { + for _, s := range slice { + if s == nil { + return true + } + } + return false +} + +func isNonSFB(v interface{}) bool { + var isString, isFloat, isBool bool + if _, ok := v.(string); ok { + isString = true + } else if _, ok := v.(float64); ok { + isFloat = true + } else if _, ok := v.(bool); ok { + isBool = true + } + if !isString && !isFloat && !isBool { + return true + } + return false +} + +func isPlainMap(mp map[string]interface{}) bool { + // First we check is it more complex. If so - it is worth investigating and we should stop checking. + for _, m := range mp { + if _, ok := m.(map[string]interface{}); ok { + return false + } else if _, ok := m.([]interface{}); ok { + return false + } + } + // Ok, it isn't. So is there any ? + if mapContainsNil(mp) { // Yes, it is - so it is a map worth investigating. This is not the map we're looking for. + return false + } + + return true // There is no and no complexity - it is plain, non-nil map. +} + +func isPlainSlice(slc []interface{}) bool { + // First we check is it more complex. If so - it is worth investigating and we should stop checking. + for _, s := range slc { + if _, ok := s.(map[string]interface{}); ok { + return false + } else if _, ok := s.([]interface{}); ok { + return false + } + } + // Ok, it isn't. It is a flat slice. So is there any ? + if sliceContainsNil(slc) { // Yes, it is - so it is a slice worth investigating. This is not the slice we're looking for. + return false + } + + return true // There is no and no complexity - it is plain, non-nil slice. +} + +func discard(slice []interface{}, n interface{}) []interface{} { + result := []interface{}{} + for _, s := range slice { + if s != n { + result = append(result, s) + } + } + return result +} + func lineAndCharacter(input string, offset int) (line int, character int) { lf := rune(0x0A) @@ -472,3 +515,47 @@ func lineAndCharacter(input string, offset int) (line int, character int) { } return line, character } + +func checkWhereIsNil(n interface{}, v interface{}, baseLevel string, logger *logger.Logger, fullPath []interface{}, dsc *interface{}) { + if v == nil { // Value we encountered is nil - this is the end of investigation. + where := "" + for _, element := range fullPath { + if stringElement, ok := element.(string); ok { + if where != "" { + where += ": " + stringElement + } else { + where = stringElement + } + } else if intElement, ok := element.(int); ok { + where += "[" + strconv.Itoa(intElement) + "]" + } + } + logger.Warning(baseLevel + ": " + where + " <--- is nil.") + } else if mp, ok := v.(map[string]interface{}); ok { // Value we encountered is a map. + if isPlainMap(mp) { // Check is it plain, non-nil map. + // It is - we shouldn't dive into. And we should remove it from the location path. + *dsc = n + } else { + for kmp, vmp := range mp { + if isNonSFB(vmp) { + fullPath = append(fullPath, kmp) + fullPath = discard(fullPath, *dsc) + checkWhereIsNil(kmp, vmp, baseLevel, logger, fullPath, dsc) + } + } + } + } else if slc, ok := v.([]interface{}); ok { // Value we encountered is a slice. + if isPlainSlice(slc) { // Check is it plain, non-nil slice. + // It is - we shouldn't dive into. And we should remove it from the location path. + *dsc = n + } else { + for islc, vslc := range slc { + if isNonSFB(vslc) { + fullPath = append(fullPath, islc) + fullPath = discard(fullPath, *dsc) + checkWhereIsNil(islc, vslc, baseLevel, logger, fullPath, dsc) + } + } + } + } +} From 2fcfab401bcddccf1b4b7f709185d33f095187fa Mon Sep 17 00:00:00 2001 From: jlampar Date: Thu, 8 Mar 2018 11:57:45 +0100 Subject: [PATCH 030/205] simplifying the obtainResources function --- offlinevalidator/offlinevalidator.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/offlinevalidator/offlinevalidator.go b/offlinevalidator/offlinevalidator.go index 1963cee..a835eaa 100644 --- a/offlinevalidator/offlinevalidator.go +++ b/offlinevalidator/offlinevalidator.go @@ -264,15 +264,13 @@ func obtainResources(goformationTemplate cloudformation.Template, perunTemplate mapstructure.Decode(goformationResources, &perunResources) - //spew.Dump(perunResources) - for propertyName, propertyContent := range perunResources { if propertyContent.Properties == nil { logger.Warning(propertyName + " <--- is nil.") } else { for element, elementValue := range propertyContent.Properties { - initPath := []interface{}{element} - var discarded interface{} + initPath := []interface{}{element} // The path from the Property name to the element. + var discarded interface{} // Container which stores the encountered nodes that aren't on the path. checkWhereIsNil(element, elementValue, propertyName, logger, initPath, &discarded) } } @@ -432,6 +430,7 @@ func sliceContainsNil(slice []interface{}) bool { return false } +// We check if the element is non-string, non-float64, non-boolean. Then it is another node or . There is no other option. func isNonSFB(v interface{}) bool { var isString, isFloat, isBool bool if _, ok := v.(string); ok { @@ -461,11 +460,11 @@ func isPlainMap(mp map[string]interface{}) bool { return false } - return true // There is no and no complexity - it is plain, non-nil map. + return true // There is no and no complexity - it is a plain, non-nil map. } func isPlainSlice(slc []interface{}) bool { - // First we check is it more complex. If so - it is worth investigating and we should stop checking. + // The same flow as in `isPlainMap` function. for _, s := range slc { if _, ok := s.(map[string]interface{}); ok { return false @@ -473,12 +472,12 @@ func isPlainSlice(slc []interface{}) bool { return false } } - // Ok, it isn't. It is a flat slice. So is there any ? - if sliceContainsNil(slc) { // Yes, it is - so it is a slice worth investigating. This is not the slice we're looking for. + + if sliceContainsNil(slc) { return false } - return true // There is no and no complexity - it is plain, non-nil slice. + return true } func discard(slice []interface{}, n interface{}) []interface{} { @@ -533,20 +532,19 @@ func checkWhereIsNil(n interface{}, v interface{}, baseLevel string, logger *log logger.Warning(baseLevel + ": " + where + " <--- is nil.") } else if mp, ok := v.(map[string]interface{}); ok { // Value we encountered is a map. if isPlainMap(mp) { // Check is it plain, non-nil map. - // It is - we shouldn't dive into. And we should remove it from the location path. - *dsc = n + // It is - we shouldn't dive into. + *dsc = n // The name is stored in the `discarded` container as the name of the blind alley. } else { for kmp, vmp := range mp { if isNonSFB(vmp) { fullPath = append(fullPath, kmp) - fullPath = discard(fullPath, *dsc) + fullPath = discard(fullPath, *dsc) // If the output path would be different, it seems that we've encountered some node which is not on the way to the . It will be discarded from the path. Otherwise the paths are the same and we hit the point. checkWhereIsNil(kmp, vmp, baseLevel, logger, fullPath, dsc) } } } - } else if slc, ok := v.([]interface{}); ok { // Value we encountered is a slice. - if isPlainSlice(slc) { // Check is it plain, non-nil slice. - // It is - we shouldn't dive into. And we should remove it from the location path. + } else if slc, ok := v.([]interface{}); ok { // The same flow as above. + if isPlainSlice(slc) { *dsc = n } else { for islc, vslc := range slc { From ea704c21c3f5668038271ef30ae46b6e026e0590 Mon Sep 17 00:00:00 2001 From: Maksymilian Wojczuk Date: Fri, 9 Feb 2018 16:00:02 +0100 Subject: [PATCH 031/205] Stack Execution Progress #17 Added support for SNS and SQS notifications - remote sink creation and deletion. Untested Stack Execution Progress #17 Added setup/destroy remote sink for creation of required resources Stack Execution Progress #17 Added Table with Tablewriter Stack Execution Progress #17 Added Table Printing Stack Execution Progress #17 Added verification for current process stack messages and auto close Stack Execution Progress #17 Fixed Pull request issues Stack Execution Progress #17 Fixed Pull request issues --- README.md | 51 ++++- cliparser/cliparser.go | 22 ++- logger/logger.go | 8 + main.go | 11 ++ progress/parsewriter.go | 94 +++++++++ progress/progress.go | 424 ++++++++++++++++++++++++++++++++++++++++ stack/stack.go | 56 +++++- 7 files changed, 652 insertions(+), 14 deletions(-) create mode 100644 progress/parsewriter.go create mode 100644 progress/progress.go diff --git a/README.md b/README.md index 3c51495..13b54b5 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ With first command a default configuration file (`defaults/main.yaml`) will be c ### Commands +#### Validation To validate your template with AWS API (*online validation*), just type: ```bash @@ -50,6 +51,7 @@ To validate your template offline (*well*, almost offline :wink: - *AWS CloudFor ~ $ perun validate_offline ``` +#### Conversion To convert your template between JSON and YAML formats you have to type: ```bash @@ -58,6 +60,7 @@ To convert your template between JSON and YAML formats you have to type: ``` +#### Configuration To create your own configuration file use `configure` mode: ```bash @@ -65,15 +68,55 @@ To create your own configuration file use `configure` mode: ``` Then type path and name of new configuration file. +#### Stack Creation To create new stack you have to type: -``~ $ perun create-stack +``` + +or + +```bash +~ $ perun create-stack --template=