From 35557e3384f208c644dbe1a3c5f115e43d8dcca0 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 17:24:07 -0400 Subject: [PATCH 01/10] YAMLParser: Fix some character escaping for string constants --- YAMLParser/KnownStuff.cs | 4 +++- YAMLParser/MsgFile.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/YAMLParser/KnownStuff.cs b/YAMLParser/KnownStuff.cs index 23c707f1..fbc7169d 100644 --- a/YAMLParser/KnownStuff.cs +++ b/YAMLParser/KnownStuff.cs @@ -82,7 +82,9 @@ public static SingleType WhatItIs(MsgFile parent, string s, string extraindent) { string[] pieces = s.Split('/'); string package = null; - if (pieces.Length == 2) + // sometimes, a string can contain the char '/', such as the following line: + // string CONTENT_JSON = "application/json" + if (pieces.Length == 2 && !s.ToLower().Contains("string")) { package = pieces[0]; s = pieces[1]; diff --git a/YAMLParser/MsgFile.cs b/YAMLParser/MsgFile.cs index e6b04212..3ab53c81 100644 --- a/YAMLParser/MsgFile.cs +++ b/YAMLParser/MsgFile.cs @@ -470,7 +470,7 @@ public override string ToString() def[i] = def[i].Replace(" ", " "); def[i] = def[i].Replace(" = ", "="); } - GUTS = GUTS.Replace("$MYMESSAGEDEFINITION", "@\"" + def.Aggregate("", (current, d) => current + (d + "\n")).Trim('\n') + "\""); + GUTS = GUTS.Replace("$MYMESSAGEDEFINITION", "@\"" + def.Aggregate("", (current, d) => current + (d + "\n")).Trim('\n').Replace("\"", "\"\"") + "\""); GUTS = GUTS.Replace("$MYHASHEADER", HasHeader.ToString().ToLower()); GUTS = GUTS.Replace("$MYFIELDS", GeneratedDictHelper.Length > 5 ? "{{" + GeneratedDictHelper + "}}" : "()"); GUTS = GUTS.Replace("$NULLCONSTBODY", ""); From b683ce764e46451a1e66532d44bc3c4626dadd74 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 17:25:23 -0400 Subject: [PATCH 02/10] YAMLParser: temp workaround to allow multiple arguments in -m --- YAMLParser/Program.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/YAMLParser/Program.cs b/YAMLParser/Program.cs index 7037478c..07f23f4c 100644 --- a/YAMLParser/Program.cs +++ b/YAMLParser/Program.cs @@ -45,8 +45,10 @@ public static void Main(params string[] args) return 1; } + List dirs = messageDirectories.Value().Split(',').ToList(); + Program.Run( - messageDirectories.HasValue() ? messageDirectories.Values : null, + messageDirectories.HasValue() ? dirs : null, assemblies.HasValue() ? assemblies.Values : null, outputDirectory.HasValue() ? outputDirectory.Value() : null, interactive.HasValue(), From 41e0c239ff4015548a49392bcb0b46980395d3c1 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 17:28:08 -0400 Subject: [PATCH 03/10] YAMLParser: expand list of CSharpKeywords --- YAMLParser/SingleType.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/YAMLParser/SingleType.cs b/YAMLParser/SingleType.cs index db114bf7..68a0549a 100644 --- a/YAMLParser/SingleType.cs +++ b/YAMLParser/SingleType.cs @@ -11,10 +11,12 @@ namespace FauxMessages { public class SingleType { - // TODO extend check to other C# keywords - private static readonly string[] CSharpKeywords = { "object", "params", "namespace", "const", "static" }; + // c# keywords listed on https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/ as of August 1st, 2018 + private static readonly string[] reserved_keywords = { "abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked", "class", "const", "continue", "decimal", "default", "delegate", "do", "double", "else", "enum", "event", "explicit", "extern", "false", "finally", "fixed", "float", "for", "foreach", "goto", "if", "implicit", "in", "int", "interface", "internal", "is", "lock", "long", "namespace", "new", "null", "object", "operator", "out", "override", "params", "private", "protected", "public", "readonly", "ref", "return", "sbyte", "sealed", "short", "sizeof", "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try", "typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", "using static", "virtual", "void", "volatile", "while" }; + private static readonly string[] contextual_keywords = { "add", "alias", "ascending", "async", "await", "descending", "dynamic", "from", "get", "global", "group", "into", "join", "let", "nameof", "orderby", "partial", "partial", "remove", "select", "set", "value", "var", "when", "where", "where", "yield" }; + private static readonly string[] CSharpKeywords = reserved_keywords.Union(contextual_keywords).ToArray(); - private static bool IsCSharpKeyword(string name) + public static bool IsCSharpKeyword(string name) { return CSharpKeywords.Contains(name); } From 1ffe0032b4ecc977381ad293142f99b46b572f02 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 17:45:03 -0400 Subject: [PATCH 04/10] YAMLParser: Add check to ensure pkgs/fields will be valid C# identifiers --- YAMLParser/MsgFileLocator.cs | 58 ++++++++++++++++++++++++++++++++++++ YAMLParser/SingleType.cs | 20 ++++++++++--- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/YAMLParser/MsgFileLocator.cs b/YAMLParser/MsgFileLocator.cs index 3147fdfb..e27d0502 100644 --- a/YAMLParser/MsgFileLocator.cs +++ b/YAMLParser/MsgFileLocator.cs @@ -1,13 +1,36 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text; +using FauxMessages; namespace YAMLParser { public class MsgFileLocation { + // Unicode categories according to https://stackoverflow.com/a/950651/4036588 + private static readonly UnicodeCategory[] validFirstChars = + { + UnicodeCategory.UppercaseLetter, // Lu + UnicodeCategory.LowercaseLetter, // Ll + UnicodeCategory.TitlecaseLetter, // Lt + UnicodeCategory.ModifierLetter, // Lm + UnicodeCategory.OtherLetter // Lo + // underscore is also permitted for the first char + }; + private static readonly UnicodeCategory[] otherValidChars = + { + UnicodeCategory.LetterNumber, // Nl + UnicodeCategory.NonSpacingMark, // Mn + UnicodeCategory.SpacingCombiningMark, // Mc + UnicodeCategory.DecimalDigitNumber, // Nd + UnicodeCategory.ConnectorPunctuation, // Pc + UnicodeCategory.Format // Cf + }; + private static readonly UnicodeCategory[] validCSharpChars = validFirstChars.Union(otherValidChars).ToArray(); + private static string[] MSG_GEN_FOLDER_NAMES = { "msg", @@ -29,6 +52,10 @@ private static string getPackageName(string path) string foldername = chunks[chunks.Length - 2]; if (MSG_GEN_FOLDER_NAMES.Contains(foldername)) foldername = chunks[chunks.Length - 3]; + + if (!IsValidCSharpIdentifier(foldername)) + throw new ArgumentException(String.Format("'{0}' from '{1}' is not a compatible C# identifier name\n\tThe package name must conform to C# Language Specifications (refer to this StackOverflow answer: https://stackoverflow.com/a/950651/4036588)\n", foldername, path)); + return foldername; } @@ -75,6 +102,37 @@ public override string ToString() { return string.Format("{0}.{1}", System.IO.Path.Combine(package, basename), extension); } + + public static bool IsValidCSharpIdentifier(string toTest) + { + if (toTest.Length == 0) // obviously..? + return false; + + if (SingleType.IsCSharpKeyword(toTest)) // best to avoid any complications + return false; + + char[] letters = toTest.ToCharArray(); + char first = letters[0]; + + if (first != '_' && !validFirstChars.Contains(CharUnicodeInfo.GetUnicodeCategory(first))) + return false; + + foreach (char c in letters) + if (!validCSharpChars.Contains(CharUnicodeInfo.GetUnicodeCategory(c))) + return false; + + return true; + + // TODO: fix this regex method, replace the above method with it, + // and get rid of the three UnicodeCategory arrays + // (regex method found at https://stackoverflow.com/a/1904462/4036588) + + //const string start = @"(\p{Lu}|\p{Ll}|\p{Lt}|\p{Lm}|\p{Lo}|\p{Nl})"; + //const string extend = @"(\p{Mn}|\p{Mc}|\p{Nd}|\p{Pc}|\p{Cf})"; + //Regex ident = new Regex(string.Format("{0}({0}|{1})*", start, extend)); + //toTest = toTest.Normalize(); + //return ident.IsMatch(toTest); + } } internal static class MsgFileLocator diff --git a/YAMLParser/SingleType.cs b/YAMLParser/SingleType.cs index 68a0549a..80d9a0fe 100644 --- a/YAMLParser/SingleType.cs +++ b/YAMLParser/SingleType.cs @@ -109,9 +109,14 @@ public void Finalize(MsgFile parent, string[] s, bool isliteral) otherstuff = " = " + parts[1]; } - if (IsCSharpKeyword(name)) + if (!MsgFileLocation.IsValidCSharpIdentifier(name) && name.Length > 0) { - name = "@" + name; + if (IsCSharpKeyword(name)) + { + name = "@" + name; + } + else + throw new ArgumentException(String.Format("Variable '{0}' from '{1}' is not a compatible C# identifier name\n\tAll variable names must conform to C# Language Specifications (refer to this StackOverflow answer: https://stackoverflow.com/a/950651/4036588)\n", name, parent.msgFileLocation.Path)); } for (int i = 2; i < s.Length; i++) @@ -227,10 +232,17 @@ public void refinalize(MsgFile parent, string REALTYPE) name = parts[0]; otherstuff = " = " + parts[1]; } - if (IsCSharpKeyword(name)) + + if (!MsgFileLocation.IsValidCSharpIdentifier(name) && name.Length > 0) { - name = "@" + name; + if (IsCSharpKeyword(name)) + { + name = "@" + name; + } + else + throw new ArgumentException(String.Format("Variable '{0}' from '{1}' is not a compatible C# identifier name\n\tAll variable names must conform to C# Language Specifications (refer to this StackOverflow answer: https://stackoverflow.com/a/950651/4036588)\n", name, parent.msgFileLocation.Path)); } + for (int i = 2; i < backup.Length; i++) otherstuff += " " + backup[i]; if (otherstuff.Contains('=')) From 70633743e56278576e457ccf6b66480774375b7b Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 18:15:30 -0400 Subject: [PATCH 05/10] YAMLParser: (experimentation) Allow strings to be constants --- YAMLParser/SingleType.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/YAMLParser/SingleType.cs b/YAMLParser/SingleType.cs index 80d9a0fe..178b3bd9 100644 --- a/YAMLParser/SingleType.cs +++ b/YAMLParser/SingleType.cs @@ -158,14 +158,16 @@ public void Finalize(MsgFile parent, string[] s, bool isliteral) string prefix = "", suffix = ""; if (isconst) { - if (!type.Equals("string", StringComparison.OrdinalIgnoreCase)) - { + // why can't strings be constants? + + //if (!type.Equals("string", StringComparison.OrdinalIgnoreCase)) + //{ if (KnownStuff.IsPrimitiveType(this)) prefix = "const "; else prefix = "static readonly "; wantsconstructor = false; - } + //} } string t = KnownStuff.GetNamespacedType(this, type); @@ -283,10 +285,12 @@ public void refinalize(MsgFile parent, string REALTYPE) string prefix = "", suffix = ""; if (isconst) { - if (!Type.Equals("string", StringComparison.OrdinalIgnoreCase)) - { + // why can't strings be constants? + + //if (!Type.Equals("string", StringComparison.OrdinalIgnoreCase)) + //{ prefix = "const "; - } + //} } if (otherstuff.Contains('=')) if (wantsconstructor) From 2e9478313a933bbe55aa5523ad40dc88bc8f13e6 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 18:16:04 -0400 Subject: [PATCH 06/10] YAMLParser: (attempt to) Fix KnownStuff.WhatItIs bug --- YAMLParser/KnownStuff.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/YAMLParser/KnownStuff.cs b/YAMLParser/KnownStuff.cs index fbc7169d..9bb45a4a 100644 --- a/YAMLParser/KnownStuff.cs +++ b/YAMLParser/KnownStuff.cs @@ -84,11 +84,16 @@ public static SingleType WhatItIs(MsgFile parent, string s, string extraindent) string package = null; // sometimes, a string can contain the char '/', such as the following line: // string CONTENT_JSON = "application/json" - if (pieces.Length == 2 && !s.ToLower().Contains("string")) + if (pieces.Length == 2) { + if (s.ToLower().Contains("string") && !MsgFile.resolver.ContainsKey(pieces[0])) + goto ResolvingStep; + package = pieces[0]; s = pieces[1]; } + + ResolvingStep: SingleType st = new SingleType(package, s, extraindent); parent.resolve(st); WhatItIs(parent, st); From ab65eea95c4ffca0f0ddfa4bc392778c577cf152 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 6 Aug 2018 18:23:12 -0400 Subject: [PATCH 07/10] Merge code to sort messages by priority --- YAMLParser/MsgFileLocator.cs | 16 ++++++++++++++++ YAMLParser/Program.cs | 2 ++ 2 files changed, 18 insertions(+) diff --git a/YAMLParser/MsgFileLocator.cs b/YAMLParser/MsgFileLocator.cs index e27d0502..172a5bc4 100644 --- a/YAMLParser/MsgFileLocator.cs +++ b/YAMLParser/MsgFileLocator.cs @@ -170,6 +170,22 @@ private static void explode(List m, List s, Li Console.WriteLine("Skipped " + (msgfiles.Length - (m.Count - mb4)) + " duplicate msgs and " + (srvfiles.Length - (s.Count - sb4)) + " duplicate srvs"); } + internal static int priority(string package) + { + switch (package) + { + case "std_msgs": return 1; + case "geometry_msgs": return 2; + case "actionlib_msgs": return 3; + default: return 9; + } + } + + internal static List sortMessages(List msgs) + { + return msgs.OrderBy(m => "" + priority(m.package) + m.package + m.basename).ToList(); + } + public static void findMessages(List msgs, List srvs, List actionFiles, params string[] args) { diff --git a/YAMLParser/Program.cs b/YAMLParser/Program.cs index 07f23f4c..37c05c75 100644 --- a/YAMLParser/Program.cs +++ b/YAMLParser/Program.cs @@ -129,6 +129,8 @@ private static void Run(List messageDirs, List assemblies = null Console.WriteLine("Looking in " + d); MsgFileLocator.findMessages(paths, pathssrv, actionFileLocations, d); } + // sort paths by priority + paths = MsgFileLocator.sortMessages(paths); // first pass: create all msg files (and register them in static resolver dictionary) var baseTypes = MessageTypeRegistry.Default.GetTypeNames().ToList(); From 0783778af9b0450892a7dc5be770d5b84b71bae3 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Tue, 7 Aug 2018 11:10:52 -0400 Subject: [PATCH 08/10] YAMLParser: Ensure Messages.dll has parsed required ROS packages for lib --- YAMLParser/Program.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/YAMLParser/Program.cs b/YAMLParser/Program.cs index 37c05c75..3992ff0b 100644 --- a/YAMLParser/Program.cs +++ b/YAMLParser/Program.cs @@ -21,6 +21,7 @@ internal class Program static List actionFiles = new List(); private static ILogger Logger { get; set; } const string DEFAULT_OUTPUT_FOLDERNAME = "Messages"; + static readonly string[] required_packages = {"std_msgs", "geometry_msgs", "actionlib_msgs", "sensor_msgs"}; public static void Main(params string[] args) { @@ -167,6 +168,12 @@ private static void Run(List messageDirs, List assemblies = null actionFiles = actionFileParser.GenerateRosMessageClasses(); //var actionFiles = new List(); + if (!StdMsgsProcessed()) // may seem obvious, but needed so that all other messages can build... + { + Console.WriteLine("Missing at least one of the following ROS packages: [\"" + String.Join("\", \"", required_packages) + "\"]. Exiting..."); + return; + } + if (paths.Count + pathssrv.Count > 0) { MakeTempDir(outputdir); @@ -188,6 +195,11 @@ private static void Run(List messageDirs, List assemblies = null } } + public static bool StdMsgsProcessed() + { + return required_packages.All(c => MsgFile.resolver.ContainsKey(c)); + } + private static void MakeTempDir(string outputdir) { if (!Directory.Exists(outputdir)) From 46e85ed355610591106f6753124a7f7b51943389 Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Tue, 7 Aug 2018 11:41:53 -0400 Subject: [PATCH 09/10] YAMLParser: comment out actionlib_msgs from required_packages check --- YAMLParser/Program.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/YAMLParser/Program.cs b/YAMLParser/Program.cs index 3992ff0b..0fea2f52 100644 --- a/YAMLParser/Program.cs +++ b/YAMLParser/Program.cs @@ -21,7 +21,7 @@ internal class Program static List actionFiles = new List(); private static ILogger Logger { get; set; } const string DEFAULT_OUTPUT_FOLDERNAME = "Messages"; - static readonly string[] required_packages = {"std_msgs", "geometry_msgs", "actionlib_msgs", "sensor_msgs"}; + static readonly string[] required_packages = {"std_msgs", "geometry_msgs", /*"actionlib_msgs",*/ "sensor_msgs"}; public static void Main(params string[] args) { @@ -170,7 +170,9 @@ private static void Run(List messageDirs, List assemblies = null if (!StdMsgsProcessed()) // may seem obvious, but needed so that all other messages can build... { + string resolvedPkgs = String.Join(", ", MsgFile.resolver.Keys.OrderBy(x => x.ToString()).ToArray()); Console.WriteLine("Missing at least one of the following ROS packages: [\"" + String.Join("\", \"", required_packages) + "\"]. Exiting..."); + Console.WriteLine("resolver's keys: [" + resolvedPkgs + "]"); return; } From 9782e7ef609cff9e61c6cdd8834833e4ca45790b Mon Sep 17 00:00:00 2001 From: YaseenAlk Date: Mon, 13 Aug 2018 18:16:24 -0400 Subject: [PATCH 10/10] YAMLParser: Check if field name == class name --- YAMLParser/SingleType.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/YAMLParser/SingleType.cs b/YAMLParser/SingleType.cs index 178b3bd9..09c21a0d 100644 --- a/YAMLParser/SingleType.cs +++ b/YAMLParser/SingleType.cs @@ -109,12 +109,14 @@ public void Finalize(MsgFile parent, string[] s, bool isliteral) otherstuff = " = " + parts[1]; } - if (!MsgFileLocation.IsValidCSharpIdentifier(name) && name.Length > 0) + if (name == parent.Name.Split(".").Last() || !MsgFileLocation.IsValidCSharpIdentifier(name) && name.Length > 0) { if (IsCSharpKeyword(name)) { name = "@" + name; } + else if (MsgFileLocation.IsValidCSharpIdentifier(name) && name == parent.Name.Split(".").Last()) + name = "_" + name; else throw new ArgumentException(String.Format("Variable '{0}' from '{1}' is not a compatible C# identifier name\n\tAll variable names must conform to C# Language Specifications (refer to this StackOverflow answer: https://stackoverflow.com/a/950651/4036588)\n", name, parent.msgFileLocation.Path)); } @@ -235,12 +237,14 @@ public void refinalize(MsgFile parent, string REALTYPE) otherstuff = " = " + parts[1]; } - if (!MsgFileLocation.IsValidCSharpIdentifier(name) && name.Length > 0) + if (name == parent.Name.Split(".").Last() || !MsgFileLocation.IsValidCSharpIdentifier(name) && name.Length > 0) { if (IsCSharpKeyword(name)) { name = "@" + name; } + else if (MsgFileLocation.IsValidCSharpIdentifier(name) && name == parent.Name.Split(".").Last()) + name = "_" + name; else throw new ArgumentException(String.Format("Variable '{0}' from '{1}' is not a compatible C# identifier name\n\tAll variable names must conform to C# Language Specifications (refer to this StackOverflow answer: https://stackoverflow.com/a/950651/4036588)\n", name, parent.msgFileLocation.Path)); }