Hideous but functional type generation using RobotOS.jl

Manifest.toml

@@ -1,3 +1,5 @@
+# This file is machine-generated - editing it directly is not advised
+
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
@@ -9,12 +11,12 @@ version = "0.8.10"
 
 [[BinaryProvider]]
 deps = ["Libdl", "Pkg", "SHA", "Test"]
-git-tree-sha1 = "b530fbeb6f41ab5a83fbe3db1fcbe879334bcd2d"
+git-tree-sha1 = "055eb2690182ebc31087859c3dd8598371d3ef9e"
 uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
-version = "0.4.2"
+version = "0.5.3"
 
 [[BufferedStreams]]
-deps = ["Compat", "Pkg", "Test"]
+deps = ["Compat", "Test"]
 git-tree-sha1 = "5d55b9486590fdda5905c275bb21ce1f0754020f"
 uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
 version = "1.0.0"
@@ -27,12 +29,18 @@ version = "0.5.0"
 
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
-git-tree-sha1 = "ae262fa91da6a74e8937add6b613f58cd56cdad4"
+git-tree-sha1 = "49269e311ffe11ac5b334681d212329002a9832a"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "1.1.0"
+version = "1.5.1"
+
+[[Conda]]
+deps = ["Compat", "JSON", "VersionParsing"]
+git-tree-sha1 = "b625d802587c2150c279a40a646fba63f9bd8187"
+uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d"
+version = "1.2.0"
 
 [[DataStructures]]
-deps = ["InteractiveUtils", "REPL", "Random", "Serialization", "Test"]
+deps = ["InteractiveUtils", "Random", "Serialization", "Test"]
 git-tree-sha1 = "5162c360c52e9265e0bfafb71415c31f746f7e04"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 version = "0.12.0"
@@ -46,11 +54,11 @@ deps = ["Mmap"]
 uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 
 [[Distributed]]
-deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"]
+deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[FastIOBuffers]]
-deps = ["Compat", "Pkg"]
+deps = ["Compat"]
 git-tree-sha1 = "dcdd29637670daffe4a017207fb919fb69160108"
 uuid = "6ccb8d0d-f5f8-5ae0-a392-f5aa62729a81"
 version = "0.0.2"
@@ -68,17 +76,23 @@ uuid = "0862f596-cf2d-50af-8ef4-f2be67dfa83f"
 version = "0.2.1"
 
 [[InteractiveUtils]]
-deps = ["LinearAlgebra", "Markdown"]
+deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
+[[JSON]]
+deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"]
+git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.20.0"
+
 [[LibCURL]]
-deps = ["BinaryProvider", "Compat", "Libdl", "Pkg", "Printf"]
-git-tree-sha1 = "ffb95ae47142be781d9f59af86b724d921acb6c1"
+deps = ["BinaryProvider", "Compat", "Libdl", "Printf"]
+git-tree-sha1 = "6339c87cb76923a3cf947fcd213cbc364355c9c9"
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
-version = "0.4.0"
+version = "0.4.1"
 
 [[LibExpat]]
-deps = ["Compat", "Pkg"]
+deps = ["Compat"]
 git-tree-sha1 = "fde352ec13479e2f90e57939da2440fb78c5e388"
 uuid = "522f3ed2-3f36-55e3-b6df-e94fee9b0c07"
 version = "0.5.0"
@@ -102,6 +116,12 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 [[Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
+[[MacroTools]]
+deps = ["Compat"]
+git-tree-sha1 = "c443e1c8d58a4e9f61b708ad0a88286c7042145b"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.4.4"
+
 [[Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -117,6 +137,12 @@ uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
+[[PyCall]]
+deps = ["Compat", "Conda", "MacroTools", "Statistics", "VersionParsing"]
+git-tree-sha1 = "ce4681263f41d98b3faf6107b28284bf8b866f5f"
+uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+version = "1.18.5"
+
 [[REPL]]
 deps = ["InteractiveUtils", "Markdown", "Sockets"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
@@ -125,6 +151,12 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
+[[RobotOS]]
+deps = ["PyCall", "Test"]
+git-tree-sha1 = "cd65ac0630c65661362bfced6ff0c7e25839bfd4"
+uuid = "22415677-39a4-5241-a37a-00beabbbdae8"
+version = "0.7.0"
+
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
@@ -163,7 +195,7 @@ uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
 version = "0.5.2"
 
 [[TranscodingStreams]]
-deps = ["DelimitedFiles", "Pkg", "Random", "Test"]
+deps = ["Pkg", "Random", "Test"]
 git-tree-sha1 = "a34a2d588e2d2825602bf14a24216d5c8b0921ec"
 uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
 version = "0.8.1"
@@ -175,12 +207,18 @@ uuid = "30578b45-9adc-5946-b283-645ec420af67"
 version = "0.4.0"
 
 [[UUIDs]]
-deps = ["Random"]
+deps = ["Random", "SHA"]
 uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [[Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
+[[VersionParsing]]
+deps = ["Compat"]
+git-tree-sha1 = "c9d5aa108588b978bd859554660c8a5c4f2f7669"
+uuid = "81def892-9a0e-5fdd-b105-ffc91e053289"
+version = "1.1.3"
+
 [[WinRPM]]
 deps = ["BinDeps", "Compat", "HTTPClient", "LibExpat", "Libdl", "Libz", "URIParser"]
 git-tree-sha1 = "2a889d320f3b77d17c037f295859fe570133cfbf"

Project.toml

@@ -9,6 +9,8 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 FastIOBuffers = "6ccb8d0d-f5f8-5ae0-a392-f5aa62729a81"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+RobotOS = "22415677-39a4-5241-a37a-00beabbbdae8"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624"
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"

src/bag.jl

@@ -1,6 +1,7 @@
 
 using FastIOBuffers: FastReadBuffer
 using DataStructures: counter
+using RobotOS
 
 """ A ROS Bag version X.Y """
 abstract type Bag{X,Y} end
@@ -29,27 +30,79 @@ index!(bag::ChunkedBag, io::IO, rec::Chunk) = begin
     skip(io, rec.data_len)
 end
 
+function _attempt_lookup(pkgstr, typstr, msg_types)
+    pkg, typ = Symbol(pkgstr), Symbol(typstr)
+    for type_mod in msg_types
+        return try
+            package = getproperty(type_mod, pkg)
+            getproperty(package, typ)
+        catch
+            continue
+        end
+    end
+    nothing
+end
+
 function setup_maps!(bag::ChunkedBag, msg_types...)
+    # we only want to call rostypegen() from RobotOS once...so keep
+    # track of the types we didn't find in "missing"
+    missing = Dict()
     for conn in bag.connections
         # (a) map connection id -> topic
         bag.topic_map[conn.conn] = conn.topic
 
         # (b) map connection id -> message type
         type_str = conn.header[:type] # eg. "sensor_msgs/Imu"
-        pkg, typ = Symbol.(split(type_str, "/"))
+        pkgstr, typstr = string.(split(type_str, '/'))
+
         # Do we have a module that implements this message type?
         # Look for eg. type_mod.sensor_msgs.Imu
-        for type_mod in msg_types
-            dtype = try
-                package = getproperty(type_mod, pkg)
-                getproperty(package, typ)
+        dtype = _attempt_lookup(pkgstr, typstr, msg_types)
+        if dtype != nothing
+            bag.type_map[conn.conn] = dtype # found it
+        else
+            # mark it for RobotOS to generate
+            missing[conn.conn] = (pkgstr, typstr)
+        end
+    end
+
+    # try to find the missing types and mark them in
+    # "found" if successful. Fails if the message definition
+    # isn't found in PYTHONPATH.
+    found = Dict()
+    for conn in keys(missing)
+        try
+            pkgstr, typstr = missing[conn]
+            RobotOS._rosimport(pkgstr, true, typstr)
+            found[conn] = missing[conn]
+        catch
+            continue
+        end
+    end
+
+    # at this point
+    # 1) the type was not in "msg_types"
+    # 2) "found" contains all the types that
+    #    RobotOS can generate, so generate them.
+    rostypegen(@__MODULE__) # causes reimport warnings, how to just build a single type?
+    for conn in keys(found)
+        try
+            pkgstr, typstr = found[conn]
+            # "found" may contain duplicate type that were generated
+            # in this loop, so first we see if the type exists,
+            # otherwise import it
+            mod = try
+                eval(Meta.parse("$pkgstr.msg.$typstr"))
             catch
-                continue
+                eval(Meta.parse("import .$pkgstr.msg"))
+                eval(Meta.parse("$pkgstr.msg.$typstr"))
             end
-            bag.type_map[conn.conn] = dtype # found it
-            break
+            bag.type_map[conn] = mod
+        catch
+            continue
         end
     end
+    bag
 end
 
 """ FileIO loader """

src/messages.jl

@@ -1,17 +1,22 @@
-
+__precompile__(false)
 module Messages
 
 using StaticArrays: SVector
 using FastIOBuffers: FastReadBuffer
 
+using RobotOS
+
 
 """
 Marks compound types that can be deserialised from a ROS datastream.
 All fields must be Readable or Bits, or Vectors/SVectors of these.
 """
 abstract type Readable end
+const MegaReadable = Union{Readable, RobotOS.AbstractMsg, RobotOS.AbstractTime}
+#const MegaReadable = Readable
+
 
-@generated function Base.read(io::IO, ::Type{T}) where {T<:Readable}
+@generated function Base.read(io::IO, ::Type{T}) where {T<:MegaReadable}
     statements = Expr[]
     for (name, E) in zip(fieldnames(T), T.types)
         push!(statements, :($name = read_field(io, $E)))
@@ -24,7 +29,7 @@ abstract type Readable end
 end
 
 """ Readable values can be read with Base.read(). """
-read_field(io::IO, ::Type{T}) where {T<:Readable} = read(io, T)
+read_field(io::IO, ::Type{T}) where {T<:MegaReadable} = read(io, T)
 
 const Bits = Union{Bool,Signed,Unsigned,Float32,Float64}