Merge branch 'bleed' into rv-engine

OpenRA.Game/ExternalMods.cs

@@ -66,6 +66,7 @@ public ExternalMods()
 			// Several types of support directory types are available, depending on
 			// how the player has installed and launched the game.
 			// Read registration metadata from all of them
+			var stringPool = new HashSet<string>(); // Reuse common strings in YAML
 			foreach (var source in GetSupportDirs(ModRegistration.User | ModRegistration.System))
 			{
 				var metadataPath = Path.Combine(source, "ModMetadata");
@@ -76,7 +77,7 @@ public ExternalMods()
 				{
 					try
 					{
-						var yaml = MiniYaml.FromStream(File.OpenRead(path), path).First().Value;
+						var yaml = MiniYaml.FromFile(path, stringPool: stringPool).First().Value;
 						LoadMod(yaml, path);
 					}
 					catch (Exception e)
@@ -205,7 +206,7 @@ internal void ClearInvalidRegistrations(ModRegistration registration)
 					string modKey = null;
 					try
 					{
-						var yaml = MiniYaml.FromStream(File.OpenRead(path), path).First().Value;
+						var yaml = MiniYaml.FromFile(path).First().Value;
 						var m = FieldLoader.Load<ExternalMod>(yaml);
 						modKey = ExternalMod.MakeKey(m);
 

OpenRA.Game/Exts.cs

@@ -130,6 +130,20 @@ public static V GetOrAdd<K, V>(this Dictionary<K, V> d, K k, Func<K, V> createFn
 			return ret;
 		}
 
+		public static T GetOrAdd<T>(this HashSet<T> set, T value)
+		{
+			if (!set.TryGetValue(value, out var ret))
+				set.Add(ret = value);
+			return ret;
+		}
+
+		public static T GetOrAdd<T>(this HashSet<T> set, T value, Func<T, T> createFn)
+		{
+			if (!set.TryGetValue(value, out var ret))
+				set.Add(ret = createFn(value));
+			return ret;
+		}
+
 		public static int IndexOf<T>(this T[] array, T value)
 		{
 			return Array.IndexOf(array, value);

OpenRA.Game/FileFormats/ReplayMetadata.cs

@@ -48,7 +48,7 @@ public ReplayMetadata(GameInformation info)
 
 			// Read game info (max 100K limit as a safeguard against corrupted files)
 			var data = fs.ReadLengthPrefixedString(Encoding.UTF8, 1024 * 100);
-			GameInfo = GameInformation.Deserialize(data);
+			GameInfo = GameInformation.Deserialize(data, path);
 		}
 
 		public void Write(BinaryWriter writer)

OpenRA.Game/GameInformation.cs

@@ -49,13 +49,13 @@ public GameInformation()
 			playersByRuntime = new Dictionary<OpenRA.Player, Player>();
 		}
 
-		public static GameInformation Deserialize(string data)
+		public static GameInformation Deserialize(string data, string path)
 		{
 			try
 			{
 				var info = new GameInformation();
 
-				var nodes = MiniYaml.FromString(data);
+				var nodes = MiniYaml.FromString(data, path);
 				foreach (var node in nodes)
 				{
 					var keyParts = node.Key.Split('@');

OpenRA.Game/Graphics/ChromeProvider.cs

@@ -77,8 +77,9 @@ public static void Initialize(ModData modData)
 			cachedPanelSprites = new Dictionary<string, Sprite[]>();
 			cachedCollectionSheets = new Dictionary<Collection, (Sheet, int)>();
 
+			var stringPool = new HashSet<string>(); // Reuse common strings in YAML
 			var chrome = MiniYaml.Merge(modData.Manifest.Chrome
-				.Select(s => MiniYaml.FromStream(fileSystem.Open(s), s)));
+				.Select(s => MiniYaml.FromStream(fileSystem.Open(s), s, stringPool: stringPool)));
 
 			foreach (var c in chrome)
 				if (!c.Key.StartsWith('^'))

OpenRA.Game/Graphics/CursorProvider.cs

@@ -24,8 +24,9 @@ public sealed class CursorProvider
 		public CursorProvider(ModData modData)
 		{
 			var fileSystem = modData.DefaultFileSystem;
+			var stringPool = new HashSet<string>(); // Reuse common strings in YAML
 			var sequenceYaml = MiniYaml.Merge(modData.Manifest.Cursors.Select(
-				s => MiniYaml.FromStream(fileSystem.Open(s), s)));
+				s => MiniYaml.FromStream(fileSystem.Open(s), s, stringPool: stringPool)));
 
 			var cursorsYaml = new MiniYaml(null, sequenceYaml).NodeWithKey("Cursors").Value;
 

OpenRA.Game/LocalPlayerProfile.cs

@@ -84,10 +84,11 @@ public void RefreshPlayerData(Action onComplete = null)
 				{
 					var client = HttpClientFactory.Create();
 
-					var httpResponseMessage = await client.GetAsync(playerDatabase.Profile + Fingerprint);
+					var url = playerDatabase.Profile + Fingerprint;
+					var httpResponseMessage = await client.GetAsync(url);
 					var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 
-					var yaml = MiniYaml.FromStream(result).First();
+					var yaml = MiniYaml.FromStream(result, url).First();
 					if (yaml.Key == "Player")
 					{
 						innerData = FieldLoader.Load<PlayerProfile>(yaml.Value);

OpenRA.Game/Manifest.cs

@@ -94,7 +94,8 @@ public Manifest(string modId, IReadOnlyPackage package)
 			Id = modId;
 			Package = package;
 
-			var nodes = MiniYaml.FromStream(package.GetStream("mod.yaml"), "mod.yaml");
+			var stringPool = new HashSet<string>(); // Reuse common strings in YAML
+			var nodes = MiniYaml.FromStream(package.GetStream("mod.yaml"), $"{package.Name}:mod.yaml", stringPool: stringPool);
 			for (var i = nodes.Count - 1; i >= 0; i--)
 			{
 				if (nodes[i].Key != "Include")
@@ -107,7 +108,7 @@ public Manifest(string modId, IReadOnlyPackage package)
 					throw new YamlException($"{nodes[i].Location}: File `{filename}` not found.");
 
 				nodes.RemoveAt(i);
-				nodes.InsertRange(i, MiniYaml.FromStream(contents, filename));
+				nodes.InsertRange(i, MiniYaml.FromStream(contents, $"{package.Name}:{filename}", stringPool: stringPool));
 			}
 
 			// Merge inherited overrides

OpenRA.Game/Map/CellCoordsRegion.cs

@@ -0,0 +1,90 @@
+#region Copyright & License Information
+/*
+ * Copyright (c) The OpenRA Developers and Contributors
+ * This file is part of OpenRA, which is free software. It is made
+ * available to you under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version. For more
+ * information, see COPYING.
+ */
+#endregion
+
+using System.Collections;
+using System.Collections.Generic;
+
+namespace OpenRA
+{
+	public readonly struct CellCoordsRegion : IEnumerable<CPos>
+	{
+		public struct CellCoordsEnumerator : IEnumerator<CPos>
+		{
+			readonly CellCoordsRegion r;
+
+			public CellCoordsEnumerator(CellCoordsRegion region)
+				: this()
+			{
+				r = region;
+				Reset();
+			}
+
+			public bool MoveNext()
+			{
+				var x = Current.X + 1;
+				var y = Current.Y;
+
+				// Check for column overflow.
+				if (x > r.BottomRight.X)
+				{
+					y++;
+					x = r.TopLeft.X;
+
+					// Check for row overflow.
+					if (y > r.BottomRight.Y)
+						return false;
+				}
+
+				Current = new CPos(x, y);
+				return true;
+			}
+
+			public void Reset()
+			{
+				Current = new CPos(r.TopLeft.X - 1, r.TopLeft.Y);
+			}
+
+			public CPos Current { get; private set; }
+
+			readonly object IEnumerator.Current => Current;
+			public readonly void Dispose() { }
+		}
+
+		public CellCoordsRegion(CPos topLeft, CPos bottomRight)
+		{
+			TopLeft = topLeft;
+			BottomRight = bottomRight;
+		}
+
+		public override string ToString()
+		{
+			return $"{TopLeft}->{BottomRight}";
+		}
+
+		public CellCoordsEnumerator GetEnumerator()
+		{
+			return new CellCoordsEnumerator(this);
+		}
+
+		IEnumerator<CPos> IEnumerable<CPos>.GetEnumerator()
+		{
+			return GetEnumerator();
+		}
+
+		IEnumerator IEnumerable.GetEnumerator()
+		{
+			return GetEnumerator();
+		}
+
+		public CPos TopLeft { get; }
+		public CPos BottomRight { get; }
+	}
+}

OpenRA.Game/Map/CellRegion.cs

@@ -102,6 +102,7 @@ public bool Contains(CPos cell)
 		}
 
 		public MapCoordsRegion MapCoords => new(mapTopLeft, mapBottomRight);
+		public CellCoordsRegion CellCoords => new(TopLeft, BottomRight);
 
 		public CellRegionEnumerator GetEnumerator()
 		{

OpenRA.Game/Map/Map.cs

@@ -361,7 +361,7 @@ public Map(ModData modData, IReadOnlyPackage package)
 			if (!Package.Contains("map.yaml") || !Package.Contains("map.bin"))
 				throw new InvalidDataException($"Not a valid map\n File: {package.Name}");
 
-			var yaml = new MiniYaml(null, MiniYaml.FromStream(Package.GetStream("map.yaml"), package.Name));
+			var yaml = new MiniYaml(null, MiniYaml.FromStream(Package.GetStream("map.yaml"), $"{package.Name}:map.yaml"));
 			foreach (var field in YamlFields)
 				field.Deserialize(this, yaml);
 

OpenRA.Game/Map/MapCache.cs

@@ -38,7 +38,7 @@ public sealed class MapCache : IEnumerable<MapPreview>, IDisposable
 		readonly object syncRoot = new();
 		readonly Queue<MapPreview> generateMinimap = new();
 
-		public Dictionary<string, string> StringPool { get; } = new Dictionary<string, string>();
+		public HashSet<string> StringPool { get; } = new();
 
 		readonly List<MapDirectoryTracker> mapDirectoryTrackers = new();
 
@@ -238,6 +238,7 @@ public void QueryRemoteMapDetails(string repositoryUrl, IEnumerable<string> uids
 			Task.Run(async () =>
 			{
 				var client = HttpClientFactory.Create();
+				var stringPool = new HashSet<string>(); // Reuse common strings in YAML
 
 				// Limit each query to 50 maps at a time to avoid request size limits
 				for (var i = 0; i < queryUids.Count; i += 50)
@@ -249,7 +250,7 @@ public void QueryRemoteMapDetails(string repositoryUrl, IEnumerable<string> uids
 						var httpResponseMessage = await client.GetAsync(url);
 						var result = await httpResponseMessage.Content.ReadAsStreamAsync();
 
-						var yaml = MiniYaml.FromStream(result);
+						var yaml = MiniYaml.FromStream(result, url, stringPool: stringPool);
 						foreach (var kv in yaml)
 							previews[kv.Key].UpdateRemoteSearch(MapStatus.DownloadAvailable, kv.Value, modData.Manifest.MapCompatibility, mapDetailsReceived);
 

OpenRA.Game/Map/MapPreview.cs

@@ -140,9 +140,10 @@ public void SetCustomRules(ModData modData, IReadOnlyFileSystem fileSystem, Dict
 							files = files.Append(mapFiles);
 						}
 
+						var stringPool = new HashSet<string>(); // Reuse common strings in YAML
 						var sources =
 							modDataRules.Select(x => x.Where(IsLoadableRuleDefinition).ToList())
-							.Concat(files.Select(s => MiniYaml.FromStream(fileSystem.Open(s), s).Where(IsLoadableRuleDefinition).ToList()));
+							.Concat(files.Select(s => MiniYaml.FromStream(fileSystem.Open(s), s, stringPool: stringPool).Where(IsLoadableRuleDefinition).ToList()));
 						if (RuleDefinitions.Nodes.Length > 0)
 							sources = sources.Append(RuleDefinitions.Nodes.Where(IsLoadableRuleDefinition).ToList());
 
@@ -334,7 +335,7 @@ public void UpdateFromMap(IReadOnlyPackage p, IReadOnlyPackage parent, MapClassi
 				if (yamlStream == null)
 					throw new FileNotFoundException("Required file map.yaml not present in this map");
 
-				yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, "map.yaml", stringPool: cache.StringPool)).ToDictionary();
+				yaml = new MiniYaml(null, MiniYaml.FromStream(yamlStream, $"{p.Name}:map.yaml", stringPool: cache.StringPool)).ToDictionary();
 			}
 
 			Package = p;
@@ -475,10 +476,12 @@ public void UpdateRemoteSearch(MapStatus status, MiniYaml yaml, string[] mapComp
 					}
 
 					var playersString = Encoding.UTF8.GetString(Convert.FromBase64String(r.players_block));
-					newData.Players = new MapPlayers(MiniYaml.FromString(playersString));
+					newData.Players = new MapPlayers(MiniYaml.FromString(playersString,
+						$"{yaml.NodeWithKey(nameof(r.players_block)).Location.Name}:{nameof(r.players_block)}"));
 
 					var rulesString = Encoding.UTF8.GetString(Convert.FromBase64String(r.rules));
-					var rulesYaml = new MiniYaml("", MiniYaml.FromString(rulesString)).ToDictionary();
+					var rulesYaml = new MiniYaml("", MiniYaml.FromString(rulesString,
+						$"{yaml.NodeWithKey(nameof(r.rules)).Location.Name}:{nameof(r.rules)}")).ToDictionary();
 					newData.SetCustomRules(modData, this, rulesYaml, null);
 
 					// Map is for a different mod: update its information so it can be displayed

OpenRA.Game/MiniYaml.cs

@@ -61,16 +61,16 @@ public sealed class MiniYamlNode
 	{
 		public readonly struct SourceLocation
 		{
-			public readonly string Filename;
+			public readonly string Name;
 			public readonly int Line;
 
-			public SourceLocation(string filename, int line)
+			public SourceLocation(string name, int line)
 			{
-				Filename = filename;
+				Name = name;
 				Line = line;
 			}
 
-			public override string ToString() { return $"{Filename}:{Line}"; }
+			public override string ToString() { return $"{Name}:{Line}"; }
 		}
 
 		public readonly SourceLocation Location;
@@ -203,9 +203,13 @@ public MiniYaml(string value, IEnumerable<MiniYamlNode> nodes)
 			Nodes = ImmutableArray.CreateRange(nodes);
 		}
 
-		static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, string filename, bool discardCommentsAndWhitespace, Dictionary<string, string> stringPool)
+		static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, string name, bool discardCommentsAndWhitespace, HashSet<string> stringPool)
 		{
-			stringPool ??= new Dictionary<string, string>();
+			// YAML config often contains repeated strings for key, values, comments.
+			// Pool these strings so we only need one copy of each unique string.
+			// This saves on long-term memory usage as parsed values can often live a long time.
+			// A caller can also provide a pool as input, allowing de-duplication across multiple parses.
+			stringPool ??= new HashSet<string>();
 
 			var result = new List<List<MiniYamlNode>>
 			{
@@ -227,7 +231,7 @@ static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, str
 				ReadOnlySpan<char> key = default;
 				ReadOnlySpan<char> value = default;
 				ReadOnlySpan<char> comment = default;
-				var location = new MiniYamlNode.SourceLocation(filename, lineNo);
+				var location = new MiniYamlNode.SourceLocation(name, lineNo);
 
 				if (line.Length > 0)
 				{
@@ -330,9 +334,9 @@ static List<MiniYamlNode> FromLines(IEnumerable<ReadOnlyMemory<char>> lines, str
 					// (i.e. a lone # at the end of a line) can be correctly re-serialized
 					var commentString = comment == default ? null : comment.ToString();
 
-					keyString = keyString == null ? null : stringPool.GetOrAdd(keyString, keyString);
-					valueString = valueString == null ? null : stringPool.GetOrAdd(valueString, valueString);
-					commentString = commentString == null ? null : stringPool.GetOrAdd(commentString, commentString);
+					keyString = keyString == null ? null : stringPool.GetOrAdd(keyString);
+					valueString = valueString == null ? null : stringPool.GetOrAdd(valueString);
+					commentString = commentString == null ? null : stringPool.GetOrAdd(commentString);
 
 					parsedLines.Add((level, keyString, valueString, commentString, location));
 				}
@@ -376,19 +380,19 @@ void BuildCompletedSubNode(int level)
 			}
 		}
 
-		public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
+		public static List<MiniYamlNode> FromFile(string path, bool discardCommentsAndWhitespace = true, HashSet<string> stringPool = null)
 		{
 			return FromStream(File.OpenRead(path), path, discardCommentsAndWhitespace, stringPool);
 		}
 
-		public static List<MiniYamlNode> FromStream(Stream s, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
+		public static List<MiniYamlNode> FromStream(Stream s, string name, bool discardCommentsAndWhitespace = true, HashSet<string> stringPool = null)
 		{
-			return FromLines(s.ReadAllLinesAsMemory(), fileName, discardCommentsAndWhitespace, stringPool);
+			return FromLines(s.ReadAllLinesAsMemory(), name, discardCommentsAndWhitespace, stringPool);
 		}
 
-		public static List<MiniYamlNode> FromString(string text, string fileName = "<no filename available>", bool discardCommentsAndWhitespace = true, Dictionary<string, string> stringPool = null)
+		public static List<MiniYamlNode> FromString(string text, string name, bool discardCommentsAndWhitespace = true, HashSet<string> stringPool = null)
 		{
-			return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), fileName, discardCommentsAndWhitespace, stringPool);
+			return FromLines(text.Split(new[] { "\r\n", "\n" }, StringSplitOptions.None).Select(s => s.AsMemory()), name, discardCommentsAndWhitespace, stringPool);
 		}
 
 		public static List<MiniYamlNode> Merge(IEnumerable<IReadOnlyCollection<MiniYamlNode>> sources)
@@ -605,7 +609,8 @@ public static List<MiniYamlNode> Load(IReadOnlyFileSystem fileSystem, IEnumerabl
 				files = files.Append(mapFiles);
 			}
 
-			IEnumerable<IReadOnlyCollection<MiniYamlNode>> yaml = files.Select(s => FromStream(fileSystem.Open(s), s));
+			var stringPool = new HashSet<string>(); // Reuse common strings in YAML
+			IEnumerable<IReadOnlyCollection<MiniYamlNode>> yaml = files.Select(s => FromStream(fileSystem.Open(s), s, stringPool: stringPool));
 			if (mapRules != null && mapRules.Nodes.Length > 0)
 				yaml = yaml.Append(mapRules.Nodes);