From b638bef1e3e001aa0e75a3e4ba512ac334045f65 Mon Sep 17 00:00:00 2001 From: Nemesh Date: Tue, 25 Apr 2023 22:46:04 +0300 Subject: [PATCH] Reimplemented ItemsParser Because of cloudflare, I had to switch to parsing game files for getting skins information. Schema didn't change, everything is the same as before if you don't compare the order of collections. --- FloatTool.sln | 2 +- ItemsParser/ItemParser.cs | 344 +++++++++++++++++++++++++++++++++ ItemsParser/ItemsParser.csproj | 1 - ItemsParser/Program.cs | 182 +++++------------ ItemsParser/Structs.cs | 59 ++++-- ItemsParser/Utils.cs | 40 ++-- ItemsParser/VdfConvert.cs | 140 ++++++++++++++ 7 files changed, 598 insertions(+), 170 deletions(-) create mode 100644 ItemsParser/ItemParser.cs create mode 100644 ItemsParser/VdfConvert.cs diff --git a/FloatTool.sln b/FloatTool.sln index 3311157..3cf68cf 100644 --- a/FloatTool.sln +++ b/FloatTool.sln @@ -5,7 +5,7 @@ VisualStudioVersion = 17.1.32203.90 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FloatTool", "FloatTool\FloatTool.csproj", "{B1434F55-DAE9-4CC5-9460-8D20D29D1802}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ItemsParser", "ItemsParser\ItemsParser.csproj", "{C5DE4078-D17D-45F7-95DC-C60747F8AABA}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ItemsParser", "ItemsParser\ItemsParser.csproj", "{C5DE4078-D17D-45F7-95DC-C60747F8AABA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/ItemsParser/ItemParser.cs b/ItemsParser/ItemParser.cs new file mode 100644 index 0000000..4b9200b --- /dev/null +++ b/ItemsParser/ItemParser.cs @@ -0,0 +1,344 @@ +/* +- Copyright(C) 2023 Prevter +- +- This program is free software: you can redistribute it and/or modify +- it under the terms of the GNU General Public License as published by +- the Free Software Foundation, either version 3 of the License, or +- (at your option) any later version. +- +- This program is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +- GNU General Public License for more details. +- +- You should have received a copy of the GNU General Public License +- along with this program. If not, see . +*/ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace ItemsParser +{ + public sealed class ItemParser + { + // I hate valve for this... + string[] ExcludeStattrak = new string[] + { + "Anubis Collection Package" + }; + + Dictionary items_game; + Dictionary language; + + List? paintKits; + List? lootLists; + Dictionary? rarities; + Dictionary translatedWeapons = new(); + + Regex weaponPaintKitRegex = new(@"\[([a-zA-Z0-9_-]{1,})\]([a-z0-9_]{1,})"); + + public ItemParser(dynamic items_game, dynamic language) + { + this.items_game = items_game; + this.language = language; + } + + public string GetTranslation(string key) + { + if (key.StartsWith('#')) + key = key[1..]; + + key = key.ToLower(); + if (language["lang"]["Tokens"].ContainsKey(key)) + return language["lang"]["Tokens"][key]; + + return key; + } + + public List GetPaintKits() + { + if (paintKits is not null) + return paintKits; + + paintKits = new(); + foreach (var key in ((Dictionary)items_game["items_game"]["paint_kits"]).Keys) + { + Dictionary paint_kit = items_game["items_game"]["paint_kits"][key]; + var paintKitObj = new PaintKit() + { + Id = key + }; + + if (paint_kit.ContainsKey("name")) + paintKitObj.Name = paint_kit["name"] as string; + + if (paint_kit.ContainsKey("description_tag")) + paintKitObj.PaintKitName = GetTranslation(paint_kit["description_tag"] as string); + + if (paint_kit.ContainsKey("wear_remap_min")) + paintKitObj.WearRemapMin = double.Parse((string)paint_kit["wear_remap_min"], CultureInfo.InvariantCulture); + + if (paint_kit.ContainsKey("wear_remap_max")) + paintKitObj.WearRemapMax = double.Parse((string)paint_kit["wear_remap_max"], CultureInfo.InvariantCulture); + + paintKits.Add(paintKitObj); + } + return paintKits; + } + + public PaintKit? FindByName(string name) + { + List paints = GetPaintKits(); + foreach (var paint in paints) + if (paint.Name == name) + return paint; + return null; + } + + public string GetTranslatedWeaponName(string weapon) + { + if (translatedWeapons.ContainsKey(weapon)) + return translatedWeapons[weapon]; + + foreach (var key in ((Dictionary)items_game["items_game"]["prefabs"]).Keys) + { + Dictionary item = items_game["items_game"]["prefabs"][key]; + if (item.ContainsKey("anim_class") && item["anim_class"] as string == weapon) + { + string translated = GetTranslation(item["item_name"] as string); + translatedWeapons.Add(weapon, translated); + return translated; + } + + if (item.ContainsKey("item_class") && item["item_class"] as string == weapon) + { + string translated = GetTranslation(item["item_name"] as string); + translatedWeapons.Add(weapon, translated); + return translated; + } + } + + return weapon; + } + + public List GetCollections() + { + List collections = new(); + foreach (var key in ((Dictionary)items_game["items_game"]["item_sets"]).Keys) + { + Dictionary item_set = items_game["items_game"]["item_sets"][key]; + List skins = new(); + + foreach (var skin in ((Dictionary)item_set["items"]).Keys) + { + if (weaponPaintKitRegex.Match(skin).Success) + skins.Add(skin); + } + + if (!skins.Any()) continue; + + collections.Add(new SkinSet + { + Id = key, + Name = GetTranslation(item_set["name"] as string), + Skins = skins + }); + } + + return collections; + } + + public Dictionary GetRarities() + { + if (rarities is not null) + return rarities; + + rarities = new(); + foreach (var key in ((Dictionary)items_game["items_game"]["rarities"]).Keys) + { + Dictionary rarity = items_game["items_game"]["rarities"][key]; + rarities.Add(key, GetTranslation(rarity["loc_key_weapon"] as string).Replace(" Grade", "")); + } + return rarities; + } + + public List GetLootLists() + { + if (lootLists is not null) + return lootLists; + + List _parseLootTable(string loot_table_id) + { + // Check if the loot table exists + if (!items_game["items_game"]["client_loot_lists"].ContainsKey(loot_table_id)) + return new List(); + + // Get the loot table + var loot_table = items_game["items_game"]["client_loot_lists"][loot_table_id]; + + // Get the items in the loot table + List items = new(); + foreach (var item_id in loot_table.Keys) + items.Add(item_id); + + return items; + } + + lootLists = new(); + foreach (var key in ((Dictionary)items_game["items_game"]["revolving_loot_lists"]).Keys) + { + var loot_list = items_game["items_game"]["revolving_loot_lists"][key]; + List items = new(); + + if (items_game["items_game"]["client_loot_lists"].ContainsKey(loot_list)) + { + var loot_table = items_game["items_game"]["client_loot_lists"][loot_list]; + foreach (var loot_id in loot_table.Keys) + { + var loot = _parseLootTable(loot_id); + if (loot.Count > 0) + items.AddRange(loot); + else + items.Add(loot_id); + } + } + + lootLists.Add(new SkinSet + { + Id = key, + Name = loot_list, + Skins = items + }); + } + return lootLists; + } + + public CaseSet? FindCaseSet(string collection_id) + { + foreach (var key in ((Dictionary)items_game["items_game"]["items"]).Keys) + { + Dictionary item = items_game["items_game"]["items"][key]; + if (item.ContainsKey("prefab") && item["prefab"].Contains("weapon_case") && item.ContainsKey("tags") + && item.ContainsKey("attributes") && !item["attributes"].ContainsKey("tournament event id") + && item["tags"].ContainsKey("ItemSet") && item["tags"]["ItemSet"]["tag_value"] == collection_id) + { + CaseSet caseSet = new(); + if (item.ContainsKey("id")) caseSet.Id = item["id"]; + else caseSet.Id = item["attributes"]["set supply crate series"]["value"]; + + caseSet.ImageInventory = item["image_inventory"].Split("/")[2]; + caseSet.Name = item["name"]; + caseSet.ItemName = item["item_name"]; + return caseSet; + } + } + return null; + } + + public string FindRarity(string item, string paintKit) + { + // find which item in client_loot_lists contains the item + foreach (var key in ((Dictionary)items_game["items_game"]["client_loot_lists"]).Keys) + { + Dictionary loot_list = items_game["items_game"]["client_loot_lists"][key]; + if (loot_list.ContainsKey(item)) + return GetRarities()[key.Split('_').Last()]; + } + // if we can't find the rarity, find it for the paintkit + if (items_game["items_game"]["paint_kits_rarity"].ContainsKey(paintKit)) + { + return GetRarities()[items_game["items_game"]["paint_kits_rarity"][paintKit] as string]; + } + return "common"; + } + + public int CompareRarity(string a, string b) + { + var rarities = GetRarities().Values.ToList(); + return rarities.IndexOf(a) - rarities.IndexOf(b); + } + + public List ComposeData() + { + List collections = new(); + + foreach (var collection in GetCollections()) + { + Console.WriteLine($"Processing collection {collection.Name}..."); + + Collection collectionObj = new(); + + var caseSet = FindCaseSet(collection.Id); + if (caseSet is null) + { + collectionObj.Name = collection.Name; + collectionObj.Link = $"https://csgostash.com/collection/{collection.Name.Replace(' ', '+')}"; + } + else + { + collectionObj.Name = GetTranslation(caseSet.ItemName); + + // Check for exclude list: + if (!ExcludeStattrak.Contains(collectionObj.Name)) + collectionObj.CanBeStattrak = true; + + var set = GetLootLists().FirstOrDefault(set => set.Name == caseSet.Name || set.Name == caseSet.ImageInventory, null); + var set_id = set is null ? caseSet.Id : set.Id; + collectionObj.Link = $"https://csgostash.com/case/{set_id}/{collection.Name.Replace(' ', '-')}"; + } + + string lowestRarity = "Contraband"; + string highestRarity = "Stock"; + + foreach (var skin in collection.Skins) + { + Match match = weaponPaintKitRegex.Match(skin); + string weapon = match.Groups[2].Value; + string paintKitName = match.Groups[1].Value; + PaintKit paintKit = FindByName(paintKitName)!; + string rarity = FindRarity(skin, paintKitName); + + if (CompareRarity(rarity, lowestRarity) < 0) + lowestRarity = rarity; + + if (CompareRarity(rarity, highestRarity) > 0) + highestRarity = rarity; + + collectionObj.Skins.Add(new Skin + { + Name = $"{GetTranslatedWeaponName(weapon)} | {paintKit.PaintKitName}", + Rarity = rarity, + MinWear = (float)paintKit.WearRemapMin, + MaxWear = (float)paintKit.WearRemapMax, + }); + } + + collectionObj.HighestRarity = highestRarity; + collectionObj.LowestRarity = lowestRarity; + + collections.Add(collectionObj); + } + + collections.Reverse(); + + return collections; + } + + + public sealed class SkinSet + { + public string Id { get; set; } = ""; + public string Name { get; set; } = ""; + public List Skins { get; set; } = new(); + } + + public sealed class CaseSet + { + public string Id { get; set; } = ""; + public string ItemName { get; set; } = ""; + public string Name { get; set; } = ""; + public string ImageInventory { get; set; } = ""; + } + + } +} diff --git a/ItemsParser/ItemsParser.csproj b/ItemsParser/ItemsParser.csproj index 4e7e10b..7d211a9 100644 --- a/ItemsParser/ItemsParser.csproj +++ b/ItemsParser/ItemsParser.csproj @@ -8,7 +8,6 @@ - diff --git a/ItemsParser/Program.cs b/ItemsParser/Program.cs index 296d1a3..76199e7 100644 --- a/ItemsParser/Program.cs +++ b/ItemsParser/Program.cs @@ -1,153 +1,57 @@ -using HtmlAgilityPack; +/* +- Copyright(C) 2023 Prevter +- +- This program is free software: you can redistribute it and/or modify +- it under the terms of the GNU General Public License as published by +- the Free Software Foundation, either version 3 of the License, or +- (at your option) any later version. +- +- This program is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +- GNU General Public License for more details. +- +- You should have received a copy of the GNU General Public License +- along with this program. If not, see . +*/ using ItemsParser; using Newtonsoft.Json; -using System.Diagnostics; -using System.Globalization; -using System.Web; -if (!Directory.Exists("cache")) - Directory.CreateDirectory("cache"); - -if (args.Contains("--redownload") || args.Contains("-r") || !File.Exists("cache/index.html")) +// Download game files +if (args.Contains("--redownload") || args.Contains("-r") +|| !File.Exists("items_game.vdf") || !File.Exists("csgo_english.vdf")) { - await Utils.DownloadFile("https://csgostash.com/", "cache/index.html"); + Console.WriteLine("Downloading files..."); + await Utils.DownloadFile("https://raw.githubusercontent.com/SteamDatabase/GameTracking-CSGO/master/csgo/scripts/items/items_game.txt", "items_game.vdf"); + await Utils.DownloadFile("https://raw.githubusercontent.com/SteamDatabase/GameTracking-CSGO/master/csgo/resource/csgo_english.txt", "csgo_english.vdf"); } -List collections = new(); -var doc = new HtmlDocument(); -doc.Load("cache/index.html"); +// Parse game files +Console.WriteLine("Parsing items_game.vdf..."); +var items_game = VdfConvert.Parse(File.ReadAllText("items_game.vdf")); -var navbar = doc.DocumentNode.SelectSingleNode("//body/div[@class=\"container header\"]/div[@class=\"navbar-wrapper\"]/nav/div/div[@id=\"navbar-expandable\"]/ul"); -var casesLinks = navbar.ChildNodes[13].SelectSingleNode("ul").ChildNodes; -var collectionsLinks = navbar.ChildNodes[15].SelectSingleNode("ul").ChildNodes; +Console.WriteLine("Parsing csgo_english.vdf..."); +var csgo_english = VdfConvert.Parse(File.ReadAllText("csgo_english.vdf")); -foreach (var caseLink in casesLinks) +// Optionally convert to JSON +if (args.Contains("--json") || args.Contains("-j")) { - if (caseLink.FirstChild is null || caseLink.FirstChild.Name != "a") - continue; - string href = caseLink.FirstChild.GetAttributeValue("href", "").Trim(); - if (!href.StartsWith("https://csgostash.com/case/")) - continue; + Console.WriteLine("Saving items_game.json"); + File.WriteAllText("items_game.json", JsonConvert.SerializeObject(items_game, Formatting.Indented)); - collections.Add(await LoadCollection(href)); + Console.WriteLine("Saving csgo_english.json"); + File.WriteAllText("csgo_english.json", JsonConvert.SerializeObject(csgo_english, Formatting.Indented)); } -foreach (var collectionLink in collectionsLinks) -{ - if (collectionLink.FirstChild is null || collectionLink.FirstChild.Name != "a") - continue; - string href = collectionLink.FirstChild.GetAttributeValue("href", "").Trim(); - - collections.Add(await LoadCollection(href)); -} - -string json = JsonConvert.SerializeObject(collections, Formatting.Indented); -File.WriteAllText("SkinList.json", json); - -static async Task LoadCollection(string url) -{ - Collection collection = new() - { - CanBeStattrak = false, - Link = url - }; - string collectionName = Utils.ReplaceInvalidChars(url.Split('/').Last()); - string collectionPath = $"cache/{collectionName}"; - - Console.Write($"Loading {collectionName}..."); - Stopwatch sw = Stopwatch.StartNew(); - if (!Directory.Exists(collectionPath)) - Directory.CreateDirectory(collectionPath); - - if (!File.Exists($"{collectionPath}/index.html")) - await Utils.DownloadFile(url, $"{collectionPath}/index.html"); - - var doc = new HtmlDocument(); - doc.Load($"{collectionPath}/index.html"); - var collectionNameFull = doc.DocumentNode.SelectSingleNode("//body/div[2]/div[2]/div/div[2]/h1").InnerText; - collection.Name = HttpUtility.HtmlDecode(collectionNameFull.Trim()); - - var contentNodes = doc.DocumentNode.SelectSingleNode("(//body/div[@class=\"container main-content\"]/div[@class=\"row\"])[4]"); - foreach (var row in contentNodes.ChildNodes) - { - if (row.InnerHtml.Contains("Knives")) - continue; - else if (row.InnerHtml.Contains("