From 48ca2270845567af77931679904a5f25aee749e0 Mon Sep 17 00:00:00 2001 From: Meisterlala <6453306+Meisterlala@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:46:29 +0100 Subject: [PATCH] Added Twitter API limit warning --- Neko/Gui/ImageSourcesWindow.cs | 17 +++++++++++++++++ Neko/Gui/MainWindow.cs | 11 +++++++++++ Neko/Helper.cs | 22 ++++++++++++++++++++++ Neko/NekoQueue.cs | 7 +++++++ Neko/Plugin.cs | 2 +- Neko/Sources/APIS/Twitter.cs | 25 +++++++++++++++++++++++++ Neko/Sources/Download.cs | 16 ++++++++++++++-- 7 files changed, 97 insertions(+), 3 deletions(-) diff --git a/Neko/Gui/ImageSourcesWindow.cs b/Neko/Gui/ImageSourcesWindow.cs index cc4701e..13a10b7 100644 --- a/Neko/Gui/ImageSourcesWindow.cs +++ b/Neko/Gui/ImageSourcesWindow.cs @@ -66,6 +66,8 @@ public ImageSourceConfig(string name, string description, string help, Type type private readonly HeaderImage.Individual Header = new(); + private static DateTime TwitterTimeout = DateTime.MinValue; + public void Draw() { // ------------ Header -------------- @@ -105,6 +107,21 @@ public void Draw() DrawTheCatAPI(); // ------------ Twitter -------------- SourceCheckbox(SourceList[9], ref Plugin.Config.Sources.Twitter.enabled); + if (Twitter.IsRateLimited) + { + ImGui.SameLine(); + ImGui.TextColored(new Vector4(1f, 0, 0f, 1f), "API limit reached"); + ImGui.SameLine(); + Common.HelpMarker("The free Twitter API is limited to 2 million tweets per Month. This limit is shared between all users of this plugin and will usually be reset on the 26st of every month."); + // Make the Twitter config unable to open + if (Plugin.Config.Sources.Twitter.enabled) + { + Plugin.Config.Sources.Twitter.enabled = false; + Plugin.Config.Save(); + Plugin.UpdateImageSource(); + } + } + if (Plugin.Config.Sources.Twitter.enabled) DrawTwitter(); diff --git a/Neko/Gui/MainWindow.cs b/Neko/Gui/MainWindow.cs index 17a4e82..61cf37e 100644 --- a/Neko/Gui/MainWindow.cs +++ b/Neko/Gui/MainWindow.cs @@ -52,6 +52,17 @@ public MainWindow() imageGrayed = false; } + ~MainWindow() + { + Dispose(); + } + + public void Dispose() + { + Slideshow.Stop(); + Queue.Dispose(); + } + public void Draw() { if (!Visible) return; diff --git a/Neko/Helper.cs b/Neko/Helper.cs index b045536..68d827e 100644 --- a/Neko/Helper.cs +++ b/Neko/Helper.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Net.Http; using System.Runtime.InteropServices; using Dalamud.Logging; using TextCopy; @@ -142,4 +143,25 @@ public static string EndWithEllipsis(string text, int maxLength) ? text : text[..(maxLength - 3)] + "..."; } + + public static HttpRequestMessage RequestClone(HttpRequestMessage req) + { + var clone = new HttpRequestMessage(req.Method, req.RequestUri) + { + Content = req.Content, + Version = req.Version + }; + + foreach (var prop in req.Options) + { + clone.Options.TryAdd(prop.Key, prop.Value); + } + + foreach (var header in req.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + return clone; + } } diff --git a/Neko/NekoQueue.cs b/Neko/NekoQueue.cs index 8eeb6f4..67fb5a2 100644 --- a/Neko/NekoQueue.cs +++ b/Neko/NekoQueue.cs @@ -35,6 +35,13 @@ public NekoQueue() tokenSource.Cancel(); } + public void Dispose() + { + tokenSource.Cancel(); + tokenSource = new(); + StopQueue = true; + } + public override string ToString() { var res = $"Queue length: {TargetDownloadCount} preloaded: {TargetPreloadCount}{(StopQueue ? " Queue Stopped" : "")}"; diff --git a/Neko/Plugin.cs b/Neko/Plugin.cs index 96785c2..201242f 100644 --- a/Neko/Plugin.cs +++ b/Neko/Plugin.cs @@ -83,7 +83,7 @@ public void Dispose() CommandManager.RemoveHandler(CommandMain); // Stop loading images - GuiMain?.Slideshow?.Stop(); + GuiMain?.Dispose(); } public static void UpdateImageSource() => ImageSource.UpdateFrom(Config.LoadSources()); diff --git a/Neko/Sources/APIS/Twitter.cs b/Neko/Sources/APIS/Twitter.cs index 6456936..b32d2e8 100644 --- a/Neko/Sources/APIS/Twitter.cs +++ b/Neko/Sources/APIS/Twitter.cs @@ -60,6 +60,25 @@ private static HttpRequestMessage AuthorizedRequest(string url) => public override bool SameAs(ImageSource other) => other is Twitter t && t.ConfigQuery == ConfigQuery; + /// + /// Checks the header to see if the response is from Twitter rate limiting + /// + /// response Message + /// True if the header is a response from Twitter + public static bool Is429Response(HttpResponseMessage response) => + response.RequestMessage?.RequestUri?.Host == "api.twitter.com" && + response.Headers.TryGetValues("x-rate-limit-remaining", out var remaining) && + remaining != null && + response.Headers.TryGetValues("x-rate-limit-reset", out var reset) && + reset != null && + response.Headers.TryGetValues("x-rate-limit-limit", out var limit) && + limit != null; + + /// + /// Checks if the API is rate limited + /// + public static bool IsRateLimited; + public class Config : IImageConfig { public bool enabled; @@ -207,6 +226,9 @@ public override NekoImage Next(CancellationToken ct = default) { return new NekoImage(async (img) => { + if (IsRateLimited) + throw new Exception("Twitter API rate limit exceeded"); + var searchResult = await URLs.GetURL(ct).ConfigureAwait(false); var response = await Download.DownloadImage(searchResult.Media.Url, typeof(Search), ct).ConfigureAwait(false); img.Description = searchResult.TweetDescription(); @@ -438,6 +460,9 @@ public override NekoImage Next(CancellationToken ct = default) { return new NekoImage(async (img) => { + if (IsRateLimited) + throw new Exception("Twitter API rate limit exceeded"); + lock (userIDTaskLock) { userIDTask ??= GetUserID(ct); diff --git a/Neko/Sources/Download.cs b/Neko/Sources/Download.cs index 17e1cad..5a377d9 100644 --- a/Neko/Sources/Download.cs +++ b/Neko/Sources/Download.cs @@ -100,6 +100,8 @@ public static async Task ParseJson(HttpRequestMessage request, Cancellatio // Handle 429 (Too Many Requests) by waiting and retrying if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) { + DebugHelper.LogNetwork(() => "API retuned 429 (Too Many Requests)\n" + response.Headers.ToString()); + var retryAfter = 2000; // in ms // Respect timeout header for WAIFU.IM if (response.Headers.TryGetValues("Retry-After", out var values) && values.Any()) @@ -109,10 +111,20 @@ public static async Task ParseJson(HttpRequestMessage request, Cancellatio retryAfter = (int)(seconds * 1000); } + // Twitter API limit reached + if (APIS.Twitter.Is429Response(response)) + { + APIS.Twitter.IsRateLimited = true; + throw new Exception("Twitter API limit reached. Wait a few days until the limit gets reset", ex); + } + + PluginLog.LogInformation($"API retuned 429 (Too Many Requests). Waiting {retryAfter / 1000.0} seconds before trying again."); // Wait 2 seconds and retry - PluginLog.LogVerbose($"API retuned 429 (Too Many Requests). Waiting {retryAfter / 1000.0} seconds before trying again."); await Task.Delay(retryAfter, ct).ConfigureAwait(false); - return await ParseJson(request, ct).ConfigureAwait(false); + ct.ThrowIfCancellationRequested(); + // Clone request, because you cant send the same one twice + var newRequest = Helper.RequestClone(request); + return await ParseJson(newRequest, ct).ConfigureAwait(false); } DebugHelper.LogNetwork(() => $"Error Downloading Json from {request.RequestUri}:\n{JsonSerializer.Serialize(response.Content.ReadAsStringAsync(ct).Result, new JsonSerializerOptions() { WriteIndented = true })}");