Skip to content

Use file storage as cache backing store #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tsunyoku wants to merge 26 commits into
base: main
Choose a base branch
from
Open

Use file storage as cache backing store #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
1eef030
Use file storage as backing store for cache
tsunyoku Sep 8, 2025
d9bbf67
Fix incorrect indexes
tsunyoku Sep 8, 2025
9072e74
Prevent Redis subscriber from swallowing exceptions
tsunyoku Sep 8, 2025
3d9cee8
Handle `RedisChannel` warning
tsunyoku Sep 8, 2025
6601ec9
Simplify `RedisDatabase` construction
tsunyoku Sep 8, 2025
588c84f
Add new environment variables to README
tsunyoku Sep 8, 2025
44da430
Remove unused cache dependency
tsunyoku Sep 8, 2025
de47020
Update tests
tsunyoku Sep 8, 2025
fd9a84c
Replace Redis solution with a caching folders per day
tsunyoku Oct 17, 2025
b60ab72
Fix test passing incorrect cache folder
tsunyoku Oct 17, 2025
5b22ca1
Fix replay cache folder for non-legacy case in tests too
tsunyoku Oct 17, 2025
232f9e5
Fix weird test method names
tsunyoku Oct 17, 2025
5d6567b
Fix incorrect method call for cache directory
tsunyoku Oct 17, 2025
686d63c
Create unique temp path for each test run
tsunyoku Oct 17, 2025
dd037c9
Fix cache lookup failing for replays cached prior to today
tsunyoku Oct 18, 2025
18309f2
Add test cases for incorrect stream handling
tsunyoku Oct 30, 2025
a263bf7
Add try-catch and error logging for expiry worker
tsunyoku Oct 30, 2025
cdcde59
Split legacy ruleset replays into subfolders instead of format-based …
tsunyoku Oct 30, 2025
4d3cf2e
Add xmldoc to explain cache folder structure
tsunyoku Oct 30, 2025
ebf806f
Move around test setup
tsunyoku Oct 30, 2025
b021ec7
Actually inspect stored replay
bdach Oct 31, 2025
80b5764
Improve xmldoc
bdach Oct 31, 2025
e3d7edc
Use `yyyyMMdd` format for cache directory
tsunyoku Dec 12, 2025
af56486
Restructure cache for date to be the higher level folder
tsunyoku Dec 12, 2025
2ed5846
Remove as much legacy specialities as possible
tsunyoku Dec 13, 2025
cb6429d
Fix binary types on legacy replay encoding
tsunyoku Dec 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,11 @@ Depending on environment, the configuration & config requirements change slightl
| `DB_PASS` | Password to use when logging into the `osu-web` MySQL instance. | ❌ No | `""` |
| `DB_NAME` | Name of database to use on the indicated MySQL instance. | ❌ No | `osu` |
| `REDIS_HOST` | Host name under which a Redis instance can be found. | ❌ No | `localhost` |
| `REDIS_DATABASE` | Database to use on the indicated Redis instance. | ❌ No | `0` |
| `REPLAY_STORAGE_TYPE` | Which type of replay storage to use. Valid values are `local` and `s3`. | ✔️ Yes | None |
| `REPLAY_CACHE_HOURS` | Cache duration for replays, in hours. | ❌ No | `24` |
| `REPLAY_CACHE_STORAGE_PATH` | The path of a directory where cached solo replays should reside. | ✔️ Yes | None |
| `LEGACY_REPLAY_CACHE_STORAGE_PATH` | The path of a directory where cached legacy replays should reside. | ✔️ Yes | None |
| `LOCAL_REPLAY_STORAGE_PATH` | The path of a directory where solo replays should reside. | ⚠️ If `REPLAY_STORAGE_TYPE=local` | None |
| `LOCAL_LEGACY_REPLAY_STORAGE_PATH` | The path of a directory where legacy replays should reside. | ⚠️ If `REPLAY_STORAGE_TYPE=local` | None |
| `S3_ACCESS_KEY` | A valid Amazon S3 access key ID. | ⚠ If `REPLAY_STORAGE_TYPE=s3` | None |
Expand Down
18 changes: 0 additions & 18 deletions osu.Server.ReplayStore.Tests/IntegrationTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ protected IntegrationTest(IntegrationTestWebApplicationFactory<Program> webAppFa
cancellationSource = Debugger.IsAttached
? new CancellationTokenSource()
: new CancellationTokenSource(20000);

emptyRedisCacheKeys();
}

private void reinitialiseDatabase()
Expand All @@ -49,22 +47,6 @@ private void reinitialiseDatabase()
db.Execute("TRUNCATE TABLE `osu_replays_mania`");
}

private void emptyRedisCacheKeys()
{
using var redisConnection = RedisAccess.GetConnection();

var endpoint = redisConnection.GetEndPoints()[0];
var redisServer = redisConnection.GetServer(endpoint);

var database = redisConnection.GetDatabase();

foreach (var key in redisServer.Keys(pattern: "solo-replay*"))
database.KeyDelete(key);

foreach (var key in redisServer.Keys(pattern: "legacy-replay*"))
database.KeyDelete(key);
}

public virtual void Dispose()
{
Client.Dispose();
Expand Down
56 changes: 31 additions & 25 deletions osu.Server.ReplayStore.Tests/LegacyReplayHelperTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@

using osu.Framework.Extensions;
using osu.Game.Beatmaps;
using osu.Game.Online.API;
using osu.Game.Rulesets;
using osu.Game.Rulesets.Osu;
using osu.Game.Rulesets.Osu.Objects;
using osu.Game.Rulesets.Scoring;
using osu.Game.Scoring.Legacy;
using osu.Game.Tests.Beatmaps;
using osu.Server.ReplayStore.Helpers;
Expand All @@ -24,23 +26,27 @@
{
using var stream = TestResources.GetResource(legacy_replay_filename)!;

var highScore = new HighScore
var score = new Score
{
score_id = 4501250208,
score = 13160096,
maxcombo = 724,
count50 = 0,
count100 = 3,
count300 = 525,
countmiss = 0,
countkatu = 3,
countgeki = 105,
perfect = true,
enabled_mods = 64,
id = 1,
legacy_score_id = 4501250208,
legacy_total_score = 13160096,
ScoreData = new SoloScoreData
{
Statistics = new Dictionary<HitResult, int>()

Check warning

Code scanning / InspectCode

Redundant empty argument list on object creation expression Warning test

Empty argument list is redundant
{
[HitResult.Great] = 525,
[HitResult.Ok] = 3,
[HitResult.Meh] = 0,
[HitResult.Miss] = 0,
},
Mods = [new APIMod { Acronym = "DT" }]
},
max_combo = 724,
user_id = 11315329,
date = new DateTimeOffset(2023, 09, 04, 21, 10, 42, TimeSpan.Zero),
ended_at = new DateTimeOffset(2023, 09, 04, 21, 10, 42, TimeSpan.Zero),
rank = "S",
replay = true,
has_replay = true,
};

var user = new User
Expand All @@ -57,23 +63,23 @@
await stream.ReadAllBytesToArrayAsync(),
rulesetId: 0,
legacy_replay_version,
highScore,
score,
user,
beatmap);

var scoreDecoder = new TestLegacyScoreDecoder();

var score = scoreDecoder.Parse(response);
var decodedScore = scoreDecoder.Parse(response);

Assert.Equal(user.username, score.ScoreInfo.RealmUser.Username);
Assert.Equal(highScore.count300, score.ScoreInfo.GetCount300());
Assert.Equal(highScore.count100, score.ScoreInfo.GetCount100());
Assert.Equal(highScore.count50, score.ScoreInfo.GetCount50());
Assert.Equal(highScore.countmiss, score.ScoreInfo.GetCountMiss());
Assert.Equal(highScore.score, score.ScoreInfo.LegacyTotalScore);
Assert.Equal(highScore.maxcombo, score.ScoreInfo.MaxCombo);
Assert.Equal(highScore.date.DateTime, score.ScoreInfo.Date);
Assert.Equal(highScore.score_id, (ulong)score.ScoreInfo.LegacyOnlineID);
Assert.Equal(user.username, decodedScore.ScoreInfo.RealmUser.Username);
Assert.Equal(score.ScoreData.Statistics[HitResult.Great], decodedScore.ScoreInfo.GetCount300());
Assert.Equal(score.ScoreData.Statistics[HitResult.Ok], decodedScore.ScoreInfo.GetCount100());
Assert.Equal(score.ScoreData.Statistics[HitResult.Meh], decodedScore.ScoreInfo.GetCount50());
Assert.Equal(score.ScoreData.Statistics[HitResult.Miss], decodedScore.ScoreInfo.GetCountMiss());
Assert.Equal(score.legacy_total_score, decodedScore.ScoreInfo.LegacyTotalScore);
Assert.Equal(score.max_combo, (uint)decodedScore.ScoreInfo.MaxCombo);
Assert.Equal(score.ended_at.DateTime, decodedScore.ScoreInfo.Date);
Assert.Equal(score.legacy_score_id, (ulong)decodedScore.ScoreInfo.LegacyOnlineID);
}
}

Expand Down
Loading
Loading