Skip to content

Commit 5894fa5

Browse files
pberingpbering
authored
Fixed media url replacement/downloads so it works on both preview and live edge endpoints (#2)
* cleanup duplicate code and better detection of media urls * fixed media url replacements from edge (preview and live)
1 parent b386d43 commit 5894fa5

5 files changed

Lines changed: 89 additions & 66 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ If you want to crawl Experience Edge with a preview context id's or a local XM C
6868

6969
Currently there a few limitations/gotchas, some may be fixed in the future:
7070

71-
1. When running `eee` in Docker, you cannot crawl a local XM Cloud instance unless they share a Docker network.
71+
1. When running `eee` in Docker, you cannot crawl a local XM Cloud instance *unless* they share the same Docker network.
7272
1. Using the `maxWidth` and `maxHeight` on `src` property fields does nothing.
7373
1. `SiteInfo.RoutesResult` only supports the `language` and `first` parameters, `excludedPaths` and `includePaths` does nothing and `after` throws `NotSupportedException`.
7474
1. `SiteInfo.DictionaryResult` only supports the `language` and `first` parameters, `after` throws `NotSupportedException`.

‎src/ExperienceEdgeEmu.Web/DataStore/Crawler/ItemPostProcessingWorker.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
3232
// queue download media messages
3333
foreach (var change in changes)
3434
{
35-
await mediaDownloadQueue.QueueMessageAsync(new MediaDownloadMessage(change.Key));
35+
var originalUri = new Uri(change.Key, UriKind.RelativeOrAbsolute);
36+
var newUri = new Uri(change.Value, UriKind.RelativeOrAbsolute);
37+
38+
await mediaDownloadQueue.QueueMessageAsync(new MediaDownloadMessage(originalUri, newUri));
3639
}
3740
}
3841
}

‎src/ExperienceEdgeEmu.Web/DataStore/Crawler/MediaUrlReplacer.cs‎

Lines changed: 69 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
1-
using System.Text.Json.Nodes;
1+
using Microsoft.Extensions.Caching.Memory;
2+
using System.Text.Json.Nodes;
23
using System.Text.RegularExpressions;
34

45
namespace ExperienceEdgeEmu.Web.DataStore.Crawler;
56

6-
public partial class MediaUrlReplacer
7+
public partial class MediaUrlReplacer(IMemoryCache processedMediaUrls)
78
{
8-
[GeneratedRegex(@"(?:https?:\/\/[^\s\""]+)?(\/-\/media\/Project\/[^\s\""]+)", RegexOptions.IgnoreCase)]
9-
private static partial Regex MediaRegex();
9+
[GeneratedRegex(@"(?:https?:\/\/[^\s""\/]+)?(\/-\/(?:media|jssmedia)\/[^\s""?\)]+)", RegexOptions.IgnoreCase)]
10+
private static partial Regex PreviewMediaRegex();
11+
12+
[GeneratedRegex(@"https?:\/\/edge\.sitecorecloud\.io\/[^\/]+\/(media\/[^\s\""\?]+)", RegexOptions.IgnoreCase)]
13+
private static partial Regex EdgeMediaRegex();
1014

1115
public Dictionary<string, string> ReplaceMediaUrlsInFields(JsonNode? node, string baseUrl)
1216
{
@@ -45,15 +49,69 @@ private void ReplaceMediaUrlsInFieldsInternal(JsonNode? node, string baseUrl, Di
4549
}
4650
else if (node is JsonArray arr)
4751
{
48-
for (var i = 0; i < arr.Count; i++)
52+
foreach (var child in arr)
4953
{
50-
var child = arr[i];
51-
5254
ReplaceMediaUrlsInFieldsInternal(child, baseUrl, changes);
5355
}
5456
}
5557
}
5658

59+
private bool TryReplaceMediaUrl(string value, string baseUrl, Dictionary<string, string> changes, out string? replaced)
60+
{
61+
replaced = null;
62+
var wasReplaced = false;
63+
var valueKey = $"{nameof(MediaUrlReplacer)}:{value}";
64+
65+
if (processedMediaUrls.TryGetValue(value, out replaced))
66+
{
67+
return true;
68+
}
69+
70+
// skip if value does not look like media urls from edge or preview instances
71+
if (!value.Contains(".sitecorecloud.io/", StringComparison.OrdinalIgnoreCase) && !value.Contains("/-/media/", StringComparison.OrdinalIgnoreCase) && !value.Contains("/-/jssmedia/", StringComparison.OrdinalIgnoreCase))
72+
{
73+
return wasReplaced;
74+
}
75+
76+
// skip if value contains .ashx which happens on urls that are not media urls
77+
if (value.Contains(".ashx", StringComparison.OrdinalIgnoreCase))
78+
{
79+
return wasReplaced;
80+
}
81+
82+
var tempString = value;
83+
84+
if (EdgeMediaRegex().Match(value).Success)
85+
{
86+
var edgePattern = $"{baseUrl.TrimEnd('/')}/-/$1";
87+
88+
tempString = EdgeMediaRegex().Replace(tempString, edgePattern);
89+
}
90+
else if (PreviewMediaRegex().Match(value).Success)
91+
{
92+
var relativePattern = $"{baseUrl.TrimEnd('/')}$1";
93+
94+
tempString = PreviewMediaRegex().Replace(tempString, relativePattern);
95+
}
96+
97+
if (!string.Equals(value, tempString, StringComparison.Ordinal))
98+
{
99+
changes.TryAdd(value, tempString);
100+
101+
replaced = tempString;
102+
wasReplaced = true;
103+
104+
processedMediaUrls.GetOrCreate(valueKey, entry =>
105+
{
106+
entry.SetSlidingExpiration(TimeSpan.FromHours(30));
107+
108+
return true;
109+
});
110+
}
111+
112+
return wasReplaced;
113+
}
114+
57115
private void ReplaceStringsRecursivelyInObject(JsonObject obj, string baseUrl, Dictionary<string, string> changes)
58116
{
59117
foreach (var kvp in obj.ToList())
@@ -63,25 +121,9 @@ private void ReplaceStringsRecursivelyInObject(JsonObject obj, string baseUrl, D
63121

64122
if (value is JsonValue val && val.TryGetValue(out string? s) && s is not null)
65123
{
66-
if (s.Contains("/-/media/Project/", StringComparison.OrdinalIgnoreCase))
124+
if (TryReplaceMediaUrl(s, baseUrl, changes, out var replaced))
67125
{
68-
foreach (Match m in MediaRegex().Matches(s))
69-
{
70-
var originalUrl = m.Value;
71-
var newUrl = MediaRegex().Replace(originalUrl, $"{baseUrl.TrimEnd('/')}$1");
72-
73-
if (!string.Equals(originalUrl, newUrl, StringComparison.Ordinal))
74-
{
75-
changes.TryAdd(originalUrl, newUrl);
76-
}
77-
}
78-
79-
var replaced = MediaRegex().Replace(s, $"{baseUrl.TrimEnd('/')}$1");
80-
81-
if (!string.Equals(replaced, s, StringComparison.Ordinal))
82-
{
83-
obj[key] = replaced;
84-
}
126+
obj[key] = replaced;
85127
}
86128
}
87129
else if (value is JsonObject childObj)
@@ -103,25 +145,9 @@ private void ReplaceStringsRecursivelyInArray(JsonArray arr, string baseUrl, Dic
103145

104146
if (v is JsonValue val && val.TryGetValue(out string? s) && s is not null)
105147
{
106-
if (s.Contains("/-/media/Project/", StringComparison.OrdinalIgnoreCase))
148+
if (TryReplaceMediaUrl(s, baseUrl, changes, out var replaced))
107149
{
108-
foreach (Match m in MediaRegex().Matches(s))
109-
{
110-
var originalUrl = m.Value;
111-
var newUrl = MediaRegex().Replace(originalUrl, $"{baseUrl.TrimEnd('/')}$1");
112-
113-
if (!string.Equals(originalUrl, newUrl, StringComparison.Ordinal))
114-
{
115-
changes.TryAdd(originalUrl, newUrl);
116-
}
117-
}
118-
119-
var replaced = MediaRegex().Replace(s, $"{baseUrl.TrimEnd('/')}$1");
120-
121-
if (!string.Equals(replaced, s, StringComparison.Ordinal))
122-
{
123-
arr[i] = replaced;
124-
}
150+
arr[i] = replaced;
125151
}
126152
}
127153
else if (v is JsonObject childObj)

‎src/ExperienceEdgeEmu.Web/Media/MediaDownloadQueue.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
namespace ExperienceEdgeEmu.Web.Media;
44

5-
public record MediaDownloadMessage(string Url);
5+
public record MediaDownloadMessage(Uri Original, Uri New);
66

77
public class MediaDownloadQueue
88
{

‎src/ExperienceEdgeEmu.Web/Media/MediaDownloadWorker.cs‎

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,52 +3,46 @@
33

44
namespace ExperienceEdgeEmu.Web.Media;
55

6-
public class MediaDownloadWorker(MediaDownloadQueue _queue, ILogger<MediaDownloadWorker> _logger, EmuFileSystem _emuFileSystem, IHttpClientFactory _httpClientFactory, IMemoryCache _processedMedia) : BackgroundService
6+
public class MediaDownloadWorker(MediaDownloadQueue queue, ILogger<MediaDownloadWorker> logger, EmuFileSystem emuFileSystem, IHttpClientFactory httpClientFactory, IMemoryCache processedMedia) : BackgroundService
77
{
88
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
99
{
10-
await foreach (var message in _queue.DequeueAllAsync(stoppingToken))
10+
await foreach (var message in queue.DequeueAllAsync(stoppingToken))
1111
{
12-
var mediaUrl = message.Url;
13-
1412
try
1513
{
16-
var uri = new Uri(mediaUrl, UriKind.RelativeOrAbsolute);
17-
18-
if (!uri.IsAbsoluteUri)
14+
if (!message.Original.IsAbsoluteUri)
1915
{
20-
_logger.LogWarning("Skipping relative url: {MediaUrl}.", mediaUrl);
16+
logger.LogWarning("Skipping relative url: {MediaOrignalUri}.", message.Original);
2117

2218
continue;
2319
}
2420

25-
var key = uri.GetLeftPart(UriPartial.Path);
21+
var mediaKey = $"{nameof(MediaDownloadWorker)}:{message.Original.GetLeftPart(UriPartial.Path)}";
2622

27-
if (_processedMedia.TryGetValue(key, out _))
23+
if (processedMedia.TryGetValue(mediaKey, out _))
2824
{
29-
_logger.LogDebug("Media {MediaUrl} has already been processed, skipping.", mediaUrl);
25+
logger.LogDebug("Media {MediaOrignalUri} has already been processed, skipping.", message.Original);
3026

3127
continue;
3228
}
3329

34-
var filePath = _emuFileSystem.GetMediaFilePath(uri);
30+
var filePath = emuFileSystem.GetMediaFilePath(message.New);
3531
var directory = Directory.GetParent(filePath)!.FullName;
3632

3733
if (!Directory.Exists(directory))
3834
{
3935
Directory.CreateDirectory(directory);
4036
}
4137

42-
_logger.LogInformation("Downloading media {MediaUrl} into {FilePath}.", mediaUrl, filePath);
38+
logger.LogInformation("Downloading media {MediaOrignalUri} into {FilePath}.", message.Original, filePath);
4339

44-
using (var mediaStream = await _httpClientFactory.CreateClient().GetStreamAsync(uri, stoppingToken))
45-
{
46-
using var fileStream = new FileStream(filePath, FileMode.Create);
40+
using var mediaStream = await httpClientFactory.CreateClient().GetStreamAsync(message.Original, stoppingToken);
41+
using var fileStream = new FileStream(filePath, FileMode.Create);
4742

48-
await mediaStream.CopyToAsync(fileStream, stoppingToken);
49-
}
43+
await mediaStream.CopyToAsync(fileStream, stoppingToken);
5044

51-
_processedMedia.GetOrCreate(key, entry =>
45+
processedMedia.GetOrCreate(mediaKey, entry =>
5246
{
5347
entry.SetSlidingExpiration(TimeSpan.FromMinutes(30));
5448

@@ -57,7 +51,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
5751
}
5852
catch (Exception ex)
5953
{
60-
_logger.LogError(ex, "Media download failed for {MediaUrl}.", mediaUrl);
54+
logger.LogError(ex, "Media download failed for {MediaOrignalUri}.", message.Original);
6155
}
6256
}
6357
}

0 commit comments

Comments
 (0)