Make image size limit configurable

Adapted from Joe Drago's pull request
AOMediaCodec#527, with the limitation
that decoder->imageSizeLimit must be less than or equal to the default
value and must not be set to 0 (reserved for future use). This way we
don't need to audit our code for integer overflows due to a large image
width or height.

Set decoder->imageSizeLimit to 11 * 1024 * 10 * 1024 in
avif_decode_fuzzer.cc to keep its memory consumption under 2560 MB.

Author: wantehchang

include/avif/avif.h

@@ -68,6 +68,10 @@ typedef int avifBool;
 
 #define AVIF_DIAGNOSTICS_ERROR_BUFFER_SIZE 256
 
+// A reasonable default for maximum image size to avoid out-of-memory errors or integer overflow in
+// (32-bit) int or unsigned int arithmetic operations.
+#define AVIF_DEFAULT_MAX_IMAGE_SIZE (16384 * 16384)
+
 // a 12 hour AVIF image sequence, running at 60 fps (a basic sanity check as this is quite ridiculous)
 #define AVIF_DEFAULT_IMAGE_COUNT_LIMIT (12 * 3600 * 60)
 
@@ -810,6 +814,12 @@ typedef struct avifDecoder
     avifBool ignoreExif;
     avifBool ignoreXMP;
 
+    // This represents the maximum size of a image (in pixel count) that libavif and the underlying
+    // AV1 decoder should attempt to decode. It defaults to AVIF_DEFAULT_MAX_IMAGE_SIZE, and can be
+    // set to a smaller value. The value 0 is reserved.
+    // Note: Only some underlying AV1 codecs support a configurable size limit (such as dav1d).
+    uint32_t imageSizeLimit;
+
     // This provides an upper bound on how many images the decoder is willing to attempt to decode,
     // to provide a bit of protection from malicious or malformed AVIFs citing millions upon
     // millions of frames, only to be invalid later. The default is AVIF_DEFAULT_IMAGE_COUNT_LIMIT

include/avif/internal.h

@@ -159,7 +159,7 @@ avifResult avifRGBImageUnpremultiplyAlphaLibYUV(avifRGBImage * rgb);
 // Scaling
 
 // This scales the YUV/A planes in-place.
-avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight, avifDiagnostics * diag);
+avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight, uint32_t imageSizeLimit, avifDiagnostics * diag);
 
 // ---------------------------------------------------------------------------
 // avifCodecDecodeInput
@@ -367,10 +367,6 @@ typedef struct avifSequenceHeader
 } avifSequenceHeader;
 avifBool avifSequenceHeaderParse(avifSequenceHeader * header, const avifROData * sample);
 
-// A maximum image size to avoid out-of-memory errors or integer overflow in
-// (32-bit) int or unsigned int arithmetic operations.
-#define AVIF_MAX_IMAGE_SIZE (16384 * 16384)
-
 #ifdef __cplusplus
 } // extern "C"
 #endif

src/codec_dav1d.c

@@ -59,6 +59,12 @@ static avifBool dav1dCodecGetNextImage(struct avifCodec * codec,
         // Give all available threads to decode a single frame as fast as possible
         codec->internal->dav1dSettings.n_frame_threads = 1;
         codec->internal->dav1dSettings.n_tile_threads = AVIF_CLAMP(decoder->maxThreads, 1, DAV1D_MAX_TILE_THREADS);
+        // Set a maximum frame size limit to avoid OOM'ing fuzzers. In 32-bit builds, if
+        // frame_size_limit > 8192 * 8192, dav1d reduces frame_size_limit to 8192 * 8192 and logs
+        // a message, so we set frame_size_limit to at most 8192 * 8192 to avoid the dav1d_log
+        // message.
+        codec->internal->dav1dSettings.frame_size_limit = (sizeof(size_t) < 8) ? AVIF_MIN(decoder->imageSizeLimit, 8192 * 8192)
+                                                                               : decoder->imageSizeLimit;
         codec->internal->dav1dSettings.operating_point = codec->operatingPoint;
         codec->internal->dav1dSettings.all_layers = codec->allLayers;
 
@@ -215,11 +221,6 @@ avifCodec * avifCodecCreateDav1d(void)
     memset(codec->internal, 0, sizeof(struct avifCodecInternal));
     dav1d_default_settings(&codec->internal->dav1dSettings);
 
-    // Set a maximum frame size limit to avoid OOM'ing fuzzers. In 32-bit builds, if
-    // frame_size_limit > 8192 * 8192, dav1d reduces frame_size_limit to 8192 * 8192 and logs a
-    // message, so we set frame_size_limit to 8192 * 8192 to avoid the dav1d_log message.
-    codec->internal->dav1dSettings.frame_size_limit = (sizeof(size_t) < 8) ? (8192 * 8192) : AVIF_MAX_IMAGE_SIZE;
-
     // Ensure that we only get the "highest spatial layer" as a single frame
     // for each input sample, instead of getting each spatial layer as its own
     // frame one at a time ("all layers").

src/read.c

@@ -1598,7 +1598,7 @@ static avifBool avifParseItemLocationBox(avifMeta * meta, const uint8_t * raw, s
     return AVIF_TRUE;
 }
 
-static avifBool avifParseImageGridBox(avifImageGrid * grid, const uint8_t * raw, size_t rawLen, avifDiagnostics * diag)
+static avifBool avifParseImageGridBox(avifImageGrid * grid, const uint8_t * raw, size_t rawLen, uint32_t imageSizeLimit, avifDiagnostics * diag)
 {
     BEGIN_STREAM(s, raw, rawLen, diag, "Box[grid]");
 
@@ -1631,7 +1631,7 @@ static avifBool avifParseImageGridBox(avifImageGrid * grid, const uint8_t * raw,
         CHECK(avifROStreamReadU32(&s, &grid->outputWidth));  // unsigned int(FieldLength) output_width;
         CHECK(avifROStreamReadU32(&s, &grid->outputHeight)); // unsigned int(FieldLength) output_height;
     }
-    if ((grid->outputWidth == 0) || (grid->outputHeight == 0) || (grid->outputWidth > (AVIF_MAX_IMAGE_SIZE / grid->outputHeight))) {
+    if ((grid->outputWidth == 0) || (grid->outputHeight == 0) || (grid->outputWidth > (imageSizeLimit / grid->outputHeight))) {
         avifDiagnosticsPrintf(diag, "Grid box contains illegal dimensions: [%u x %u]", grid->outputWidth, grid->outputHeight);
         return AVIF_FALSE;
     }
@@ -2322,7 +2322,7 @@ static avifBool avifParseMetaBox(avifMeta * meta, const uint8_t * raw, size_t ra
     return AVIF_TRUE;
 }
 
-static avifBool avifParseTrackHeaderBox(avifTrack * track, const uint8_t * raw, size_t rawLen, avifDiagnostics * diag)
+static avifBool avifParseTrackHeaderBox(avifTrack * track, const uint8_t * raw, size_t rawLen, uint32_t imageSizeLimit, avifDiagnostics * diag)
 {
     BEGIN_STREAM(s, raw, rawLen, diag, "Box[tkhd]");
 
@@ -2365,7 +2365,7 @@ static avifBool avifParseTrackHeaderBox(avifTrack * track, const uint8_t * raw,
     track->width = width >> 16;
     track->height = height >> 16;
 
-    if ((track->width == 0) || (track->height == 0) || (track->width > (AVIF_MAX_IMAGE_SIZE / track->height))) {
+    if ((track->width == 0) || (track->height == 0) || (track->width > (imageSizeLimit / track->height))) {
         avifDiagnosticsPrintf(diag, "Track ID [%u] has an invalid size [%ux%u]", track->id, track->width, track->height);
         return AVIF_FALSE;
     }
@@ -2644,7 +2644,7 @@ static avifBool avifTrackReferenceBox(avifTrack * track, const uint8_t * raw, si
     return AVIF_TRUE;
 }
 
-static avifBool avifParseTrackBox(avifDecoderData * data, const uint8_t * raw, size_t rawLen)
+static avifBool avifParseTrackBox(avifDecoderData * data, const uint8_t * raw, size_t rawLen, uint32_t imageSizeLimit)
 {
     BEGIN_STREAM(s, raw, rawLen, data->diag, "Box[trak]");
 
@@ -2655,7 +2655,7 @@ static avifBool avifParseTrackBox(avifDecoderData * data, const uint8_t * raw, s
         CHECK(avifROStreamReadBoxHeader(&s, &header));
 
         if (!memcmp(header.type, "tkhd", 4)) {
-            CHECK(avifParseTrackHeaderBox(track, avifROStreamCurrent(&s), header.size, data->diag));
+            CHECK(avifParseTrackHeaderBox(track, avifROStreamCurrent(&s), header.size, imageSizeLimit, data->diag));
         } else if (!memcmp(header.type, "meta", 4)) {
             CHECK(avifParseMetaBox(track->meta, avifROStreamCurrent(&s), header.size, data->diag));
         } else if (!memcmp(header.type, "mdia", 4)) {
@@ -2669,7 +2669,7 @@ static avifBool avifParseTrackBox(avifDecoderData * data, const uint8_t * raw, s
     return AVIF_TRUE;
 }
 
-static avifBool avifParseMoovBox(avifDecoderData * data, const uint8_t * raw, size_t rawLen)
+static avifBool avifParseMoovBox(avifDecoderData * data, const uint8_t * raw, size_t rawLen, uint32_t imageSizeLimit)
 {
     BEGIN_STREAM(s, raw, rawLen, data->diag, "Box[moov]");
 
@@ -2678,7 +2678,7 @@ static avifBool avifParseMoovBox(avifDecoderData * data, const uint8_t * raw, si
         CHECK(avifROStreamReadBoxHeader(&s, &header));
 
         if (!memcmp(header.type, "trak", 4)) {
-            CHECK(avifParseTrackBox(data, avifROStreamCurrent(&s), header.size));
+            CHECK(avifParseTrackBox(data, avifROStreamCurrent(&s), header.size, imageSizeLimit));
         }
 
         CHECK(avifROStreamSkip(&s, header.size));
@@ -2779,7 +2779,7 @@ static avifResult avifParse(avifDecoder * decoder)
             metaSeen = AVIF_TRUE;
         } else if (!memcmp(header.type, "moov", 4)) {
             CHECKERR(!moovSeen, AVIF_RESULT_BMFF_PARSE_FAILED);
-            CHECKERR(avifParseMoovBox(data, boxContents.data, boxContents.size), AVIF_RESULT_BMFF_PARSE_FAILED);
+            CHECKERR(avifParseMoovBox(data, boxContents.data, boxContents.size, decoder->imageSizeLimit), AVIF_RESULT_BMFF_PARSE_FAILED);
             moovSeen = AVIF_TRUE;
         }
 
@@ -2847,6 +2847,7 @@ avifDecoder * avifDecoderCreate(void)
     avifDecoder * decoder = (avifDecoder *)avifAlloc(sizeof(avifDecoder));
     memset(decoder, 0, sizeof(avifDecoder));
     decoder->maxThreads = 1;
+    decoder->imageSizeLimit = AVIF_DEFAULT_MAX_IMAGE_SIZE;
     decoder->imageCountLimit = AVIF_DEFAULT_IMAGE_COUNT_LIMIT;
     decoder->strictFlags = AVIF_STRICT_ENABLED;
     return decoder;
@@ -3033,6 +3034,11 @@ avifResult avifDecoderParse(avifDecoder * decoder)
 {
     avifDiagnosticsClearError(&decoder->diag);
 
+    // An imageSizeLimit greater than AVIF_DEFAULT_MAX_IMAGE_SIZE and the special value of 0 to
+    // disable the limit are not yet implemented.
+    if ((decoder->imageSizeLimit > AVIF_DEFAULT_MAX_IMAGE_SIZE) || (decoder->imageSizeLimit == 0)) {
+        return AVIF_RESULT_NOT_IMPLEMENTED;
+    }
     if (!decoder->io || !decoder->io->read) {
         return AVIF_RESULT_IO_NOT_SET;
     }
@@ -3073,7 +3079,7 @@ avifResult avifDecoderParse(avifDecoder * decoder)
             item->width = ispeProp->u.ispe.width;
             item->height = ispeProp->u.ispe.height;
 
-            if ((item->width == 0) || (item->height == 0) || (item->width > (AVIF_MAX_IMAGE_SIZE / item->height))) {
+            if ((item->width == 0) || (item->height == 0) || (item->width > (decoder->imageSizeLimit / item->height))) {
                 avifDiagnosticsPrintf(data->diag, "Item ID [%u] has an invalid size [%ux%u]", item->id, item->width, item->height);
                 return AVIF_RESULT_BMFF_PARSE_FAILED;
             }
@@ -3302,7 +3308,7 @@ avifResult avifDecoderReset(avifDecoder * decoder)
                 if (readResult != AVIF_RESULT_OK) {
                     return readResult;
                 }
-                if (!avifParseImageGridBox(&data->colorGrid, readData.data, readData.size, data->diag)) {
+                if (!avifParseImageGridBox(&data->colorGrid, readData.data, readData.size, decoder->imageSizeLimit, data->diag)) {
                     return AVIF_RESULT_INVALID_IMAGE_GRID;
                 }
             }
@@ -3342,7 +3348,7 @@ avifResult avifDecoderReset(avifDecoder * decoder)
                     if (readResult != AVIF_RESULT_OK) {
                         return readResult;
                     }
-                    if (!avifParseImageGridBox(&data->alphaGrid, readData.data, readData.size, data->diag)) {
+                    if (!avifParseImageGridBox(&data->alphaGrid, readData.data, readData.size, decoder->imageSizeLimit, data->diag)) {
                         return AVIF_RESULT_INVALID_IMAGE_GRID;
                     }
                 }
@@ -3611,7 +3617,7 @@ avifResult avifDecoderNextImage(avifDecoder * decoder)
 
         // Scale the decoded image so that it corresponds to this tile's output dimensions
         if ((tile->width != tile->image->width) || (tile->height != tile->image->height)) {
-            if (!avifImageScale(tile->image, tile->width, tile->height, &decoder->diag)) {
+            if (!avifImageScale(tile->image, tile->width, tile->height, decoder->imageSizeLimit, &decoder->diag)) {
                 avifDiagnosticsPrintf(&decoder->diag, "avifImageScale() failed");
                 return tile->input->alpha ? AVIF_RESULT_DECODE_ALPHA_FAILED : AVIF_RESULT_DECODE_COLOR_FAILED;
             }

src/scale.c

@@ -5,11 +5,12 @@
 
 #if !defined(AVIF_LIBYUV_ENABLED)
 
-avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight, avifDiagnostics * diag)
+avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight, uint32_t imageSizeLimit, avifDiagnostics * diag)
 {
     (void)image;
     (void)dstWidth;
     (void)dstHeight;
+    (void)imageSizeLimit;
     avifDiagnosticsPrintf(diag, "avifImageScale() called, but is unimplemented without libyuv!");
     return AVIF_FALSE;
 }
@@ -28,14 +29,14 @@ avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight
 // This should be configurable and/or smarter. kFilterBox has the highest quality but is the slowest.
 #define AVIF_LIBYUV_FILTER_MODE kFilterBox
 
-avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight, avifDiagnostics * diag)
+avifBool avifImageScale(avifImage * image, uint32_t dstWidth, uint32_t dstHeight, uint32_t imageSizeLimit, avifDiagnostics * diag)
 {
     if ((image->width == dstWidth) && (image->height == dstHeight)) {
         // Nothing to do
         return AVIF_TRUE;
     }
 
-    if ((dstWidth == 0) || (dstHeight == 0) || (dstWidth > (AVIF_MAX_IMAGE_SIZE / dstHeight))) {
+    if ((dstWidth == 0) || (dstHeight == 0) || (dstWidth > (imageSizeLimit / dstHeight))) {
         avifDiagnosticsPrintf(diag, "avifImageScale requested invalid dst dimensions [%ux%u]", dstWidth, dstHeight);
         return AVIF_FALSE;
     }

tests/oss-fuzz/avif_decode_fuzzer.cc

@@ -18,6 +18,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * Data, size_t Size)
     static size_t yuvDepthsCount = sizeof(yuvDepths) / sizeof(yuvDepths[0]);
 
     avifDecoder * decoder = avifDecoderCreate();
+    // ClusterFuzz passes -rss_limit_mb=2560 to avif_decode_fuzzer. Empirically setting
+    // decoder->imageSizeLimit to this value allows avif_decode_fuzzer to consume no more than
+    // 2560 MB of memory.
+    static_assert(11 * 1024 * 10 * 1024 <= AVIF_DEFAULT_MAX_IMAGE_SIZE, "");
+    decoder->imageSizeLimit = 11 * 1024 * 10 * 1024;
     avifResult result = avifDecoderSetIOMemory(decoder, Data, Size);
     if (result == AVIF_RESULT_OK) {
         result = avifDecoderParse(decoder);