Perform bounds checks in vpx_write_bit_buffer

Add the `size` and `error` members to the vpx_write_bit_buffer struct.
Add the vpx_wb_init() and vpx_wb_has_error() functions.

Instances of the vpx_write_bit_buffer struct are only allocated in the
vp9_pack_bitstream() function. So vp9_pack_bitstream() is the only
function outside vpx_dsp/bitwriter_buffer.* that needs updating.

This CL completes the work of adding output buffer bounds checks to
vp9/encoder/vp9_bitstream.c.

Bug: webm:1844
Change-Id: I6b362be572852ee51d96023b35bfb334faada7e1
(cherry picked from commit d790001)

vp9/encoder/vp9_bitstream.c

@@ -1393,14 +1393,19 @@ void vp9_pack_bitstream(VP9_COMP *cpi, uint8_t *dest, size_t dest_size,
   uint8_t *data = dest;
   size_t data_size = dest_size;
   size_t uncompressed_hdr_size, compressed_hdr_size;
-  struct vpx_write_bit_buffer wb = { data, 0 };
+  struct vpx_write_bit_buffer wb;
   struct vpx_write_bit_buffer saved_wb;
 
 #if CONFIG_BITSTREAM_DEBUG
   bitstream_queue_reset_write();
 #endif
 
+  vpx_wb_init(&wb, data, data_size);
   write_uncompressed_header(cpi, &wb);
+  if (vpx_wb_has_error(&wb)) {
+    vpx_internal_error(&cm->error, VPX_CODEC_ERROR,
+                       "vp9_pack_bitstream: output buffer full");
+  }
 
   // Skip the rest coding process if use show existing frame.
   if (cm->show_existing_frame) {
@@ -1413,6 +1418,10 @@ void vp9_pack_bitstream(VP9_COMP *cpi, uint8_t *dest, size_t dest_size,
   saved_wb = wb;
   // don't know in advance compressed header size
   vpx_wb_write_literal(&wb, 0, 16);
+  if (vpx_wb_has_error(&wb)) {
+    vpx_internal_error(&cm->error, VPX_CODEC_ERROR,
+                       "vp9_pack_bitstream: output buffer full");
+  }
 
   uncompressed_hdr_size = vpx_wb_bytes_written(&wb);
   data += uncompressed_hdr_size;
@@ -1428,6 +1437,7 @@ void vp9_pack_bitstream(VP9_COMP *cpi, uint8_t *dest, size_t dest_size,
                        "compressed_hdr_size > 16 bits");
   }
   vpx_wb_write_literal(&saved_wb, (int)compressed_hdr_size, 16);
+  assert(!vpx_wb_has_error(&saved_wb));
 
   data += encode_tiles(cpi, data, data_size);
 

vpx_dsp/bitwriter_buffer.c

@@ -8,24 +8,43 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <assert.h>
 #include <limits.h>
 #include <stdlib.h>
 
 #include "./vpx_config.h"
 #include "./bitwriter_buffer.h"
 
+void vpx_wb_init(struct vpx_write_bit_buffer *wb, uint8_t *bit_buffer,
+                 size_t size) {
+  wb->error = 0;
+  wb->bit_offset = 0;
+  wb->size = size;
+  wb->bit_buffer = bit_buffer;
+}
+
+int vpx_wb_has_error(const struct vpx_write_bit_buffer *wb) {
+  return wb->error;
+}
+
 size_t vpx_wb_bytes_written(const struct vpx_write_bit_buffer *wb) {
+  assert(!wb->error);
   return wb->bit_offset / CHAR_BIT + (wb->bit_offset % CHAR_BIT > 0);
 }
 
 void vpx_wb_write_bit(struct vpx_write_bit_buffer *wb, int bit) {
+  if (wb->error) return;
   const int off = (int)wb->bit_offset;
   const int p = off / CHAR_BIT;
   const int q = CHAR_BIT - 1 - off % CHAR_BIT;
+  if ((size_t)p >= wb->size) {
+    wb->error = 1;
+    return;
+  }
   if (q == CHAR_BIT - 1) {
     wb->bit_buffer[p] = bit << q;
   } else {
-    wb->bit_buffer[p] &= ~(1 << q);
+    assert((wb->bit_buffer[p] & (1 << q)) == 0);
     wb->bit_buffer[p] |= bit << q;
   }
   wb->bit_offset = off + 1;

vpx_dsp/bitwriter_buffer.h

@@ -18,10 +18,24 @@ extern "C" {
 #endif
 
 struct vpx_write_bit_buffer {
-  uint8_t *bit_buffer;
+  // Whether there has been an error.
+  int error;
+  // We maintain the invariant that bit_offset <= size * CHAR_BIT, i.e., we
+  // never write beyond the end of bit_buffer. If bit_offset would be
+  // incremented to be greater than size * CHAR_BIT, leave bit_offset unchanged
+  // and set error to 1.
   size_t bit_offset;
+  // Size of bit_buffer in bytes.
+  size_t size;
+  uint8_t *bit_buffer;
 };
 
+void vpx_wb_init(struct vpx_write_bit_buffer *wb, uint8_t *bit_buffer,
+                 size_t size);
+
+int vpx_wb_has_error(const struct vpx_write_bit_buffer *wb);
+
+// Must not be called if vpx_wb_has_error(wb) returns true.
 size_t vpx_wb_bytes_written(const struct vpx_write_bit_buffer *wb);
 
 void vpx_wb_write_bit(struct vpx_write_bit_buffer *wb, int bit);