From 692ef3e9d9d0839dc29d368b5332e4840df60374 Mon Sep 17 00:00:00 2001
From: Ari Koivula <ari@koivu.la>
Date: Fri, 8 Nov 2013 16:13:48 +0200
Subject: [PATCH] Add guard against illegal SAO edge offsets.

---
 src/encoder.c | 13 ++++++++-----
 src/sao.c     |  9 +++++++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/encoder.c b/src/encoder.c
index 624f647e..15e2ce63 100644
--- a/src/encoder.c
+++ b/src/encoder.c
@@ -799,11 +799,14 @@ void encode_sao_color(encoder_control *encoder, sao_info *sao, color_index color
   if (sao->type != SAO_TYPE_NONE) {
     sao_eo_cat i;
   
-    for (i = SAO_EO_CAT1; i <= SAO_EO_CAT4; ++i) {
-      //CABAC_BIN_EP(&cabac, abs(sao->offsets[i]), "sao_offset_abs");
-      // TR cMax=7 (for 8bit), cRiseParam=0
-      cabac_write_unary_max_symbol_ep(&cabac, abs(sao->offsets[i]), 
-                                      SAO_ABS_OFFSET_MAX);
+    // TR cMax=7 (for 8bit), cRiseParam=0
+    for (i = SAO_EO_CAT1; i <= SAO_EO_CAT2; ++i) {
+      assert(sao->offsets[i] >= 0);
+      cabac_write_unary_max_symbol_ep(&cabac, sao->offsets[i], SAO_ABS_OFFSET_MAX);
+    }
+    for (i = SAO_EO_CAT3; i <= SAO_EO_CAT4; ++i) {
+      assert(sao->offsets[i] <= 0);
+      cabac_write_unary_max_symbol_ep(&cabac, -sao->offsets[i], SAO_ABS_OFFSET_MAX);
     }
 
     if (sao->type == SAO_TYPE_BAND) {
diff --git a/src/sao.c b/src/sao.c
index bb9b129b..51875567 100644
--- a/src/sao.c
+++ b/src/sao.c
@@ -249,6 +249,15 @@ void sao_search_best_mode(const pixel *data, const pixel *recdata,
         offset = (cat_sum + (cat_cnt >> 1)) / cat_cnt;
         offset = CLIP(-SAO_ABS_OFFSET_MAX, SAO_ABS_OFFSET_MAX, offset);
       }
+
+      // Sharpening edge offsets can't be encoded, so set them to 0 here.
+      if (edge_cat >= SAO_EO_CAT1 && edge_cat <= SAO_EO_CAT2 && offset < 0) {
+        offset = 0;
+      }
+      if (edge_cat >= SAO_EO_CAT3 && edge_cat <= SAO_EO_CAT4 && offset > 0) {
+        offset = 0;
+      }
+
       edge_offset[edge_cat] = offset;
       // The ddistortion is amount by which the SSE of data changes. It should
       // be negative for all categories, if offset was chosen correctly.