From 720731083868f4c914daf6810bedad34cd76bf4d Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 1 Aug 2017 23:51:57 -0700 Subject: [PATCH] Fix two cases of signed integer overflow. This tweaks two cases where there is potential for signed integer overflow: * in `rewind_scanlines`, we switch to unsigned ints * in `not_equal`, we switch for operations which cannot overflow. These cases were caught by matplotlib-using tests being run under [ASan](https://github.com/google/sanitizers) with a check for signed integer overflow. --- .../include/agg_rasterizer_scanline_aa_nogamma.h | 2 +- extern/agg24-svn/include/agg_scanline_storage_aa.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extern/agg24-svn/include/agg_rasterizer_scanline_aa_nogamma.h b/extern/agg24-svn/include/agg_rasterizer_scanline_aa_nogamma.h index 6c3d96fc6160..7729b3359a10 100644 --- a/extern/agg24-svn/include/agg_rasterizer_scanline_aa_nogamma.h +++ b/extern/agg24-svn/include/agg_rasterizer_scanline_aa_nogamma.h @@ -60,7 +60,7 @@ namespace agg int not_equal(int ex, int ey, const cell_aa&) const { - return (ex - x) | (ey - y); + return ex != x || ey != y; } }; diff --git a/extern/agg24-svn/include/agg_scanline_storage_aa.h b/extern/agg24-svn/include/agg_scanline_storage_aa.h index 7148c8992224..b3471fce7682 100644 --- a/extern/agg24-svn/include/agg_scanline_storage_aa.h +++ b/extern/agg24-svn/include/agg_scanline_storage_aa.h @@ -720,10 +720,10 @@ namespace agg m_ptr = m_data; if(m_ptr < m_end) { - m_min_x = read_int32() + m_dx; - m_min_y = read_int32() + m_dy; - m_max_x = read_int32() + m_dx; - m_max_y = read_int32() + m_dy; + m_min_x = read_int32u() + m_dx; + m_min_y = read_int32u() + m_dy; + m_max_x = read_int32u() + m_dx; + m_max_y = read_int32u() + m_dy; } return m_ptr < m_end; }