[poppler] 2 commits - poppler/JArithmeticDecoder.cc poppler/JArithmeticDecoder.h poppler/JBIG2Stream.cc poppler/JBIG2Stream.h

[GitLab Mirror]

 poppler/JArithmeticDecoder.cc |    6 ++++--
 poppler/JArithmeticDecoder.h  |    1 +
 poppler/JBIG2Stream.cc        |   28 ++++++++++++++++++++++++----
 poppler/JBIG2Stream.h         |    2 +-
 4 files changed, 30 insertions(+), 7 deletions(-)

New commits:
commit 8c4d5da844efd1b9e99ae0304d6625170a069d4b
Author: Even Rouault <even.rouault at spatialys.com>
Date:   Sun Aug 9 19:09:27 2020 +0200

    JBIG2: avoid potential undefined bit-wise shift

diff --git a/poppler/JBIG2Stream.cc b/poppler/JBIG2Stream.cc
index 4d78abfc..da3625b1 100644
--- a/poppler/JBIG2Stream.cc
+++ b/poppler/JBIG2Stream.cc
@@ -4056,6 +4056,9 @@ bool JBIG2Stream::resetIntStats(int symCodeLen)
     iardwStats->reset();
     iardhStats->reset();
     iariStats->reset();
+    if (symCodeLen + 1 >= 31) {
+        return false;
+    }
     if (iaidStats != nullptr && iaidStats->getContextSize() == 1 << (symCodeLen + 1)) {
         iaidStats->reset();
     } else {
commit 9e853438c5e9d56c07141220f2b30d7215ee9278
Author: Even Rouault <even.rouault at spatialys.com>
Date:   Sun Aug 9 19:07:11 2020 +0200

    JBIG2: avoid abort() on large memory allocation
    
    Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=24772
    
    When numInputSyms + numNewSyms is large enough, a fatal out of memory
    allocation can occur in JArithmeticDecoderStats() constructor per
    
    ```
        #0 0xf7f6bf19 in [vdso]
        #1 0xf7d40d08 in gsignal (/lib32/libc.so.6+0x2bd08)
        #2 0xf7d42206 in abort (/lib32/libc.so.6+0x2d206)
        #3 0xbdc0049 in gmalloc(unsigned int, bool) gdal/poppler/goo/gmem.h:52:5
        #4 0xbdf3c61 in gmallocn(int, int, bool) gdal/poppler/goo/gmem.h:119:12
        #5 0xc1391fd in JArithmeticDecoderStats::JArithmeticDecoderStats(int) gdal/poppler/poppler/JArithmeticDecoder.cc:36:30
        #6 0xc1130d5 in JBIG2Stream::resetIntStats(int) gdal/poppler/poppler/JBIG2Stream.cc:4052:25
        #7 0xc1083df in JBIG2Stream::readSymbolDictSeg(unsigned int, unsigned int, unsigned int*, unsigned int) gdal/poppler/poppler/JBIG2Stream.cc:1624:9
        #8 0xc105305 in JBIG2Stream::readSegments() gdal/poppler/poppler/JBIG2Stream.cc:1318:18
        #9 0xc103f5a in JBIG2Stream::reset() gdal/poppler/poppler/JBIG2Stream.cc:1142:5
    ```
    
    Avoid it and return nicely.

diff --git a/poppler/JArithmeticDecoder.cc b/poppler/JArithmeticDecoder.cc
index 833ba5fb..843bcef5 100644
--- a/poppler/JArithmeticDecoder.cc
+++ b/poppler/JArithmeticDecoder.cc
@@ -33,8 +33,10 @@
 JArithmeticDecoderStats::JArithmeticDecoderStats(int contextSizeA)
 {
     contextSize = contextSizeA;
-    cxTab = (unsigned char *)gmallocn(contextSize, sizeof(unsigned char));
-    reset();
+    cxTab = (unsigned char *)gmallocn_checkoverflow(contextSize, sizeof(unsigned char));
+    if (cxTab) {
+        reset();
+    }
 }
 
 JArithmeticDecoderStats::~JArithmeticDecoderStats()
diff --git a/poppler/JArithmeticDecoder.h b/poppler/JArithmeticDecoder.h
index c8604e55..ce87d01f 100644
--- a/poppler/JArithmeticDecoder.h
+++ b/poppler/JArithmeticDecoder.h
@@ -44,6 +44,7 @@ public:
     int getContextSize() { return contextSize; }
     void copyFrom(JArithmeticDecoderStats *stats);
     void setEntry(unsigned int cx, int i, int mps);
+    bool isValid() const { return cxTab != nullptr; }
 
 private:
     unsigned char *cxTab; // cxTab[cx] = (i[cx] << 1) + mps[cx]
diff --git a/poppler/JBIG2Stream.cc b/poppler/JBIG2Stream.cc
index b22c4628..4d78abfc 100644
--- a/poppler/JBIG2Stream.cc
+++ b/poppler/JBIG2Stream.cc
@@ -1621,7 +1621,9 @@ bool JBIG2Stream::readSymbolDictSeg(unsigned int segNum, unsigned int length, un
         } else {
             resetGenericStats(sdTemplate, nullptr);
         }
-        resetIntStats(symCodeLen);
+        if (!resetIntStats(symCodeLen)) {
+            goto syntaxError;
+        }
         arithDecoder->start();
     }
 
@@ -1716,6 +1718,9 @@ bool JBIG2Stream::readSymbolDictSeg(unsigned int segNum, unsigned int length, un
                     huffDecoder->reset();
                     arithDecoder->start();
                 } else {
+                    if (iaidStats == nullptr) {
+                        goto syntaxError;
+                    }
                     symID = arithDecoder->decodeIAID(symCodeLen, iaidStats);
                     arithDecoder->decodeInt(&refDX, iardxStats);
                     arithDecoder->decodeInt(&refDY, iardyStats);
@@ -2127,7 +2132,9 @@ void JBIG2Stream::readTextRegionSeg(unsigned int segNum, bool imm, bool lossless
     }
 
     if (!huff) {
-        resetIntStats(symCodeLen);
+        if (!resetIntStats(symCodeLen)) {
+            return;
+        }
         arithDecoder->start();
     }
     if (refine) {
@@ -2252,6 +2259,10 @@ JBIG2Bitmap *JBIG2Stream::readTextRegion(bool huff, bool refine, int w, int h, u
                     symID = huffDecoder->readBits(symCodeLen);
                 }
             } else {
+                if (iaidStats == nullptr) {
+                    delete bitmap;
+                    return nullptr;
+                }
                 symID = arithDecoder->decodeIAID(symCodeLen, iaidStats);
             }
 
@@ -4030,7 +4041,7 @@ void JBIG2Stream::resetRefinementStats(unsigned int templ, JArithmeticDecoderSta
     }
 }
 
-void JBIG2Stream::resetIntStats(int symCodeLen)
+bool JBIG2Stream::resetIntStats(int symCodeLen)
 {
     iadhStats->reset();
     iadwStats->reset();
@@ -4045,12 +4056,18 @@ void JBIG2Stream::resetIntStats(int symCodeLen)
     iardwStats->reset();
     iardhStats->reset();
     iariStats->reset();
-    if (iaidStats->getContextSize() == 1 << (symCodeLen + 1)) {
+    if (iaidStats != nullptr && iaidStats->getContextSize() == 1 << (symCodeLen + 1)) {
         iaidStats->reset();
     } else {
         delete iaidStats;
         iaidStats = new JArithmeticDecoderStats(1 << (symCodeLen + 1));
+        if (!iaidStats->isValid()) {
+            delete iaidStats;
+            iaidStats = nullptr;
+            return false;
+        }
     }
+    return true;
 }
 
 bool JBIG2Stream::readUByte(unsigned int *x)
diff --git a/poppler/JBIG2Stream.h b/poppler/JBIG2Stream.h
index c8c33954..69ab587b 100644
--- a/poppler/JBIG2Stream.h
+++ b/poppler/JBIG2Stream.h
@@ -88,7 +88,7 @@ private:
     void discardSegment(unsigned int segNum);
     void resetGenericStats(unsigned int templ, JArithmeticDecoderStats *prevStats);
     void resetRefinementStats(unsigned int templ, JArithmeticDecoderStats *prevStats);
-    void resetIntStats(int symCodeLen);
+    bool resetIntStats(int symCodeLen);
     bool readUByte(unsigned int *x);
     bool readByte(int *x);
     bool readUWord(unsigned int *x);