[Libreoffice-commits] core.git: sw/source

[Stephan Bergmann]

 sw/source/core/layout/frmtool.cxx |    6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

New commits:
commit 398d10a506f9f1c4109d012f22f0e85ee571fe9f
Author: Stephan Bergmann <sbergman at redhat.com>
Date:   Wed Feb 28 09:01:18 2018 +0100

    UBSan: signed integer overflow
    
    It apparently happens during test_resize_table_with_keyboard_tdf53460
    (UITest_writer_tests) that CalcRowRstHeight doesn't get into the while loop at
    all, and thus returned LONG_MAX where it presumably should have returned 0:
    
    > /sw/source/core/layout/frmtool.cxx:3149:24: runtime error: signed integer overflow: 9223372036854775807 + 9223372036854775807 cannot be represented in type 'long'
    >     #0 0x2b7365e0b23c in lcl_CalcCellRstHeight(SwLayoutFrame*) /sw/source/core/layout/frmtool.cxx:3149:24
    >     #1 0x2b7365e085a4 in CalcRowRstHeight(SwLayoutFrame*) /sw/source/core/layout/frmtool.cxx:3162:43
    >     #2 0x2b7365e0b1f8 in lcl_CalcCellRstHeight(SwLayoutFrame*) /sw/source/core/layout/frmtool.cxx:3149:27
    >     #3 0x2b7365e085a4 in CalcRowRstHeight(SwLayoutFrame*) /sw/source/core/layout/frmtool.cxx:3162:43
    >     #4 0x2b736478608b in lcl_SetSelLineHeight(SwTableLine*, CR_SetLineHeight&, long, bool) /sw/source/core/doc/tblrwcl.cxx:3950:30
    >     #5 0x2b7364779d8e in SwTable::SetRowHeight(SwTableBox&, TableChgWidthHeightType, long, long, SwUndo**) /sw/source/core/doc/tblrwcl.cxx:4165:28
    >     #6 0x2b7364a775b9 in SwDoc::SetColRowWidthHeight(SwTableBox&, TableChgWidthHeightType, long, long) /sw/source/core/docnode/ndtbl.cxx:3975:16
    >     #7 0x2b7365824397 in SwFEShell::SetColRowWidthHeight(TableChgWidthHeightType, unsigned short) /sw/source/core/frmedt/fetab.cxx:2236:17
    >     #8 0x2b736ac1f100 in SwEditWin::KeyInput(KeyEvent const&) /sw/source/uibase/docvw/edtwin.cxx:2644:50
    >     #9 0x2b730315ee16 in WindowUIObject::execute(rtl::OUString const&, std::__debug::map<rtl::OUString const, rtl::OUString, std::less<rtl::OUString const>, std::allocator<std::pair<rtl::OUString const, rtl::OUString> > > const&) /vcl/source/uitest/uiobject.cxx:343:17
    >     #10 0x2b736b9960fc in SwEditWinUIObject::execute(rtl::OUString const&, std::__debug::map<rtl::OUString const, rtl::OUString, std::less<rtl::OUString const>, std::allocator<std::pair<rtl::OUString const, rtl::OUString> > > const&) /sw/source/uibase/uitest/uiobject.cxx:115:9
    >     #11 0x2b73031e89c2 in UIObjectUnoObj::executeAction(rtl::OUString const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&)::$_0::operator()() const /vcl/source/uitest/uno/uiobject_uno.cxx:145:9
    >     #12 0x2b73031e7399 in std::_Function_handler<void (), UIObjectUnoObj::executeAction(rtl::OUString const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&)::$_0>::_M_invoke(std::_Any_data const&) /home/tdf/lode/opt_private/lib/gcc/x86_64-unknown-linux-gnu/5.2.0/../../../../include/c++/5.2.0/functional:1871:2
    >     #13 0x2b73003d6ba0 in std::function<void ()>::operator()() const /home/tdf/lode/opt_private/lib/gcc/x86_64-unknown-linux-gnu/5.2.0/../../../../include/c++/5.2.0/functional:2271:14
    >     #14 0x2b73031e5f6a in (anonymous namespace)::ExecuteWrapper::ExecuteActionHdl(Timer*) /vcl/source/uitest/uno/uiobject_uno.cxx:92:13
    ...
    
    Change-Id: I2263908e3136cc5205f838723e7e8a6ab18d6073
    Reviewed-on: https://gerrit.libreoffice.org/50489
    Tested-by: Jenkins <ci at libreoffice.org>
    Reviewed-by: Miklos Vajna <vmiklos at collabora.co.uk>

diff --git a/sw/source/core/layout/frmtool.cxx b/sw/source/core/layout/frmtool.cxx
index f5a1be87b1ec..0df9ee9c5900 100644
--- a/sw/source/core/layout/frmtool.cxx
+++ b/sw/source/core/layout/frmtool.cxx
@@ -3155,8 +3155,12 @@ static SwTwips lcl_CalcCellRstHeight( SwLayoutFrame *pCell )
 
 SwTwips CalcRowRstHeight( SwLayoutFrame *pRow )
 {
-    SwTwips nRstHeight = LONG_MAX;
     SwFrame *pLow = pRow->Lower();
+    if (!(pLow && pLow->IsLayoutFrame()))
+    {
+        return 0;
+    }
+    SwTwips nRstHeight = LONG_MAX;
     while (pLow && pLow->IsLayoutFrame())
     {
         nRstHeight = std::min(nRstHeight, ::lcl_CalcCellRstHeight(static_cast<SwLayoutFrame*>(pLow)));