[ooo-build-commit] .: patches/dev300

Kohei Yoshida kohei at kemper.freedesktop.org
Thu Feb 11 14:03:03 PST 2010


 patches/dev300/apply                              |    3 
 patches/dev300/calc-perf-extref-shrink-range.diff |  762 ++++++++++++++++++++++
 2 files changed, 765 insertions(+)

New commits:
commit a31406ed613cb41fb93df9bb35a84b38a1ff0aaa
Author: Kohei Yoshida <kyoshida at novell.com>
Date:   Thu Feb 11 17:00:44 2010 -0500

    Performance improvement on external reference handling.
    
    * patches/dev300/apply:
    * patches/dev300/calc-perf-extref-shrink-range.diff: re-designed
      the external ref manager to reduce cache size & improve
      performance on cached data lookup. (i#109168)

diff --git a/patches/dev300/apply b/patches/dev300/apply
index 694392f..eaa3c03 100644
--- a/patches/dev300/apply
+++ b/patches/dev300/apply
@@ -3534,6 +3534,9 @@ calc-english-func-names-officecfg.diff, i#38765, kohei
 # Keep track of cells with SUBTOTAL functions the right way.
 calc-subtotal-function-update.diff, n#578802, kohei
 
+# Reduce cache table size & improve lookup performance on external references.
+calc-perf-extref-shrink-range.diff, i#109168, kohei
+
 [ NovellEvaluation ]
 # enable the Subscription menu
 novell-subscription-enable-in-menu.diff
diff --git a/patches/dev300/calc-perf-extref-shrink-range.diff b/patches/dev300/calc-perf-extref-shrink-range.diff
new file mode 100644
index 0000000..384c2e7
--- /dev/null
+++ b/patches/dev300/calc-perf-extref-shrink-range.diff
@@ -0,0 +1,762 @@
+diff --git sc/inc/externalrefmgr.hxx sc/inc/externalrefmgr.hxx
+index 22e114d..2906113 100644
+--- sc/inc/externalrefmgr.hxx
++++ sc/inc/externalrefmgr.hxx
+@@ -39,6 +39,7 @@
+ #include "vcl/timer.hxx"
+ #include "svtools/zforlist.hxx"
+ #include "scmatrix.hxx"
++#include "rangelst.hxx"
+ 
+ #include <hash_map>
+ #include <hash_set>
+@@ -129,6 +130,15 @@ public:
+     class Table;
+     friend class ScExternalRefCache::Table;
+ 
++    /** 
++     * Represents a single cached table in an external document.  It only 
++     * stores non-empty cells; empty cells should never be stored in the data 
++     * cache. Instead, cached ranges should be used to determine whether or 
++     * not a cell is empty or needs fetching from the source document.  If a 
++     * cell's value is not stored but its address is within the cached ranges, 
++     * that cell is already queried in the source document and we know it's 
++     * empty. 
++     */
+     class Table
+     {
+     public:
+@@ -143,7 +153,7 @@ public:
+         Table();
+         ~Table();
+ 
+-        SC_DLLPUBLIC void setCell(SCCOL nCol, SCROW nRow, TokenRef pToken, sal_uInt32 nFmtIndex = 0);
++        SC_DLLPUBLIC void setCell(SCCOL nCol, SCROW nRow, TokenRef pToken, sal_uInt32 nFmtIndex = 0, bool bSetCacheRange = true);
+         TokenRef getCell(SCCOL nCol, SCROW nRow, sal_uInt32* pnFmtIndex = NULL) const;
+         bool hasRow( SCROW nRow ) const;
+         /** Set/clear referenced status flag only if current status is not 
+@@ -154,14 +164,35 @@ public:
+         ReferencedFlag getReferencedFlag() const;
+         bool isReferenced() const;
+         /// Obtain a sorted vector of rows.
+-        void getAllRows(::std::vector<SCROW>& rRows) const;
++        void getAllRows(::std::vector<SCROW>& rRows, SCROW nLow = 0, SCROW nHigh = MAXROW) const;
+         /// Obtain a sorted vector of columns.
+-        void getAllCols(SCROW nRow, ::std::vector<SCCOL>& rCols) const;
++        void getAllCols(SCROW nRow, ::std::vector<SCCOL>& rCols, SCCOL nLow = 0, SCCOL nHigh = MAXCOL) const;
+         void getAllNumberFormats(::std::vector<sal_uInt32>& rNumFmts) const;
++        const ScRangeList& getCachedRanges() const;
++        bool isRangeCached(SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2) const;
++
++        void setCachedCell(SCCOL nCol, SCROW nRow);
++        void setCachedCellRange(SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2);
++
++        /** 
++         * Call this to mark the entire table "cached".  This will prevent all
++         * future attempts to access the source document even when non-cached 
++         * cells are queried.  In such case, non-cached cells are treated as 
++         * empty cells.  Useful when loading a document with own external data 
++         * cache. 
++         */
++        SC_DLLPUBLIC void setWholeTableCached();
++    private:
++        bool isInCachedRanges(SCCOL nCol, SCROW nRow) const;
++        TokenRef getEmptyOrNullToken(SCCOL nCol, SCROW nRow) const;
+ 
+     private:
+-        RowsDataType   maRows;
+-        ReferencedFlag meReferenced;
++        /** Data cache */
++        RowsDataType                    maRows;
++        /** Collection of individual cached ranges.  The table ranges are
++         *  not used & always zero. */
++        ScRangeList                     maCachedRanges;
++        ReferencedFlag                  meReferenced;
+     };
+ 
+     typedef ::boost::shared_ptr<Table>      TableTypeRef;
+@@ -184,8 +215,7 @@ public:
+      * @return pointer to the token instance in the cache. 
+      */
+     ScExternalRefCache::TokenRef getCellData(
+-        sal_uInt16 nFileId, const String& rTabName, SCCOL nCol, SCROW nRow,
+-        bool bEmptyCellOnNull, bool bWriteEmpty, sal_uInt32* pnFmtIndex);
++        sal_uInt16 nFileId, const String& rTabName, SCCOL nCol, SCROW nRow, sal_uInt32* pnFmtIndex);
+ 
+     /**
+      * Get a cached cell range data.
+@@ -195,12 +225,12 @@ public:
+      *         guaranteed if the TokenArrayRef is properly used..
+      */
+     ScExternalRefCache::TokenArrayRef getCellRangeData(
+-        sal_uInt16 nFileId, const String& rTabName, const ScRange& rRange, bool bEmptyCellOnNull, bool bWriteEmpty);
++        sal_uInt16 nFileId, const String& rTabName, const ScRange& rRange);
+ 
+     ScExternalRefCache::TokenArrayRef getRangeNameTokens(sal_uInt16 nFileId, const String& rName);
+     void setRangeNameTokens(sal_uInt16 nFileId, const String& rName, TokenArrayRef pArray);
+ 
+-    void setCellData(sal_uInt16 nFileId, const String& rTabName, SCROW nRow, SCCOL nCol, TokenRef pToken, sal_uInt32 nFmtIndex);
++    void setCellData(sal_uInt16 nFileId, const String& rTabName, SCCOL nCol, SCROW nRow, TokenRef pToken, sal_uInt32 nFmtIndex);
+ 
+     struct SingleRangeData
+     {
+diff --git sc/source/core/data/documen5.cxx sc/source/core/data/documen5.cxx
+index 24cbf48..564bf3f 100644
+--- sc/source/core/data/documen5.cxx
++++ sc/source/core/data/documen5.cxx
+@@ -930,6 +930,9 @@ void ScDocument::UpdateChartListenerCollection()
+                 SdrPage* pPage = pDrawLayer->GetPage(static_cast<sal_uInt16>(nTab));
+                 DBG_ASSERT(pPage,"Page ?");
+ 
++                if (!pPage)
++                    continue;
++
+                 SdrObjListIter aIter( *pPage, IM_DEEPNOGROUPS );
+                 SdrObject* pObject = aIter.Next();
+                 while (pObject)
+diff --git sc/source/core/data/document.cxx sc/source/core/data/document.cxx
+index 8c4c4a3..eb83c8a 100644
+--- sc/source/core/data/document.cxx
++++ sc/source/core/data/document.cxx
+@@ -713,6 +713,10 @@ bool ScDocument::ShrinkToDataArea(SCTAB nTab, SCCOL& rStartCol, SCROW& rStartRow
+     if (nRow2 < rEndRow)
+         rEndRow = nRow2;
+ 
++    if (rStartCol > rEndCol || rStartRow > rEndRow)
++        // invalid range.
++        return false;
++
+     return true;  // success!
+ }
+ 
+diff --git sc/source/filter/excel/xilink.cxx sc/source/filter/excel/xilink.cxx
+index 0026821..b76baf6 100644
+--- sc/source/filter/excel/xilink.cxx
++++ sc/source/filter/excel/xilink.cxx
+@@ -570,6 +570,7 @@ void XclImpSupbook::LoadCachedValues()
+         const String& rTabName = pTab->GetTabName();
+         ScExternalRefCache::TableTypeRef pCacheTable = pRefMgr->getCacheTable(nFileId, rTabName, true);
+         pTab->LoadCachedValues(pCacheTable);
++        pCacheTable->setWholeTableCached();
+     }
+ }
+ 
+diff --git sc/source/filter/xml/xmltabi.cxx sc/source/filter/xml/xmltabi.cxx
+index a2995a6..9fa3a12 100644
+--- sc/source/filter/xml/xmltabi.cxx
++++ sc/source/filter/xml/xmltabi.cxx
+@@ -217,6 +217,7 @@ ScXMLTableContext::ScXMLTableContext( ScXMLImport& rImport,
+                 ScExternalRefManager* pRefMgr = pDoc->GetExternalRefManager();
+                 pExternalRefInfo->mnFileId = pRefMgr->getExternalFileId(aExtUrl);
+                 pExternalRefInfo->mpCacheTable = pRefMgr->getCacheTable(pExternalRefInfo->mnFileId, aExtTabName, true);
++                pExternalRefInfo->mpCacheTable->setWholeTableCached();
+             }
+         }
+         else
+diff --git sc/source/ui/docshell/externalrefmgr.cxx sc/source/ui/docshell/externalrefmgr.cxx
+index da04ce2..9081d07 100644
+--- sc/source/ui/docshell/externalrefmgr.cxx
++++ sc/source/ui/docshell/externalrefmgr.cxx
+@@ -68,6 +68,8 @@
+ #include <memory>
+ #include <algorithm>
+ 
++#include <boost/scoped_ptr.hpp>
++
+ using ::std::auto_ptr;
+ using ::com::sun::star::uno::Any;
+ using ::rtl::OUString;
+@@ -173,7 +175,7 @@ bool ScExternalRefCache::Table::isReferenced() const
+     return meReferenced != UNREFERENCED;
+ }
+ 
+-void ScExternalRefCache::Table::setCell(SCCOL nCol, SCROW nRow, TokenRef pToken, sal_uInt32 nFmtIndex)
++void ScExternalRefCache::Table::setCell(SCCOL nCol, SCROW nRow, TokenRef pToken, sal_uInt32 nFmtIndex, bool bSetCacheRange)
+ {
+     using ::std::pair;
+     RowsDataType::iterator itrRow = maRows.find(nRow);
+@@ -196,6 +198,8 @@ void ScExternalRefCache::Table::setCell(SCCOL nCol, SCROW nRow, TokenRef pToken,
+     aCell.mxToken = pToken;
+     aCell.mnFmtIndex = nFmtIndex;
+     rRow.insert(RowDataType::value_type(nCol, aCell));
++    if (bSetCacheRange)
++        setCachedCell(nCol, nRow);
+ }
+ 
+ ScExternalRefCache::TokenRef ScExternalRefCache::Table::getCell(SCCOL nCol, SCROW nRow, sal_uInt32* pnFmtIndex) const
+@@ -204,7 +208,7 @@ ScExternalRefCache::TokenRef ScExternalRefCache::Table::getCell(SCCOL nCol, SCRO
+     if (itrTable == maRows.end())
+     {
+         // this table doesn't have the specified row.
+-        return TokenRef();
++        return getEmptyOrNullToken(nCol, nRow);
+     }
+ 
+     const RowDataType& rRowData = itrTable->second;
+@@ -212,7 +216,7 @@ ScExternalRefCache::TokenRef ScExternalRefCache::Table::getCell(SCCOL nCol, SCRO
+     if (itrRow == rRowData.end())
+     {
+         // this row doesn't have the specified column.
+-        return TokenRef();
++        return getEmptyOrNullToken(nCol, nRow);
+     }
+ 
+     const Cell& rCell = itrRow->second;
+@@ -228,20 +232,21 @@ bool ScExternalRefCache::Table::hasRow( SCROW nRow ) const
+     return itrRow != maRows.end();
+ }
+ 
+-void ScExternalRefCache::Table::getAllRows(vector<SCROW>& rRows) const
++void ScExternalRefCache::Table::getAllRows(vector<SCROW>& rRows, SCROW nLow, SCROW nHigh) const
+ {
+     vector<SCROW> aRows;
+     aRows.reserve(maRows.size());
+     RowsDataType::const_iterator itr = maRows.begin(), itrEnd = maRows.end();
+     for (; itr != itrEnd; ++itr)
+-        aRows.push_back(itr->first);
++        if (nLow <= itr->first && itr->first <= nHigh)
++            aRows.push_back(itr->first);
+ 
+     // hash map is not ordered, so we need to explicitly sort it.
+     ::std::sort(aRows.begin(), aRows.end());
+     rRows.swap(aRows);
+ }
+ 
+-void ScExternalRefCache::Table::getAllCols(SCROW nRow, vector<SCCOL>& rCols) const
++void ScExternalRefCache::Table::getAllCols(SCROW nRow, vector<SCCOL>& rCols, SCCOL nLow, SCCOL nHigh) const
+ {
+     RowsDataType::const_iterator itrRow = maRows.find(nRow);
+     if (itrRow == maRows.end())
+@@ -253,7 +258,8 @@ void ScExternalRefCache::Table::getAllCols(SCROW nRow, vector<SCCOL>& rCols) con
+     aCols.reserve(rRowData.size());
+     RowDataType::const_iterator itrCol = rRowData.begin(), itrColEnd = rRowData.end();
+     for (; itrCol != itrColEnd; ++itrCol)
+-        aCols.push_back(itrCol->first);
++        if (nLow <= itrCol->first && itrCol->first <= nHigh)
++            aCols.push_back(itrCol->first);
+ 
+     // hash map is not ordered, so we need to explicitly sort it.
+     ::std::sort(aCols.begin(), aCols.end());
+@@ -275,6 +281,54 @@ void ScExternalRefCache::Table::getAllNumberFormats(vector<sal_uInt32>& rNumFmts
+     }
+ }
+ 
++const ScRangeList& ScExternalRefCache::Table::getCachedRanges() const
++{
++    return maCachedRanges;
++}
++
++bool ScExternalRefCache::Table::isRangeCached(SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2) const
++{
++    return maCachedRanges.In(ScRange(nCol1, nRow1, 0, nCol2, nRow2, 0));
++}
++
++void ScExternalRefCache::Table::setCachedCell(SCCOL nCol, SCROW nRow)
++{
++    setCachedCellRange(nCol, nRow, nCol, nRow);
++}
++
++void ScExternalRefCache::Table::setCachedCellRange(SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2)
++{
++    ScRange aRange(nCol1, nRow1, 0, nCol2, nRow2, 0);
++    if (!maCachedRanges.Count())
++        maCachedRanges.Append(aRange);
++    else
++        maCachedRanges.Join(aRange);
++
++    String aStr;
++    maCachedRanges.Format(aStr, SCA_VALID);
++}
++
++void ScExternalRefCache::Table::setWholeTableCached()
++{
++    setCachedCellRange(0, 0, MAXCOL, MAXROW);
++}
++
++bool ScExternalRefCache::Table::isInCachedRanges(SCCOL nCol, SCROW nRow) const
++{
++    return maCachedRanges.In(ScRange(nCol, nRow, 0, nCol, nRow, 0));
++}
++
++ScExternalRefCache::TokenRef ScExternalRefCache::Table::getEmptyOrNullToken(
++    SCCOL nCol, SCROW nRow) const
++{
++    if (isInCachedRanges(nCol, nRow))
++    {
++        TokenRef p(new ScEmptyCellToken(false, false));
++        return p;
++    }
++    return TokenRef();
++}
++
+ // ----------------------------------------------------------------------------
+ 
+ ScExternalRefCache::TableName::TableName(const String& rUpper, const String& rReal) :
+@@ -339,8 +393,7 @@ const String* ScExternalRefCache::getRealRangeName(sal_uInt16 nFileId, const Str
+ }
+ 
+ ScExternalRefCache::TokenRef ScExternalRefCache::getCellData(
+-    sal_uInt16 nFileId, const String& rTabName, SCCOL nCol, SCROW nRow, 
+-    bool bEmptyCellOnNull, bool bWriteEmpty, sal_uInt32* pnFmtIndex)
++    sal_uInt16 nFileId, const String& rTabName, SCCOL nCol, SCROW nRow, sal_uInt32* pnFmtIndex)
+ {
+     DocDataType::const_iterator itrDoc = maDocs.find(nFileId);
+     if (itrDoc == maDocs.end())
+@@ -365,18 +418,11 @@ ScExternalRefCache::TokenRef ScExternalRefCache::getCellData(
+         return TokenRef();
+     }
+ 
+-    TokenRef pToken = pTableData->getCell(nCol, nRow, pnFmtIndex);
+-    if (!pToken && bEmptyCellOnNull)
+-    {
+-        pToken.reset(new ScEmptyCellToken(false, false));
+-        if (bWriteEmpty)
+-            pTableData->setCell(nCol, nRow, pToken);
+-    }
+-    return pToken;
++    return pTableData->getCell(nCol, nRow, pnFmtIndex);
+ }
+ 
+ ScExternalRefCache::TokenArrayRef ScExternalRefCache::getCellRangeData(
+-    sal_uInt16 nFileId, const String& rTabName, const ScRange& rRange, bool bEmptyCellOnNull, bool bWriteEmpty)
++    sal_uInt16 nFileId, const String& rTabName, const ScRange& rRange)
+ {
+     DocDataType::iterator itrDoc = maDocs.find(nFileId);
+     if (itrDoc == maDocs.end())
+@@ -406,13 +452,14 @@ ScExternalRefCache::TokenArrayRef ScExternalRefCache::getCellRangeData(
+         return TokenArrayRef();
+ 
+     ScRange aCacheRange( nCol1, nRow1, static_cast<SCTAB>(nTabFirstId), nCol2, nRow2, static_cast<SCTAB>(nTabLastId));
++
+     RangeArrayMap::const_iterator itrRange = rDoc.maRangeArrays.find( aCacheRange);
+     if (itrRange != rDoc.maRangeArrays.end())
+-    {
++        // Cache hit!
+         return itrRange->second;
+-    }
+ 
+-    TokenArrayRef pArray(new ScTokenArray);
++    ::boost::scoped_ptr<ScRange> pNewRange;
++    TokenArrayRef pArray;
+     bool bFirstTab = true;
+     for (size_t nTab = nTabFirstId; nTab <= nTabLastId; ++nTab)
+     {
+@@ -420,27 +467,63 @@ ScExternalRefCache::TokenArrayRef ScExternalRefCache::getCellRangeData(
+         if (!pTab.get())
+             return TokenArrayRef();
+ 
++        SCCOL nDataCol1 = nCol1, nDataCol2 = nCol2;
++        SCROW nDataRow1 = nRow1, nDataRow2 = nRow2;
++
++        if (!pTab->isRangeCached(nDataCol1, nDataRow1, nDataCol2, nDataRow2))
++        {
++            // specified range is not entirely within cached ranges.
++            return TokenArrayRef();
++        }
++
+         ScMatrixRef xMat = new ScMatrix(
+-            static_cast<SCSIZE>(nCol2-nCol1+1), static_cast<SCSIZE>(nRow2-nRow1+1));
++            static_cast<SCSIZE>(nDataCol2-nDataCol1+1), static_cast<SCSIZE>(nDataRow2-nDataRow1+1));
+ 
+-        for (SCROW nRow = nRow1; nRow <= nRow2; ++nRow)
++#if 0
++        // TODO: Switch to this code block once we have support for sparsely-filled 
++        // matrices in ScMatrix.
++
++        // Only fill non-empty cells, for better performance.
++        vector<SCROW> aRows;
++        pTab->getAllRows(aRows, nDataRow1, nDataRow2);
++        for (vector<SCROW>::const_iterator itr = aRows.begin(), itrEnd = aRows.end(); itr != itrEnd; ++itr)
+         {
+-            for (SCCOL nCol = nCol1; nCol <= nCol2; ++nCol)
++            SCROW nRow = *itr;
++            vector<SCCOL> aCols;
++            pTab->getAllCols(nRow, aCols, nDataCol1, nDataCol2);
++            for (vector<SCCOL>::const_iterator itrCol = aCols.begin(), itrColEnd = aCols.end(); itrCol != itrColEnd; ++itrCol)
+             {
++                SCCOL nCol = *itrCol;
+                 TokenRef pToken = pTab->getCell(nCol, nRow);
+                 if (!pToken)
++                    // This should never happen!
++                    return TokenArrayRef();
++
++                SCSIZE nC = nCol - nDataCol1, nR = nRow - nDataRow1;
++                switch (pToken->GetType())
+                 {
+-                    if (bEmptyCellOnNull)
+-                    {
+-                        pToken.reset(new ScEmptyCellToken(false, false));
+-                        if (bWriteEmpty)
+-                            pTab->setCell(nCol, nRow, pToken);
+-                    }
+-                    else
+-                        return TokenArrayRef();
++                    case svDouble:
++                        xMat->PutDouble(pToken->GetDouble(), nC, nR);
++                    break;
++                    case svString:
++                        xMat->PutString(pToken->GetString(), nC, nR);
++                    break;
++                    default:
++                        ;
+                 }
+-
++            }
++        }
++#else
++        // Empty all matrix elements first, and fill only non-empty elements.
++        for (SCROW nRow = nDataRow1; nRow <= nDataRow2; ++nRow)
++        {
++            for (SCCOL nCol = nDataCol1; nCol <= nDataCol2; ++nCol)
++            {
++                TokenRef pToken = pTab->getCell(nCol, nRow);
+                 SCSIZE nC = nCol - nCol1, nR = nRow - nRow1;
++                if (!pToken)
++                    return TokenArrayRef();
++
+                 switch (pToken->GetType())
+                 {
+                     case svDouble:
+@@ -454,17 +537,27 @@ ScExternalRefCache::TokenArrayRef ScExternalRefCache::getCellRangeData(
+                 }
+             }
+         }
++#endif
+ 
+         if (!bFirstTab)
+             pArray->AddOpCode(ocSep);
+ 
+         ScMatrix* pMat2 = xMat;
+         ScMatrixToken aToken(pMat2);
++        if (!pArray)
++            pArray.reset(new ScTokenArray);
+         pArray->AddToken(aToken);
+ 
+         bFirstTab = false;
++
++        if (!pNewRange)
++            pNewRange.reset(new ScRange(nDataCol1, nDataRow1, 0, nDataCol2, nDataRow2, 0));
++        else
++            pNewRange->ExtendTo(ScRange(nDataCol1, nDataRow1, 0, nDataCol2, nDataRow2, 0));
+     }
+-    rDoc.maRangeArrays.insert( RangeArrayMap::value_type( aCacheRange, pArray));
++
++    if (pNewRange)
++        rDoc.maRangeArrays.insert( RangeArrayMap::value_type(*pNewRange, pArray));
+     return pArray;
+ }
+ 
+@@ -495,7 +588,7 @@ void ScExternalRefCache::setRangeNameTokens(sal_uInt16 nFileId, const String& rN
+     pDoc->maRealRangeNameMap.insert(NamePairMap::value_type(aUpperName, rName));
+ }
+ 
+-void ScExternalRefCache::setCellData(sal_uInt16 nFileId, const String& rTabName, SCROW nRow, SCCOL nCol,
++void ScExternalRefCache::setCellData(sal_uInt16 nFileId, const String& rTabName, SCCOL nCol, SCROW nRow,
+                                      TokenRef pToken, sal_uInt32 nFmtIndex)
+ {
+     if (!isDocInitialized(nFileId))
+@@ -520,6 +613,7 @@ void ScExternalRefCache::setCellData(sal_uInt16 nFileId, const String& rTabName,
+         pTableData.reset(new Table);
+ 
+     pTableData->setCell(nCol, nRow, pToken, nFmtIndex);
++    pTableData->setCachedCell(nCol, nRow);
+ }
+ 
+ void ScExternalRefCache::setCellRangeData(sal_uInt16 nFileId, const ScRange& rRange, const vector<SingleRangeData>& rData,
+@@ -565,20 +659,27 @@ void ScExternalRefCache::setCellRangeData(sal_uInt16 nFileId, const ScRange& rRa
+                 SCSIZE nC = nCol - nCol1, nR = nRow - nRow1;
+                 TokenRef pToken;
+                 const ScMatrixRef& pMat = itrData->mpRangeData;
++                if (pMat->IsEmpty(nC, nR))
++                    // Don't cache empty cells.
++                    continue;
++
+                 if (pMat->IsValue(nC, nR))
+                     pToken.reset(new formula::FormulaDoubleToken(pMat->GetDouble(nC, nR)));
+                 else if (pMat->IsString(nC, nR))
+                     pToken.reset(new formula::FormulaStringToken(pMat->GetString(nC, nR)));
+-                else
+-                    pToken.reset(new ScEmptyCellToken(false, false));
+ 
+-                pTabData->setCell(nCol, nRow, pToken);
++                if (pToken)
++                    // Don't mark this cell 'cached' here, for better performance.
++                    pTabData->setCell(nCol, nRow, pToken, 0, false);
+             }
+         }
++        // Mark the whole range 'cached'.
++        pTabData->setCachedCellRange(nCol1, nRow1, nCol2, nRow2);
+     }
+ 
+     size_t nTabLastId = nTabFirstId + rRange.aEnd.Tab() - rRange.aStart.Tab();
+     ScRange aCacheRange( nCol1, nRow1, static_cast<SCTAB>(nTabFirstId), nCol2, nRow2, static_cast<SCTAB>(nTabLastId));
++
+     rDoc.maRangeArrays.insert( RangeArrayMap::value_type( aCacheRange, pArray));
+ }
+ 
+@@ -975,6 +1076,9 @@ ScExternalRefCache::TableTypeRef ScExternalRefCache::getCacheTable(sal_uInt16 nF
+     {
+         // specified table found.
+         if( pnIndex ) *pnIndex = nIndex;
++        if (bCreateNew && !rDoc.maTables[nIndex])
++            rDoc.maTables[nIndex].reset(new Table);
++
+         return rDoc.maTables[nIndex];
+     }
+ 
+@@ -1142,11 +1246,11 @@ static FormulaToken* lcl_convertToToken(ScBaseCell* pCell)
+     return NULL;
+ }
+ 
+-static ScTokenArray* lcl_convertToTokenArray(ScDocument* pSrcDoc, const ScRange& rRange,
++static ScTokenArray* lcl_convertToTokenArray(ScDocument* pSrcDoc, ScRange& rRange,
+                                              vector<ScExternalRefCache::SingleRangeData>& rCacheData)
+ {
+-    const ScAddress& s = rRange.aStart;
+-    const ScAddress& e = rRange.aEnd;
++    ScAddress& s = rRange.aStart;
++    ScAddress& e = rRange.aEnd;
+ 
+     SCTAB nTab1 = s.Tab(), nTab2 = e.Tab();
+     SCCOL nCol1 = s.Col(), nCol2 = e.Col();
+@@ -1160,19 +1264,35 @@ static ScTokenArray* lcl_convertToTokenArray(ScDocument* pSrcDoc, const ScRange&
+         // range to it.
+         return NULL;
+ 
++    ::boost::scoped_ptr<ScRange> pUsedRange;
++
+     auto_ptr<ScTokenArray> pArray(new ScTokenArray);
+     bool bFirstTab = true;
+     vector<ScExternalRefCache::SingleRangeData>::iterator
+         itrCache = rCacheData.begin(), itrCacheEnd = rCacheData.end();
++
+     for (SCTAB nTab = nTab1; nTab <= nTab2 && itrCache != itrCacheEnd; ++nTab, ++itrCache)
+     {
++        // Only loop within the data area.
++        SCCOL nDataCol1 = nCol1, nDataCol2 = nCol2;
++        SCROW nDataRow1 = nRow1, nDataRow2 = nRow2;
++        if (!pSrcDoc->ShrinkToDataArea(nTab, nDataCol1, nDataRow1, nDataCol2, nDataRow2))
++            // no data within specified range.
++            continue;
++
++        if (pUsedRange.get())
++            // Make sure the used area only grows, not shrinks.
++            pUsedRange->ExtendTo(ScRange(nDataCol1, nDataRow1, 0, nDataCol2, nDataRow2, 0));
++        else
++            pUsedRange.reset(new ScRange(nDataCol1, nDataRow1, 0, nDataCol2, nDataRow2, 0));
++
+         ScMatrixRef xMat = new ScMatrix(
+-            static_cast<SCSIZE>(nCol2-nCol1+1),
+-            static_cast<SCSIZE>(nRow2-nRow1+1));
++            static_cast<SCSIZE>(nDataCol2-nDataCol1+1),
++            static_cast<SCSIZE>(nDataRow2-nDataRow1+1));
+ 
+-        for (SCCOL nCol = nCol1; nCol <= nCol2; ++nCol)
++        for (SCCOL nCol = nDataCol1; nCol <= nDataCol2; ++nCol)
+         {
+-            for (SCROW nRow = nRow1; nRow <= nRow2; ++nRow)
++            for (SCROW nRow = nDataRow1; nRow <= nDataRow2; ++nRow)
+             {
+                 SCSIZE nC = nCol - nCol1, nR = nRow - nRow1;
+                 ScBaseCell* pCell;
+@@ -1239,6 +1359,31 @@ static ScTokenArray* lcl_convertToTokenArray(ScDocument* pSrcDoc, const ScRange&
+ 
+         bFirstTab = false;
+     }
++
++    if (!pUsedRange.get())
++        return NULL;
++
++    s.SetCol(pUsedRange->aStart.Col());
++    s.SetRow(pUsedRange->aStart.Row());
++    e.SetCol(pUsedRange->aEnd.Col());
++    e.SetRow(pUsedRange->aEnd.Row());
++
++    return pArray.release();
++}
++
++static ScTokenArray* lcl_fillEmptyMatrix(const ScRange& rRange)
++{
++    SCSIZE nC = static_cast<SCSIZE>(rRange.aEnd.Col()-rRange.aStart.Col()+1);
++    SCSIZE nR = static_cast<SCSIZE>(rRange.aEnd.Row()-rRange.aStart.Row()+1);
++    ScMatrixRef xMat = new ScMatrix(nC, nR);
++    for (SCSIZE i = 0; i < nC; ++i)
++        for (SCSIZE j = 0; j < nR; ++j)
++            xMat->PutEmpty(i, j);
++
++    ScMatrix* pMat2 = xMat;
++    ScMatrixToken aToken(pMat2);
++    auto_ptr<ScTokenArray> pArray(new ScTokenArray);
++    pArray->AddToken(aToken);
+     return pArray.release();
+ }
+ 
+@@ -1598,20 +1743,13 @@ ScExternalRefCache::TokenRef ScExternalRefManager::getSingleRefToken(
+     if (pFmt)
+         pFmt->mbIsSet = false;
+ 
+-    bool bLoading = mpDoc->IsImportingXML();
+-
+     // Check if the given table name and the cell position is cached.
+-    // #i101304# When loading a file, the saved cache (hidden sheet)
+-    // is assumed to contain all data for the loaded formulas.
+-    // No cache entries are created from empty cells in the saved sheet,
+-    // so they have to be created here (bWriteEmpty parameter).
+-    // Otherwise, later interpretation of the loaded formulas would
+-    // load the source document even if the user didn't want to update.
+     sal_uInt32 nFmtIndex = 0;
+     ScExternalRefCache::TokenRef pToken = maRefCache.getCellData(
+-        nFileId, rTabName, rCell.Col(), rCell.Row(), bLoading, bLoading, &nFmtIndex);
++        nFileId, rTabName, rCell.Col(), rCell.Row(), &nFmtIndex);
+     if (pToken)
+     {
++        // Cache hit !
+         if (pFmt)
+         {
+             short nFmtType = mpDoc->GetFormatTable()->GetType(nFmtIndex);
+@@ -1629,11 +1767,8 @@ ScExternalRefCache::TokenRef ScExternalRefManager::getSingleRefToken(
+     ScDocument* pSrcDoc = getSrcDocument(nFileId);
+     if (!pSrcDoc)
+     {
+-        // Source document is not reachable.  Try to get data from the cache 
+-        // once again, but this time treat a non-cached cell as an empty cell
+-        // as long as the table itself is cached.
+-        pToken = maRefCache.getCellData(
+-            nFileId, rTabName, rCell.Col(), rCell.Row(), true, false, &nFmtIndex);
++        // Source document not reachable.  Throw a reference error.
++        pToken.reset(new FormulaErrorToken(errNoRef));
+         return pToken;
+     }
+ 
+@@ -1642,12 +1777,30 @@ ScExternalRefCache::TokenRef ScExternalRefManager::getSingleRefToken(
+     if (!pSrcDoc->GetTable(rTabName, nTab))
+     {
+         // specified table name doesn't exist in the source document.
+-        return ScExternalRefCache::TokenRef();
++        pToken.reset(new FormulaErrorToken(errNoRef));
++        return pToken;
+     }
+ 
+     if (pTab)
+         *pTab = nTab;
+ 
++    SCCOL nDataCol1 = 0, nDataCol2 = MAXCOL;
++    SCROW nDataRow1 = 0, nDataRow2 = MAXROW;
++    pSrcDoc->ShrinkToDataArea(nTab, nDataCol1, nDataRow1, nDataCol2, nDataRow2);
++    if (rCell.Col() < nDataCol1 || nDataCol2 < rCell.Col() || rCell.Row() < nDataRow1 || nDataRow2 < rCell.Row())
++    {
++        // requested cell is outside the data area.  Don't even bother caching
++        // this data, but add it to the cached range to prevent accessing the
++        // source document time and time again.
++        ScExternalRefCache::TableTypeRef pCacheTab = 
++            maRefCache.getCacheTable(nFileId, rTabName, true, NULL);
++        if (pCacheTab)
++            pCacheTab->setCachedCell(rCell.Col(), rCell.Row());
++
++        pToken.reset(new ScEmptyCellToken(false, false));
++        return pToken;
++    }
++
+     pSrcDoc->GetCell(rCell.Col(), rCell.Row(), nTab, pCell);
+     ScExternalRefCache::TokenRef pTok(lcl_convertToToken(pCell));
+ 
+@@ -1670,39 +1823,45 @@ ScExternalRefCache::TokenRef ScExternalRefManager::getSingleRefToken(
+         pTok.reset( new FormulaErrorToken( errNoValue));
+     }
+ 
+-    // Now, insert the token into cache table.
+-    maRefCache.setCellData(nFileId, rTabName, rCell.Row(), rCell.Col(), pTok, nFmtIndex);
++    // Now, insert the token into cache table but don't cache empty cells.
++    if (pTok->GetType() != formula::svEmptyCell)
++        maRefCache.setCellData(nFileId, rTabName, rCell.Col(), rCell.Row(), pTok, nFmtIndex);
++
+     return pTok;
+ }
+ 
+-ScExternalRefCache::TokenArrayRef ScExternalRefManager::getDoubleRefTokens(sal_uInt16 nFileId, const String& rTabName, const ScRange& rRange, const ScAddress* pCurPos)
++ScExternalRefCache::TokenArrayRef ScExternalRefManager::getDoubleRefTokens(
++    sal_uInt16 nFileId, const String& rTabName, const ScRange& rRange, const ScAddress* pCurPos)
+ {
+     if (pCurPos)
+         insertRefCell(nFileId, *pCurPos);
+ 
+     maybeLinkExternalFile(nFileId);
+ 
+-    bool bLoading = mpDoc->IsImportingXML();
+-
+     // Check if the given table name and the cell position is cached.
+-    // #i101304# When loading, put empty cells into cache, see getSingleRefToken.
+-    ScExternalRefCache::TokenArrayRef p = maRefCache.getCellRangeData(nFileId, rTabName, rRange, bLoading, bLoading);
+-    if (p.get())
+-        return p;
++    ScExternalRefCache::TokenArrayRef pArray = 
++        maRefCache.getCellRangeData(nFileId, rTabName, rRange);
++    if (pArray)
++        // Cache hit !
++        return pArray;
+ 
+     ScDocument* pSrcDoc = getSrcDocument(nFileId);
+     if (!pSrcDoc)
+     {
+-        // Source document is not reachable.  Try to get data from the cache 
+-        // once again, but this time treat non-cached cells as empty cells as
+-        // long as the table itself is cached.
+-        return maRefCache.getCellRangeData(nFileId, rTabName, rRange, true, false);
++        // Source document is not reachable.  Throw a reference error.
++        pArray.reset(new ScTokenArray);
++        pArray->AddToken(FormulaErrorToken(errNoRef));
++        return pArray;
+     }
+ 
+     SCTAB nTab1;
+     if (!pSrcDoc->GetTable(rTabName, nTab1))
++    {
+         // specified table name doesn't exist in the source document.
+-        return ScExternalRefCache::TokenArrayRef();
++        pArray.reset(new ScTokenArray);
++        pArray->AddToken(FormulaErrorToken(errNoRef));
++        return pArray;
++    }
+ 
+     ScRange aRange(rRange);
+     SCTAB nTabSpan = aRange.aEnd.Tab() - aRange.aStart.Tab();
+@@ -1726,12 +1885,24 @@ ScExternalRefCache::TokenArrayRef ScExternalRefManager::getDoubleRefTokens(sal_u
+     aRange.aStart.SetTab(nTab1);
+     aRange.aEnd.SetTab(nTab1 + nTabSpan);
+ 
+-    ScExternalRefCache::TokenArrayRef pArray;
+     pArray.reset(lcl_convertToTokenArray(pSrcDoc, aRange, aCacheData));
+ 
+     if (pArray)
+         // Cache these values.
+-        maRefCache.setCellRangeData(nFileId, rRange, aCacheData, pArray);
++        maRefCache.setCellRangeData(nFileId, aRange, aCacheData, pArray);
++    else
++    {    
++        // Array is empty.  Fill it with an empty matrix of the required size.
++        pArray.reset(lcl_fillEmptyMatrix(rRange));
++
++        // Make sure to set this range 'cached', to prevent unnecessarily 
++        // accessing the src document time and time again.
++        ScExternalRefCache::TableTypeRef pCacheTab = 
++            maRefCache.getCacheTable(nFileId, rTabName, true, NULL);
++        if (pCacheTab)
++            pCacheTab->setCachedCellRange(
++                rRange.aStart.Col(), rRange.aStart.Row(), rRange.aEnd.Col(), rRange.aEnd.Row());
++    }
+ 
+     return pArray;
+ }
+@@ -1858,6 +2029,12 @@ ScDocument* ScExternalRefManager::getSrcDocument(sal_uInt16 nFileId)
+ 
+     if (itr != itrEnd)
+     {
++        // document already loaded.
++
++        // TODO: Find out a way to access a document that's already open in
++        // memory and re-use that instance, instead of loading it from the
++        // disk again.
++
+         SfxObjectShell* p = itr->second.maShell;
+         itr->second.maLastAccess = Time();
+         return static_cast<ScDocShell*>(p)->GetDocument();


More information about the ooo-build-commit mailing list