[Libreoffice-commits] core.git: 5 commits - include/comphelper sal/qa sal/rtl sdext/source sw/qa vcl/source

Zdeněk Crhonek zcrhonek at gmail.com
Tue Jan 16 21:37:24 UTC 2018


 include/comphelper/hash.hxx                    |    5 
 sal/qa/rtl/digest/rtl_digest.cxx               |   92 +++++++++++++
 sal/rtl/digest.cxx                             |    2 
 sdext/source/pdfimport/filterdet.cxx           |   17 +-
 sdext/source/pdfimport/pdfparse/pdfentries.cxx |   56 +++-----
 sw/qa/uitest/writer_tests/trackedChanges.py    |  154 ++++++++++++++++++++++
 vcl/source/gdi/pdfwriter_impl.cxx              |   41 +-----
 vcl/source/gdi/pdfwriter_impl.hxx              |    5 
 vcl/source/gdi/pdfwriter_impl2.cxx             |  169 +++++++++----------------
 9 files changed, 361 insertions(+), 180 deletions(-)

New commits:
commit a6f35f870c1d9b418535a6d8f1f591273cb0ffc1
Author: Zdeněk Crhonek <zcrhonek at gmail.com>
Date:   Thu Dec 21 20:48:28 2017 +0100

    UITEST - add test for writer Tracked changes
    
    Change-Id: I53773d52bfb3d171988764f76af0487ba571f13c
    Reviewed-on: https://gerrit.libreoffice.org/46928
    Tested-by: Jenkins <ci at libreoffice.org>
    Reviewed-by: Markus Mohrhard <markus.mohrhard at googlemail.com>

diff --git a/sw/qa/uitest/writer_tests/trackedChanges.py b/sw/qa/uitest/writer_tests/trackedChanges.py
new file mode 100644
index 000000000000..5bc8fecf7b03
--- /dev/null
+++ b/sw/qa/uitest/writer_tests/trackedChanges.py
@@ -0,0 +1,154 @@
+# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# tests for tracked changes ; tdf912270
+
+from uitest.framework import UITestCase
+from uitest.debug import sleep
+import time
+from uitest.uihelper.common import get_state_as_dict, type_text
+
+class trackedchanges(UITestCase):
+
+    def test_tdf91270(self):
+
+        self.ui_test.create_doc_in_start_center("writer")
+
+        xWriterDoc = self.xUITest.getTopFocusWindow()
+        xWriterEdit = xWriterDoc.getChild("writer_edit")
+        type_text(xWriterEdit, "Test")
+
+        self.xUITest.executeCommand(".uno:TrackChanges")
+
+        selection = self.xUITest.executeCommand(".uno:SelectAll")  #select whole text
+        self.xUITest.executeCommand(".uno:Cut")   #cut  text
+
+        self.ui_test.execute_modeless_dialog_through_command(".uno:AcceptTrackedChanges")
+        xTrackDlg = self.xUITest.getTopFocusWindow()
+        xCancBtn = xTrackDlg.getChild("close")
+        xCancBtn.executeAction("CLICK", tuple())
+
+        self.ui_test.close_doc()
+
+    def test_tracked_changes_accept(self):
+
+        self.ui_test.create_doc_in_start_center("writer")
+        document = self.ui_test.get_component()
+        xWriterDoc = self.xUITest.getTopFocusWindow()
+        xWriterEdit = xWriterDoc.getChild("writer_edit")
+
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, "Test LibreOffice")
+        self.xUITest.executeCommand(".uno:ShowTrackedChanges")
+
+        self.ui_test.execute_modeless_dialog_through_command(".uno:AcceptTrackedChanges")
+        xTrackDlg = self.xUITest.getTopFocusWindow()
+        xAccBtn = xTrackDlg.getChild("accept")
+        xAccBtn.executeAction("CLICK", tuple())
+        xCancBtn = xTrackDlg.getChild("close")
+        xCancBtn.executeAction("CLICK", tuple())
+
+        self.assertEqual(document.Text.String[0:16], "Test LibreOffice")
+        self.ui_test.close_doc()
+
+    def test_tracked_changes_acceptall(self):
+
+        self.ui_test.create_doc_in_start_center("writer")
+        document = self.ui_test.get_component()
+        xWriterDoc = self.xUITest.getTopFocusWindow()
+        xWriterEdit = xWriterDoc.getChild("writer_edit")
+
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, "Test LibreOffice")
+        self.xUITest.executeCommand(".uno:ShowTrackedChanges")
+
+        self.ui_test.execute_modeless_dialog_through_command(".uno:AcceptTrackedChanges")
+        xTrackDlg = self.xUITest.getTopFocusWindow()
+
+        xAccBtn = xTrackDlg.getChild("acceptall")
+        xAccBtn.executeAction("CLICK", tuple())
+        xCancBtn = xTrackDlg.getChild("close")
+        xCancBtn.executeAction("CLICK", tuple())
+
+
+        self.assertEqual(document.Text.String[0:16], "Test LibreOffice")
+        self.ui_test.close_doc()
+
+    def test_tracked_changes_reject(self):
+
+        self.ui_test.create_doc_in_start_center("writer")
+        document = self.ui_test.get_component()
+        xWriterDoc = self.xUITest.getTopFocusWindow()
+        xWriterEdit = xWriterDoc.getChild("writer_edit")
+
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, "Test LibreOffice")
+        self.xUITest.executeCommand(".uno:ShowTrackedChanges")
+
+        self.ui_test.execute_modeless_dialog_through_command(".uno:AcceptTrackedChanges")
+        xTrackDlg = self.xUITest.getTopFocusWindow()
+
+        xRejBtn = xTrackDlg.getChild("reject")
+        xRejBtn.executeAction("CLICK", tuple())
+        xCancBtn = xTrackDlg.getChild("close")
+        xCancBtn.executeAction("CLICK", tuple())
+
+        self.assertEqual(document.Text.String[0:1], "")
+        self.ui_test.close_doc()
+
+    def test_tracked_changes_rejectall(self):
+
+        self.ui_test.create_doc_in_start_center("writer")
+        document = self.ui_test.get_component()
+        xWriterDoc = self.xUITest.getTopFocusWindow()
+        xWriterEdit = xWriterDoc.getChild("writer_edit")
+
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, "Test LibreOffice")
+        self.xUITest.executeCommand(".uno:ShowTrackedChanges")
+
+        self.ui_test.execute_modeless_dialog_through_command(".uno:AcceptTrackedChanges")
+        xTrackDlg = self.xUITest.getTopFocusWindow()
+
+        xAccBtn = xTrackDlg.getChild("rejectall")
+        xAccBtn.executeAction("CLICK", tuple())
+        xCancBtn = xTrackDlg.getChild("close")
+        xCancBtn.executeAction("CLICK", tuple())
+
+        self.assertEqual(document.Text.String[0:1], "")
+        self.ui_test.close_doc()
+
+    def test_tracked_changes_zprev_next(self):
+
+        self.ui_test.create_doc_in_start_center("writer")
+        document = self.ui_test.get_component()
+        xWriterDoc = self.xUITest.getTopFocusWindow()
+        xWriterEdit = xWriterDoc.getChild("writer_edit")
+
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, "Test LibreOffice")
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, " Test2")
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, " Test3")
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, " Test4")
+        self.xUITest.executeCommand(".uno:TrackChanges")
+        type_text(xWriterEdit, " Test5")
+        self.xUITest.executeCommand(".uno:PreviousTrackedChange")
+        self.xUITest.executeCommand(".uno:RejectTrackedChange")
+        self.assertEqual(document.Text.String[0:37], "Test LibreOffice Test2 Test3 Test4")
+
+        self.xUITest.executeCommand(".uno:PreviousTrackedChange")
+        self.xUITest.executeCommand(".uno:PreviousTrackedChange")
+        self.xUITest.executeCommand(".uno:AcceptTrackedChange")
+        self.assertEqual(document.Text.String[0:37], "Test LibreOffice Test2 Test3 Test4")
+
+        self.xUITest.executeCommand(".uno:NextTrackedChange")
+        self.xUITest.executeCommand(".uno:RejectTrackedChange")
+        self.assertEqual(document.Text.String[0:30], "Test LibreOffice Test2 Test4")
+
+        self.ui_test.close_doc()
+# vim: set shiftwidth=4 softtabstop=4 expandtab:
commit 18b022cadfa590df9dbefe0433b58838bcc3d2af
Author: Michael Stahl <mstahl at redhat.com>
Date:   Tue Jan 16 21:59:01 2018 +0100

    tdf#114939 sal: fix endMD5() off-by-one
    
    Because of the odd non-standard rtl_digest_rawMD5() API that
    is apparently necessary for MS Office interop, and there not
    being any good reason for bug-compatibility here, just fix the bug.
    
    Change-Id: Iaa0f0af4e24a5ddb9113c1ebd126f9822b5af1f6

diff --git a/sal/qa/rtl/digest/rtl_digest.cxx b/sal/qa/rtl/digest/rtl_digest.cxx
index e022cd500d65..34932a7e71cb 100644
--- a/sal/qa/rtl/digest/rtl_digest.cxx
+++ b/sal/qa/rtl/digest/rtl_digest.cxx
@@ -415,6 +415,97 @@ public:
         }
     }
 
+    void testMD5()
+    {
+        unsigned char const data[] = {
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+            0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
+        };
+        OString const expected[] = {
+            "d41d8cd98f00b204e9800998ecf8427e",
+            "cfcd208495d565ef66e7dff9f98764da",
+            "b4b147bc522828731f1a016bfa72c073",
+            "c6f057b86584942e415435ffb1fa93d4",
+            "4a7d1ed414474e4033ac29ccb8653d9b",
+            "dcddb75469b4b4875094e14561e573d8",
+            "670b14728ad9902aecba32e22fa4f6bd",
+            "29c3eea3f305d6b823f562ac4be35217",
+            "dd4b21e9ef71e1291183a46b913ae6f2",
+            "4c93008615c2d041e33ebac605d14b5b",
+            "f1b708bba17f1ce948dc979f4d7092bc",
+            "645a8aca5a5b84527c57ee2f153f1946",
+            "35b9ab5a36f3234dd26db357fd4a0dc1",
+            "4aad0d9ff11812ebdd5e376fdbef6222",
+            "c47532bbb1e2883c902071591ae1ec9b",
+            "5284047f4ffb4e04824a2fd1d1f0cd62",
+            "1e4a1b03d1b6cd8a174a826f76e009f4",
+            "0e7b9f29a828b6f953b482fc299e536b",
+            "3ea032bf79e8c116b05f4698d5a8e044",
+            "15f47c8a3e5e9685307dd65a653b8dc0",
+            "cc545187d0745132de1e9941db0ef6ce",
+            "0585e303e79acd837c3a3e2a2bec8b18",
+            "b28ccfdee4b9f39ba18b58a4f61a03d1",
+            "d018229b1183c926c10ea688350afec8",
+            "660719b4a7591769583a7c8d20c6dfa4",
+            "1e2432adacf481836265fcc62ee8f3e3",
+            "6e88e2af74c1d9d7d7d652b90d03751e",
+            "780ca685003cec1d617beaa6f346e1be",
+            "7f2e1dcfd6e2a3f5c38f31e640136ff6",
+            "1a3dee46117aeb8010cf365b8653faa8",
+            "1d0064395af3c745f6c3194e92373d7a",
+            "b52582043219f2deb2d3c9cb05d6448a",
+            "cd9e459ea708a948d5c2f5a6ca8838cf",
+            "00de800ecd7a4fb2813986c987e46d51",
+            "15336d4b38561a82bd24c9398b781aed",
+            "5fe699d3c461ab5a795505f59d5adf15",
+            "c5e0eb03cbb4bea95ce3f8f48fca77d5",
+            "355c1410373ef02fff2b03844d72c7d4",
+            "02df97da8207de2b3afa69c151ca8958",
+            "82c66dbf3e73f87ffc9564b2098d6a4f",
+            "b373e3ddc3438d7c10c76f3ad9d4c401",
+            "fac901a4a3dbc4461541731a33a31d15",
+            "f573e011b414bf3f9dd284f7dad29592",
+            "11694570cc5dda099669f2ba3660a70d",
+            "60997cc8aef7fedd9995e6b3ca89ce26",
+            "63c5fcf83c2275fe64e880dd8dfc5cd6",
+            "c7a0a100057ebbfc63ee169562026aea",
+            "42c2dec247919384edece38033458627",
+            "b505acf9fc996902b0c547a2abfc62b2",
+            "2fa7a1321d6b5fa0e04ad46785f574f3",
+            "86d2bfc0bab44eecf21e1432be7b3efc",
+            "7ca318f12a0955a3e637dc5645a2f96e",
+            "3eda02765b8fb8bb9b20c735f4537827",
+            "26dead12262c9a5c115b01e0a3c805b6",
+            "978b0444e93c5f7d714575f28a77dca1",
+            "d7fe636bd28e2ee2ba4d6c5898318699",
+            "ce992c2ad906967c63c3f9ab0c2294a9",
+            "1f3b814e9d417e9fd8750299982feb1f",
+            "1a2f42174eaa78ce6a67d75e98a59cb6",
+            "17c772c45c9a09f6e56b7228ddd161a7",
+            "5b19445b70b493c78f3bc06eb7962315",
+            "e590c24cc612bdedd522dfe23bb29b42",
+            "4d78c699a0167bc0cfce8a5c5a715c0e",
+            "5703db92acb9d45e3975822c9206453f",
+            "10eab6008d5642cf42abd2aa41f847cb",
+        };
+        rtlDigest digest = rtl_digest_createMD5();
+        for (size_t i = 0; i < sizeof(data); ++i)
+        {
+            rtl_digest_updateMD5(digest, &data, i);
+            sal_uInt8 buf[RTL_DIGEST_LENGTH_MD5];
+            rtl_digest_getMD5(digest, &buf[0], sizeof(buf));
+            OString const sResult = createHex(&buf[0], sizeof(buf));
+            CPPUNIT_ASSERT_EQUAL(expected[i], sResult);
+        }
+        rtl_digest_destroyMD5(digest);
+    }
+
     CPPUNIT_TEST_SUITE(DigestTest);
     CPPUNIT_TEST(testCreate);
     CPPUNIT_TEST(testQuery);
@@ -426,6 +517,7 @@ public:
     CPPUNIT_TEST(testUpdate);
     CPPUNIT_TEST(testGet);
     CPPUNIT_TEST(testSHA1SumForBiggerInputData);
+    CPPUNIT_TEST(testMD5);
 
     CPPUNIT_TEST_SUITE_END();
 };
diff --git a/sal/rtl/digest.cxx b/sal/rtl/digest.cxx
index f296fa5fbef8..e3db60461d5e 100644
--- a/sal/rtl/digest.cxx
+++ b/sal/rtl/digest.cxx
@@ -623,7 +623,7 @@ static void endMD5(DigestContextMD5 *ctx)
 
     i += 1;
 
-    if (i >= (DIGEST_LBLOCK_MD5 - 2))
+    if (i > (DIGEST_LBLOCK_MD5 - 2))
     {
         for (; i < DIGEST_LBLOCK_MD5; i++)
         {
commit ea794efe656d3ab2dd4e414aa023fd2983088e20
Author: Michael Stahl <mstahl at redhat.com>
Date:   Tue Jan 16 17:47:52 2018 +0100

    tdf#114939 sdext: don't use StarOffice MD5 in PDF import
    
    Always use real MD5 here, to avoid interop issues.
    
    Change-Id: Id6f43952bace1654ac761aecc77676d933737d1d

diff --git a/sdext/source/pdfimport/filterdet.cxx b/sdext/source/pdfimport/filterdet.cxx
index 99c0f0359fa4..6a0d1b699cb8 100644
--- a/sdext/source/pdfimport/filterdet.cxx
+++ b/sdext/source/pdfimport/filterdet.cxx
@@ -33,6 +33,7 @@
 #include <com/sun/star/io/XSeekable.hpp>
 #include <com/sun/star/io/TempFile.hpp>
 #include <comphelper/fileurl.hxx>
+#include <comphelper/hash.hxx>
 #include <cppuhelper/supportsservice.hxx>
 #include <memory>
 #include <string.h>
@@ -429,16 +430,15 @@ bool checkDocChecksum( const OUString& rInPDFFileURL,
     }
 
     // open file and calculate actual checksum up to index nBytes
-    sal_uInt8 nActualChecksum[ RTL_DIGEST_LENGTH_MD5 ];
-    memset( nActualChecksum, 0, sizeof(nActualChecksum) );
-    rtlDigest aActualDigest = rtl_digest_createMD5();
+    ::std::vector<unsigned char> nChecksum;
+    ::comphelper::Hash aDigest(::comphelper::HashType::MD5);
     oslFileHandle aRead = nullptr;
     oslFileError aErr = osl_File_E_None;
     if( (aErr = osl_openFile(rInPDFFileURL.pData,
                              &aRead,
                              osl_File_OpenFlag_Read )) == osl_File_E_None )
     {
-        sal_Int8 aBuf[4096];
+        sal_uInt8 aBuf[4096];
         sal_uInt32 nCur = 0;
         sal_uInt64 nBytesRead = 0;
         while( nCur < nBytes )
@@ -451,15 +451,16 @@ bool checkDocChecksum( const OUString& rInPDFFileURL,
             }
             nPass = static_cast<sal_uInt32>(nBytesRead);
             nCur += nPass;
-            rtl_digest_updateMD5( aActualDigest, aBuf, nPass );
+            aDigest.update(aBuf, nPass);
         }
-        rtl_digest_getMD5( aActualDigest, nActualChecksum, sizeof(nActualChecksum) );
+
+        nChecksum = aDigest.finalize();
         osl_closeFile( aRead );
     }
-    rtl_digest_destroyMD5( aActualDigest );
 
     // compare the contents
-    return (0 == memcmp( nActualChecksum, nTestChecksum, sizeof( nActualChecksum ) ));
+    return nChecksum.size() == RTL_DIGEST_LENGTH_MD5
+        && (0 == memcmp(nChecksum.data(), nTestChecksum, nChecksum.size()));
 }
 
 uno::Reference< io::XStream > getAdditionalStream( const OUString&                          rInPDFFileURL,
diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
index b973bd40f9eb..d9828d355afa 100644
--- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx
+++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
@@ -20,6 +20,8 @@
 
 #include <pdfparse.hxx>
 
+#include <comphelper/hash.hxx>
+
 #include <rtl/strbuf.hxx>
 #include <rtl/ustring.hxx>
 #include <rtl/ustrbuf.hxx>
@@ -1013,7 +1015,6 @@ struct PDFFileImplData
     sal_uInt32  m_nPEntry;
     OString     m_aDocID;
     rtlCipher   m_aCipher;
-    rtlDigest   m_aDigest;
 
     sal_uInt8   m_aDecryptionKey[ENCRYPTION_KEY_LEN+5]; // maximum handled key length
 
@@ -1024,8 +1025,7 @@ struct PDFFileImplData
         m_nStandardRevision( 0 ),
         m_nKeyLength( 0 ),
         m_nPEntry( 0 ),
-        m_aCipher( nullptr ),
-        m_aDigest( nullptr )
+        m_aCipher( nullptr )
     {
         memset( m_aOEntry, 0, sizeof( m_aOEntry ) );
         memset( m_aUEntry, 0, sizeof( m_aUEntry ) );
@@ -1036,8 +1036,6 @@ struct PDFFileImplData
     {
         if( m_aCipher )
             rtl_cipher_destroyARCFOUR( m_aCipher );
-        if( m_aDigest )
-            rtl_digest_destroyMD5( m_aDigest );
     }
 };
 }
@@ -1073,16 +1071,15 @@ bool PDFFile::decrypt( const sal_uInt8* pInBuffer, sal_uInt32 nLen, sal_uInt8* p
     m_pData->m_aDecryptionKey[i++] = sal_uInt8(nGeneration&0xff);
     m_pData->m_aDecryptionKey[i++] = sal_uInt8((nGeneration>>8)&0xff);
 
-    sal_uInt8 aSum[ENCRYPTION_KEY_LEN];
-    rtl_digest_updateMD5( m_pData->m_aDigest, m_pData->m_aDecryptionKey, i );
-    rtl_digest_getMD5( m_pData->m_aDigest, aSum, sizeof( aSum ) );
+    ::std::vector<unsigned char> const aSum(::comphelper::Hash::calculateHash(
+                m_pData->m_aDecryptionKey, i, ::comphelper::HashType::MD5));
 
     if( i > 16 )
         i = 16;
 
     rtlCipherError aErr = rtl_cipher_initARCFOUR( m_pData->m_aCipher,
                                                   rtl_Cipher_DirectionDecode,
-                                                  aSum, i,
+                                                  aSum.data(), i,
                                                   nullptr, 0 );
     if( aErr == rtl_Cipher_E_None )
         aErr = rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
@@ -1116,32 +1113,32 @@ static sal_uInt32 password_to_key( const OString& rPwd, sal_uInt8* pOutKey, PDFF
     // encrypt pad string
     sal_Char aPadPwd[ENCRYPTION_BUF_LEN];
     pad_or_truncate_to_32( rPwd, aPadPwd );
-    rtl_digest_updateMD5( pData->m_aDigest, aPadPwd, sizeof( aPadPwd ) );
+    ::comphelper::Hash aDigest(::comphelper::HashType::MD5);
+    aDigest.update(reinterpret_cast<unsigned char const*>(aPadPwd), sizeof(aPadPwd));
     if( ! bComputeO )
     {
-        rtl_digest_updateMD5( pData->m_aDigest, pData->m_aOEntry, 32 );
+        aDigest.update(pData->m_aOEntry, 32);
         sal_uInt8 aPEntry[4];
         aPEntry[0] = static_cast<sal_uInt8>(pData->m_nPEntry & 0xff);
         aPEntry[1] = static_cast<sal_uInt8>((pData->m_nPEntry >> 8 ) & 0xff);
         aPEntry[2] = static_cast<sal_uInt8>((pData->m_nPEntry >> 16) & 0xff);
         aPEntry[3] = static_cast<sal_uInt8>((pData->m_nPEntry >> 24) & 0xff);
-        rtl_digest_updateMD5( pData->m_aDigest, aPEntry, sizeof(aPEntry) );
-        rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() );
+        aDigest.update(aPEntry, sizeof(aPEntry));
+        aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength());
     }
-    sal_uInt8 nSum[RTL_DIGEST_LENGTH_MD5];
-    rtl_digest_getMD5( pData->m_aDigest, nSum, sizeof(nSum) );
+    ::std::vector<unsigned char> nSum(aDigest.finalize());
     if( pData->m_nStandardRevision == 3 )
     {
         for( int i = 0; i < 50; i++ )
         {
-            rtl_digest_updateMD5( pData->m_aDigest, nSum, sizeof(nSum) );
-            rtl_digest_getMD5( pData->m_aDigest, nSum, sizeof(nSum) );
+            nSum = ::comphelper::Hash::calculateHash(nSum.data(), nSum.size(),
+                    ::comphelper::HashType::MD5);
         }
     }
     sal_uInt32 nLen = pData->m_nKeyLength;
     if( nLen > RTL_DIGEST_LENGTH_MD5 )
         nLen = RTL_DIGEST_LENGTH_MD5;
-    memcpy( pOutKey, nSum, nLen );
+    memcpy( pOutKey, nSum.data(), nLen );
     return nLen;
 }
 
@@ -1150,13 +1147,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
     // see PDF reference 1.4 Algorithm 3.6
     bool bValid = false;
     sal_uInt8 aKey[ENCRYPTION_KEY_LEN];
-    sal_uInt8 nEncryptedEntry[ENCRYPTION_BUF_LEN];
-    memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) );
     sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, pData, false );
     // save (at this time potential) decryption key for later use
     memcpy( pData->m_aDecryptionKey, aKey, nKeyLen );
     if( pData->m_nStandardRevision == 2 )
     {
+        sal_uInt8 nEncryptedEntry[ENCRYPTION_BUF_LEN];
+        memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) );
         // see PDF reference 1.4 Algorithm 3.4
         // encrypt pad string
         rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
@@ -1169,14 +1166,15 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
     else if( pData->m_nStandardRevision == 3 )
     {
         // see PDF reference 1.4 Algorithm 3.5
-        rtl_digest_updateMD5( pData->m_aDigest, nPadString, sizeof( nPadString ) );
-        rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() );
-        rtl_digest_getMD5( pData->m_aDigest, nEncryptedEntry, sizeof(nEncryptedEntry) );
+        ::comphelper::Hash aDigest(::comphelper::HashType::MD5);
+        aDigest.update(nPadString, sizeof(nPadString));
+        aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength());
+        ::std::vector<unsigned char> nEncryptedEntry(aDigest.finalize());
         rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
                                 aKey, sizeof(aKey), nullptr, 0 );
         rtl_cipher_encodeARCFOUR( pData->m_aCipher,
-                                  nEncryptedEntry, 16,
-                                  nEncryptedEntry, 16 ); // encrypt in place
+                                  nEncryptedEntry.data(), 16,
+                                  nEncryptedEntry.data(), 16 ); // encrypt in place
         for( int i = 1; i <= 19; i++ ) // do it 19 times, start with 1
         {
             sal_uInt8 aTempKey[ENCRYPTION_KEY_LEN];
@@ -1186,10 +1184,10 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
             rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
                                     aTempKey, sizeof(aTempKey), nullptr, 0 );
             rtl_cipher_encodeARCFOUR( pData->m_aCipher,
-                                      nEncryptedEntry, 16,
-                                      nEncryptedEntry, 16 ); // encrypt in place
+                                      nEncryptedEntry.data(), 16,
+                                      nEncryptedEntry.data(), 16 ); // encrypt in place
         }
-        bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 16 ) == 0);
+        bValid = (memcmp( nEncryptedEntry.data(), pData->m_aUEntry, 16 ) == 0);
     }
     return bValid;
 }
@@ -1214,8 +1212,6 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
 
     if( ! m_pData->m_aCipher )
         m_pData->m_aCipher = rtl_cipher_createARCFOUR(rtl_Cipher_ModeStream);
-    if( ! m_pData->m_aDigest )
-        m_pData->m_aDigest = rtl_digest_createMD5();
 
     // first try user password
     bool bValid = check_user_password( rPwd, m_pData.get() );
commit 1929f23143e37cd50fc90425f5bd610827bff745
Author: Michael Stahl <mstahl at redhat.com>
Date:   Tue Jan 16 17:15:04 2018 +0100

    tdf#114939 vcl: don't use StarOffice MD5 in PDF export
    
    Unfortunately rtl_digest_MD5 has the same bug as SHA1.
    
    Always use real MD5 here, to avoid interop issues.
    
    Change-Id: I97173dc6d33a8f096702c29efa31a4544e582ad1

diff --git a/vcl/source/gdi/pdfwriter_impl.cxx b/vcl/source/gdi/pdfwriter_impl.cxx
index 75e11feb27ba..8c8868bf9d2f 100644
--- a/vcl/source/gdi/pdfwriter_impl.cxx
+++ b/vcl/source/gdi/pdfwriter_impl.cxx
@@ -1702,9 +1702,8 @@ void PDFWriterImpl::PDFPage::appendWaveLine( sal_Int32 nWidth, sal_Int32 nY, sal
         m_aContext(rContext),
         m_aFile(m_aContext.URL),
         m_bOpen(false),
-        m_aDocDigest( rtl_digest_createMD5() ),
+        m_DocDigest(::comphelper::HashType::MD5),
         m_aCipher( nullptr ),
-        m_aDigest( nullptr ),
         m_nKeyLength(0),
         m_nRC4KeyLength(0),
         m_bEncryptThisStream( false ),
@@ -1755,7 +1754,6 @@ void PDFWriterImpl::PDFPage::appendWaveLine( sal_Int32 nWidth, sal_Int32 nY, sal
 
     /* prepare the cypher engine, can be done in CTOR, free in DTOR */
     m_aCipher = rtl_cipher_createARCFOUR( rtl_Cipher_ModeStream );
-    m_aDigest = rtl_digest_createMD5();
 
     /* the size of the Codec default maximum */
     /* is this 0x4000 required to be the same as MAX_SIGNATURE_CONTENT_LENGTH or just coincidentally the same at the moment? */
@@ -1818,14 +1816,10 @@ void PDFWriterImpl::PDFPage::appendWaveLine( sal_Int32 nWidth, sal_Int32 nY, sal
 
 PDFWriterImpl::~PDFWriterImpl()
 {
-    if( m_aDocDigest )
-        rtl_digest_destroyMD5( m_aDocDigest );
     m_pReferenceDevice.disposeAndClear();
 
     if( m_aCipher )
         rtl_cipher_destroyARCFOUR( m_aCipher );
-    if( m_aDigest )
-        rtl_digest_destroyMD5( m_aDigest );
 
     rtl_freeMemory( m_pEncryptionBuffer );
 }
@@ -1977,21 +1971,11 @@ void PDFWriterImpl::computeDocumentIdentifier( std::vector< sal_uInt8 >& o_rIden
     aInfoValuesOut = aID.makeStringAndClear();
     o_rCString2 = aCreationMetaDateString.makeStringAndClear();
 
-    rtlDigest aDigest = rtl_digest_createMD5();
-    OSL_ENSURE( aDigest != nullptr, "PDFWriterImpl::computeDocumentIdentifier: cannot obtain a digest object !" );
-    if( aDigest )
-    {
-        rtlDigestError nError = rtl_digest_updateMD5( aDigest, &aGMT, sizeof( aGMT ) );
-        if( nError == rtl_Digest_E_None )
-            nError = rtl_digest_updateMD5( aDigest, aInfoValuesOut.getStr(), aInfoValuesOut.getLength() );
-        if( nError == rtl_Digest_E_None )
-        {
-            o_rIdentifier = std::vector< sal_uInt8 >( 16, 0 );
-            //the binary form of the doc id is needed for encryption stuff
-            rtl_digest_getMD5( aDigest, &o_rIdentifier[0], 16 );
-        }
-        rtl_digest_destroyMD5(aDigest);
-    }
+    ::comphelper::Hash aDigest(::comphelper::HashType::MD5);
+    aDigest.update(reinterpret_cast<unsigned char const*>(&aGMT), sizeof(aGMT));
+    aDigest.update(reinterpret_cast<unsigned char const*>(aInfoValuesOut.getStr()), aInfoValuesOut.getLength());
+    //the binary form of the doc id is needed for encryption stuff
+    o_rIdentifier = aDigest.finalize();
 }
 
 /* i12626 methods */
@@ -2172,8 +2156,7 @@ bool PDFWriterImpl::writeBuffer( const void* pBuffer, sal_uInt64 nBytes )
         }
 
         const void* pWriteBuffer = ( m_bEncryptThisStream && buffOK ) ? m_pEncryptionBuffer  : pBuffer;
-        if( m_aDocDigest )
-            rtl_digest_updateMD5( m_aDocDigest, pWriteBuffer, static_cast<sal_uInt32>(nBytes) );
+        m_DocDigest.update(static_cast<unsigned char const*>(pWriteBuffer), static_cast<sal_uInt32>(nBytes));
 
         if (m_aFile.write(pWriteBuffer, nBytes, nWritten) != osl::File::E_None)
             nWritten = 0;
@@ -5894,13 +5877,9 @@ bool PDFWriterImpl::emitTrailer()
 
     // prepare document checksum
     OStringBuffer aDocChecksum( 2*RTL_DIGEST_LENGTH_MD5+1 );
-    if( m_aDocDigest )
-    {
-        sal_uInt8 nMD5Sum[ RTL_DIGEST_LENGTH_MD5 ];
-        rtl_digest_getMD5( m_aDocDigest, nMD5Sum, sizeof(nMD5Sum) );
-        for(sal_uInt8 i : nMD5Sum)
-            appendHex( i, aDocChecksum );
-    }
+    ::std::vector<unsigned char> const nMD5Sum(m_DocDigest.finalize());
+    for (sal_uInt8 i : nMD5Sum)
+        appendHex( i, aDocChecksum );
     // document id set in setDocInfo method
     // emit trailer
     aLine.setLength( 0 );
diff --git a/vcl/source/gdi/pdfwriter_impl.hxx b/vcl/source/gdi/pdfwriter_impl.hxx
index b2890823fad3..002b2e4cb655 100644
--- a/vcl/source/gdi/pdfwriter_impl.hxx
+++ b/vcl/source/gdi/pdfwriter_impl.hxx
@@ -30,7 +30,6 @@
 #include <com/sun/star/uno/Sequence.h>
 #include <osl/file.hxx>
 #include <rtl/cipher.h>
-#include <rtl/digest.h>
 #include <rtl/strbuf.hxx>
 #include <rtl/ustring.hxx>
 #include <tools/gen.hxx>
@@ -41,6 +40,7 @@
 #include <vcl/pdfwriter.hxx>
 #include <vcl/wall.hxx>
 #include <o3tl/typed_flags_set.hxx>
+#include <comphelper/hash.hxx>
 
 #include <sallayout.hxx>
 #include <outdata.hxx>
@@ -769,7 +769,7 @@ private:
     std::vector< PDFAddStream >             m_aAdditionalStreams;
     std::set< PDFWriter::ErrorCode >        m_aErrors;
 
-    rtlDigest                               m_aDocDigest;
+    ::comphelper::Hash                      m_DocDigest;
 
 /*
 variables for PDF security
@@ -777,7 +777,6 @@ i12626
 */
 /* used to cipher the stream data and for password management */
     rtlCipher                               m_aCipher;
-    rtlDigest                               m_aDigest;
     /* pad string used for password in Standard security handler */
     static const sal_uInt8                  s_nPadString[ENCRYPTED_PWD_SIZE];
 
diff --git a/vcl/source/gdi/pdfwriter_impl2.cxx b/vcl/source/gdi/pdfwriter_impl2.cxx
index 18ac2e32e162..53632c7a4919 100644
--- a/vcl/source/gdi/pdfwriter_impl2.cxx
+++ b/vcl/source/gdi/pdfwriter_impl2.cxx
@@ -34,6 +34,7 @@
 #include <tools/stream.hxx>
 
 #include <comphelper/fileformat.h>
+#include <comphelper/hash.hxx>
 #include <comphelper/processfactory.hxx>
 
 #include <com/sun/star/beans/PropertyValue.hpp>
@@ -1082,23 +1083,23 @@ void PDFWriterImpl::playMetafile( const GDIMetaFile& i_rMtf, vcl::PDFExtOutDevDa
 
 // Encryption methods
 
-/* a crutch to transport an rtlDigest safely though UNO API
+/* a crutch to transport a ::comphelper::Hash safely though UNO API
    this is needed for the PDF export dialog, which otherwise would have to pass
    clear text passwords down till they can be used in PDFWriter. Unfortunately
    the MD5 sum of the password (which is needed to create the PDF encryption key)
-   is not sufficient, since an rtl MD5 digest cannot be created in an arbitrary state
+   is not sufficient, since an MD5 digest cannot be created in an arbitrary state
    which would be needed in PDFWriterImpl::computeEncryptionKey.
 */
 class EncHashTransporter : public cppu::WeakImplHelper < css::beans::XMaterialHolder >
 {
-    rtlDigest                   maUDigest;
+    ::std::unique_ptr<::comphelper::Hash> m_pDigest;
     sal_IntPtr                  maID;
     std::vector< sal_uInt8 >    maOValue;
 
     static std::map< sal_IntPtr, EncHashTransporter* >      sTransporters;
 public:
     EncHashTransporter()
-    : maUDigest( rtl_digest_createMD5() )
+        : m_pDigest(new ::comphelper::Hash(::comphelper::HashType::MD5))
     {
         maID = reinterpret_cast< sal_IntPtr >(this);
         while( sTransporters.find( maID ) != sTransporters.end() ) // paranoia mode
@@ -1109,20 +1110,14 @@ public:
     virtual ~EncHashTransporter() override
     {
         sTransporters.erase( maID );
-        if( maUDigest )
-            rtl_digest_destroyMD5( maUDigest );
         SAL_INFO( "vcl", "EncHashTransporter freed" );
     }
 
-    rtlDigest getUDigest() const { return maUDigest; };
+    ::comphelper::Hash* getUDigest() { return m_pDigest.get(); };
     std::vector< sal_uInt8 >& getOValue() { return maOValue; }
     void invalidate()
     {
-        if( maUDigest )
-        {
-            rtl_digest_destroyMD5( maUDigest );
-            maUDigest = nullptr;
-        }
+        m_pDigest.reset();
     }
 
     // XMaterialHolder
@@ -1180,12 +1175,12 @@ void PDFWriterImpl::checkAndEnableStreamEncryption( sal_Int32 nObject )
         m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>( nObject >> 16 );
         // the other location of m_nEncryptionKey is already set to 0, our fixed generation number
         // do the MD5 hash
-        sal_uInt8 nMD5Sum[ RTL_DIGEST_LENGTH_MD5 ];
+        ::std::vector<unsigned char> const nMD5Sum(::comphelper::Hash::calculateHash(
+            &m_aContext.Encryption.EncryptionKey[0], i+2, ::comphelper::HashType::MD5));
         // the i+2 to take into account the generation number, always zero
-        rtl_digest_MD5( &m_aContext.Encryption.EncryptionKey[0], i+2, nMD5Sum, sizeof(nMD5Sum) );
         // initialize the RC4 with the key
         // key length: see algorithm 3.1, step 4: (N+5) max 16
-        rtl_cipher_initARCFOUR( m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum, m_nRC4KeyLength, nullptr, 0 );
+        rtl_cipher_initARCFOUR( m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum.data(), m_nRC4KeyLength, nullptr, 0 );
     }
 }
 
@@ -1199,12 +1194,12 @@ void PDFWriterImpl::enableStringEncryption( sal_Int32 nObject )
         m_aContext.Encryption.EncryptionKey[i++] = static_cast<sal_uInt8>( nObject >> 16 );
         // the other location of m_nEncryptionKey is already set to 0, our fixed generation number
         // do the MD5 hash
-        sal_uInt8 nMD5Sum[ RTL_DIGEST_LENGTH_MD5 ];
         // the i+2 to take into account the generation number, always zero
-        rtl_digest_MD5( &m_aContext.Encryption.EncryptionKey[0], i+2, nMD5Sum, sizeof(nMD5Sum) );
+        ::std::vector<unsigned char> const nMD5Sum(::comphelper::Hash::calculateHash(
+            &m_aContext.Encryption.EncryptionKey[0], i+2, ::comphelper::HashType::MD5));
         // initialize the RC4 with the key
         // key length: see algorithm 3.1, step 4: (N+5) max 16
-        rtl_cipher_initARCFOUR( m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum, m_nRC4KeyLength, nullptr, 0 );
+        rtl_cipher_initARCFOUR( m_aCipher, rtl_Cipher_DirectionEncode, nMD5Sum.data(), m_nRC4KeyLength, nullptr, 0 );
     }
 }
 
@@ -1233,9 +1228,7 @@ uno::Reference< beans::XMaterialHolder > PDFWriterImpl::initEncryption( const OU
 
         if( computeODictionaryValue( aPadOPW, aPadUPW, pTransporter->getOValue(), nKeyLength ) )
         {
-            rtlDigest aDig = pTransporter->getUDigest();
-            if( rtl_digest_updateMD5( aDig, aPadUPW, ENCRYPTED_PWD_SIZE ) != rtl_Digest_E_None )
-                xResult.clear();
+            pTransporter->getUDigest()->update(aPadUPW, ENCRYPTED_PWD_SIZE);
         }
         else
             xResult.clear();
@@ -1333,16 +1326,15 @@ TODO: in pdf ver 1.5 and 1.6 the step 6 is different, should be implemented. See
 bool PDFWriterImpl::computeEncryptionKey( EncHashTransporter* i_pTransporter, vcl::PDFWriter::PDFEncryptionProperties& io_rProperties, sal_Int32 i_nAccessPermissions )
 {
     bool bSuccess = true;
-    sal_uInt8 nMD5Sum[ RTL_DIGEST_LENGTH_MD5 ];
+    ::std::vector<unsigned char> nMD5Sum;
 
     // transporter contains an MD5 digest with the padded user password already
-    rtlDigest aDigest = i_pTransporter->getUDigest();
-    rtlDigestError nError = rtl_Digest_E_None;
-    if( aDigest )
+    ::comphelper::Hash *const pDigest = i_pTransporter->getUDigest();
+    if (pDigest)
     {
         //step 3
         if( ! io_rProperties.OValue.empty() )
-            nError = rtl_digest_updateMD5( aDigest, &io_rProperties.OValue[0] , sal_Int32(io_rProperties.OValue.size()) );
+            pDigest->update(&io_rProperties.OValue[0], io_rProperties.OValue.size());
         else
             bSuccess = false;
         //Step 4
@@ -1353,32 +1345,17 @@ bool PDFWriterImpl::computeEncryptionKey( EncHashTransporter* i_pTransporter, vc
         nPerm[2] = static_cast<sal_uInt8>( i_nAccessPermissions >> 16 );
         nPerm[3] = static_cast<sal_uInt8>( i_nAccessPermissions >> 24 );
 
-        if( nError == rtl_Digest_E_None )
-            nError = rtl_digest_updateMD5( aDigest, nPerm , sizeof( nPerm ) );
+        pDigest->update(nPerm, sizeof(nPerm));
 
         //step 5, get the document ID, binary form
-        if( nError == rtl_Digest_E_None )
-            nError = rtl_digest_updateMD5( aDigest, &io_rProperties.DocumentIdentifier[0], sal_Int32(io_rProperties.DocumentIdentifier.size()) );
+        pDigest->update(&io_rProperties.DocumentIdentifier[0], io_rProperties.DocumentIdentifier.size());
         //get the digest
-        if( nError == rtl_Digest_E_None )
-        {
-            rtl_digest_getMD5( aDigest, nMD5Sum, sizeof( nMD5Sum ) );
+        nMD5Sum = pDigest->finalize();
 
-            //step 6, only if 128 bit
-            for( sal_Int32 i = 0; i < 50; i++ )
-            {
-                nError = rtl_digest_updateMD5( aDigest, &nMD5Sum, sizeof( nMD5Sum ) );
-                if( nError != rtl_Digest_E_None )
-                {
-                    bSuccess =  false;
-                    break;
-                }
-                rtl_digest_getMD5( aDigest, nMD5Sum, sizeof( nMD5Sum ) );
-            }
-        }
-        else
+        //step 6, only if 128 bit
+        for (sal_Int32 i = 0; i < 50; i++)
         {
-            bSuccess = false;
+            nMD5Sum = ::comphelper::Hash::calculateHash(nMD5Sum.data(), nMD5Sum.size(), ::comphelper::HashType::MD5);
         }
     }
     else
@@ -1413,69 +1390,53 @@ bool PDFWriterImpl::computeODictionaryValue( const sal_uInt8* i_pPaddedOwnerPass
 
     io_rOValue.resize( ENCRYPTED_PWD_SIZE );
 
-    rtlDigest aDigest = rtl_digest_createMD5();
     rtlCipher aCipher = rtl_cipher_createARCFOUR( rtl_Cipher_ModeStream );
-    if( aDigest && aCipher)
+    if (aCipher)
     {
         //step 1 already done, data is in i_pPaddedOwnerPassword
         //step 2
 
-        rtlDigestError nError = rtl_digest_updateMD5( aDigest, i_pPaddedOwnerPassword, ENCRYPTED_PWD_SIZE );
-        if( nError == rtl_Digest_E_None )
+        ::std::vector<unsigned char> nMD5Sum(::comphelper::Hash::calculateHash(
+            i_pPaddedOwnerPassword, ENCRYPTED_PWD_SIZE, ::comphelper::HashType::MD5));
+        //step 3, only if 128 bit
+        if (i_nKeyLength == SECUR_128BIT_KEY)
         {
-            sal_uInt8 nMD5Sum[ RTL_DIGEST_LENGTH_MD5 ];
-
-            rtl_digest_getMD5( aDigest, nMD5Sum, sizeof(nMD5Sum) );
-//step 3, only if 128 bit
-            if( i_nKeyLength == SECUR_128BIT_KEY )
+            sal_Int32 i;
+            for (i = 0; i < 50; i++)
             {
-                sal_Int32 i;
-                for( i = 0; i < 50; i++ )
-                {
-                    nError = rtl_digest_updateMD5( aDigest, nMD5Sum, sizeof( nMD5Sum ) );
-                    if( nError != rtl_Digest_E_None )
-                    {
-                        bSuccess = false;
-                        break;
-                    }
-                    rtl_digest_getMD5( aDigest, nMD5Sum, sizeof( nMD5Sum ) );
-                }
+                nMD5Sum = ::comphelper::Hash::calculateHash(nMD5Sum.data(), nMD5Sum.size(), ::comphelper::HashType::MD5);
             }
-            //Step 4, the key is in nMD5Sum
-            //step 5 already done, data is in i_pPaddedUserPassword
-            //step 6
-            rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
-                                     nMD5Sum, i_nKeyLength , nullptr, 0 );
-            // encrypt the user password using the key set above
-            rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted
-                                      &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data
-            //Step 7, only if 128 bit
-            if( i_nKeyLength == SECUR_128BIT_KEY )
+        }
+        //Step 4, the key is in nMD5Sum
+        //step 5 already done, data is in i_pPaddedUserPassword
+        //step 6
+        rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
+                                 nMD5Sum.data(), i_nKeyLength , nullptr, 0 );
+        // encrypt the user password using the key set above
+        rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted
+                                  &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data
+        //Step 7, only if 128 bit
+        if( i_nKeyLength == SECUR_128BIT_KEY )
+        {
+            sal_uInt32 i, y;
+            sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key
+
+            for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1
             {
-                sal_uInt32 i, y;
-                sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key
+                for( y = 0; y < sizeof( nLocalKey ); y++ )
+                    nLocalKey[y] = static_cast<sal_uInt8>( nMD5Sum[y] ^ i );
 
-                for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1
-                {
-                    for( y = 0; y < sizeof( nLocalKey ); y++ )
-                        nLocalKey[y] = static_cast<sal_uInt8>( nMD5Sum[y] ^ i );
-
-                    rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
-                                            nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ); //destination data area, on init can be NULL
-                    rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted
-                                              &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"
-                    //step 8, store in class data member
-                }
+                rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
+                                        nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ); //destination data area, on init can be NULL
+                rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted
+                                          &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"
+                //step 8, store in class data member
             }
         }
-        else
-            bSuccess = false;
     }
     else
         bSuccess = false;
 
-    if( aDigest )
-        rtl_digest_destroyMD5( aDigest );
     if( aCipher )
         rtl_cipher_destroyARCFOUR( aCipher );
 
@@ -1497,9 +1458,9 @@ bool PDFWriterImpl::computeUDictionaryValue( EncHashTransporter* i_pTransporter,
 
     io_rProperties.UValue.resize( ENCRYPTED_PWD_SIZE );
 
-    rtlDigest aDigest = rtl_digest_createMD5();
+    ::comphelper::Hash aDigest(::comphelper::HashType::MD5);
     rtlCipher aCipher = rtl_cipher_createARCFOUR( rtl_Cipher_ModeStream );
-    if( aDigest && aCipher )
+    if (aCipher)
     {
         //step 1, common to both 3.4 and 3.5
         if( computeEncryptionKey( i_pTransporter, io_rProperties, i_nAccessPermissions ) )
@@ -1513,19 +1474,15 @@ bool PDFWriterImpl::computeUDictionaryValue( EncHashTransporter* i_pTransporter,
             for(sal_uInt32 i = MD5_DIGEST_SIZE; i < sal_uInt32(io_rProperties.UValue.size()); i++)
                 io_rProperties.UValue[i] = 0;
             //steps 2 and 3
-            if (rtl_digest_updateMD5( aDigest, s_nPadString, sizeof( s_nPadString ) ) != rtl_Digest_E_None
-                || rtl_digest_updateMD5( aDigest, &io_rProperties.DocumentIdentifier[0], sal_Int32(io_rProperties.DocumentIdentifier.size()) ) != rtl_Digest_E_None)
-            {
-                bSuccess = false;
-            }
+            aDigest.update(s_nPadString, sizeof(s_nPadString));
+            aDigest.update(&io_rProperties.DocumentIdentifier[0], io_rProperties.DocumentIdentifier.size());
 
-            sal_uInt8 nMD5Sum[ RTL_DIGEST_LENGTH_MD5 ];
-            rtl_digest_getMD5( aDigest, nMD5Sum, sizeof(nMD5Sum) );
+            ::std::vector<unsigned char> const nMD5Sum(aDigest.finalize());
             //Step 4
             rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
                                     &io_rProperties.EncryptionKey[0], SECUR_128BIT_KEY, nullptr, 0 ); //destination data area
-            rtl_cipher_encodeARCFOUR( aCipher, nMD5Sum, sizeof( nMD5Sum ), // the data to be encrypted
-                                      &io_rProperties.UValue[0], sizeof( nMD5Sum ) ); //encrypted data, stored in class data member
+            rtl_cipher_encodeARCFOUR( aCipher, nMD5Sum.data(), nMD5Sum.size(), // the data to be encrypted
+                                      &io_rProperties.UValue[0], SECUR_128BIT_KEY ); //encrypted data, stored in class data member
             //step 5
             sal_uInt32 i, y;
             sal_uInt8 nLocalKey[SECUR_128BIT_KEY];
@@ -1548,8 +1505,6 @@ bool PDFWriterImpl::computeUDictionaryValue( EncHashTransporter* i_pTransporter,
     else
         bSuccess = false;
 
-    if( aDigest )
-        rtl_digest_destroyMD5( aDigest );
     if( aCipher )
         rtl_cipher_destroyARCFOUR( aCipher );
 
commit 6339072a5638995925ea403e71f01a454fdac74e
Author: Michael Stahl <mstahl at redhat.com>
Date:   Tue Jan 16 17:13:22 2018 +0100

    comphelper: put some include guards around stark naked header
    
    Change-Id: I407dc215ea6948518d0d744c73ff2de717063962

diff --git a/include/comphelper/hash.hxx b/include/comphelper/hash.hxx
index d8145604ac6b..df70757f4042 100644
--- a/include/comphelper/hash.hxx
+++ b/include/comphelper/hash.hxx
@@ -7,6 +7,9 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+#ifndef INCLUDED_COMPHELPER_HASH_HXX
+#define INCLUDED_COMPHELPER_HASH_HXX
+
 #include <comphelper/comphelperdllapi.h>
 
 #include <memory>
@@ -45,4 +48,6 @@ public:
 
 }
 
+#endif // INCLUDED_COMPHELPER_HASH_HXX
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */


More information about the Libreoffice-commits mailing list