[Libreoffice-commits] core.git: Branch 'distro/collabora/cp-5.1' - 2 commits - configure.ac download.lst external/icu i18npool/source
Eike Rathke
erack at redhat.com
Mon Apr 24 21:10:44 UTC 2017
configure.ac | 2
dev/null |binary
download.lst | 2
external/icu/UnpackedTarball_icu.mk | 4
external/icu/clang-cl.patch.0 | 26
external/icu/icu4c-changeset-39671.patch.1 | 61
external/icu/khmerbreakengine.patch | 1110 ----------------
i18npool/source/breakiterator/breakiterator_unicode.cxx | 2
8 files changed, 43 insertions(+), 1164 deletions(-)
New commits:
commit e1c627b22b76e2479f47310db85b893cdb3c3a56
Author: Eike Rathke <erack at redhat.com>
Date: Thu Apr 20 22:06:23 2017 +0200
add ICU changeset-39671 fix for CVE-2017-7867 CVE-2017-7868
http://bugs.icu-project.org/trac/changeset/39671
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=213
https://bugzilla.redhat.com/show_bug.cgi?id=1444101
Reviewed-on: https://gerrit.libreoffice.org/36754
Reviewed-by: Eike Rathke <erack at redhat.com>
Tested-by: Jenkins <ci at libreoffice.org>
(cherry picked from commit c7de8233d15ed0c90fef6c49a54d60cf10119f58)
Backported to older MSVC using the UGLY_SIZEOF_MAPTOUCHARS macro instead
of sizeof(UTF8Buf::mapToUChars).
Reviewed-on: https://gerrit.libreoffice.org/36776
Tested-by: Jenkins <ci at libreoffice.org>
Reviewed-by: Michael Stahl <mstahl at redhat.com>
(cherry picked from commit 91f5d002884cae1a60768e9caa9d182f41fb7be6)
Change-Id: I4e776ad4fe63c77057b0c823f8672a2b6703346f
diff --git a/external/icu/UnpackedTarball_icu.mk b/external/icu/UnpackedTarball_icu.mk
index ad28fcea0746..37931250e1a6 100644
--- a/external/icu/UnpackedTarball_icu.mk
+++ b/external/icu/UnpackedTarball_icu.mk
@@ -26,6 +26,7 @@ $(eval $(call gb_UnpackedTarball_add_patches,icu,\
external/icu/rtti.patch.0 \
$(if $(filter-out ANDROID,$(OS)),external/icu/icu4c-icudata-stdlibs.diff) \
$(if $(filter EMSCRIPTEN,$(OS)),external/icu/icu4c-emscripten.patch.1) \
+ external/icu/icu4c-changeset-39671.patch.1 \
))
# vim: set noet sw=4 ts=4:
diff --git a/external/icu/icu4c-changeset-39671.patch.1 b/external/icu/icu4c-changeset-39671.patch.1
new file mode 100644
index 000000000000..d397fed52307
--- /dev/null
+++ b/external/icu/icu4c-changeset-39671.patch.1
@@ -0,0 +1,208 @@
+diff -ur icu.org/source/common/utext.cpp icu/source/common/utext.cpp
+--- icu.org/source/common/utext.cpp 2016-06-15 20:58:17.000000000 +0200
++++ icu/source/common/utext.cpp 2017-04-21 16:38:15.993398034 +0200
+@@ -847,9 +847,15 @@
+ //------------------------------------------------------------------------------
+
+ // Chunk size.
+-// Must be less than 85, because of byte mapping from UChar indexes to native indexes.
+-// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes
+-// to two UChars.)
++// Must be less than 42 (256/6), because of byte mapping from UChar indexes to native indexes.
++// Worst case there are six UTF-8 bytes per UChar.
++// obsolete 6 byte form fd + 5 trails maps to fffd
++// obsolete 5 byte form fc + 4 trails maps to fffd
++// non-shortest 4 byte forms maps to fffd
++// normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit
++// mapToUChars array size must allow for the worst case, 6.
++// This could be brought down to 4, by treating fd and fc as pure illegal,
++// rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros.
+ //
+ enum { UTF8_TEXT_CHUNK_SIZE=32 };
+
+@@ -867,6 +873,15 @@
+ // pair. Doing this is simpler than checking for the edge case.
+ //
+
++// erAck: older MSVC used on libreoffice-5-3 and 5-2 bails out with
++// error C2070: 'unknown': illegal sizeof operand
++// for sizeof(UTF8Buf::mapToUChars)
++// so have an ugly workaround:
++// First define a macro of the original size expression, so a follow-up patch
++// on the original code would fail..
++#define UGLY_MAPTOUCHARS_SIZE (UTF8_TEXT_CHUNK_SIZE*6+6)
++#define UGLY_SIZEOF_MAPTOUCHARS (sizeof(uint8_t)*(UGLY_MAPTOUCHARS_SIZE))
++
+ struct UTF8Buf {
+ int32_t bufNativeStart; // Native index of first char in UChar buf
+ int32_t bufNativeLimit; // Native index following last char in buf.
+@@ -889,7 +904,7 @@
+ // Requires two extra slots,
+ // one for a supplementary starting in the last normal position,
+ // and one for an entry for the buffer limit position.
+- uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to
++ uint8_t mapToUChars[UGLY_MAPTOUCHARS_SIZE]; // Map native offset from bufNativeStart to
+ // correspoding offset in filled part of buf.
+ int32_t align;
+ };
+@@ -1032,6 +1047,7 @@
+ // Requested index is in this buffer.
+ u8b = (UTF8Buf *)ut->p; // the current buffer
+ mapIndex = ix - u8b->toUCharsMapStart;
++ U_ASSERT(mapIndex < (int32_t)UGLY_SIZEOF_MAPTOUCHARS);
+ ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
+ return TRUE;
+
+@@ -1298,6 +1314,10 @@
+ // Can only do this if the incoming index is somewhere in the interior of the string.
+ // If index is at the end, there is no character there to look at.
+ if (ix != ut->b) {
++ // Note: this function will only move the index back if it is on a trail byte
++ // and there is a preceding lead byte and the sequence from the lead
++ // through this trail could be part of a valid UTF-8 sequence
++ // Otherwise the index remains unchanged.
+ U8_SET_CP_START(s8, 0, ix);
+ }
+
+@@ -1311,7 +1331,10 @@
+ UChar *buf = u8b->buf;
+ uint8_t *mapToNative = u8b->mapToNative;
+ uint8_t *mapToUChars = u8b->mapToUChars;
+- int32_t toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1);
++ int32_t toUCharsMapStart = ix - UGLY_SIZEOF_MAPTOUCHARS + 1;
++ // Note that toUCharsMapStart can be negative. Happens when the remaining
++ // text from current position to the beginning is less than the buffer size.
++ // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry.
+ int32_t destIx = UTF8_TEXT_CHUNK_SIZE+2; // Start in the overflow region
+ // at end of buffer to leave room
+ // for a surrogate pair at the
+@@ -1338,6 +1361,7 @@
+ if (c<0x80) {
+ // Special case ASCII range for speed.
+ buf[destIx] = (UChar)c;
++ U_ASSERT(toUCharsMapStart <= srcIx);
+ mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx;
+ mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
+ } else {
+@@ -1367,6 +1391,7 @@
+ do {
+ mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx;
+ } while (sIx >= srcIx);
++ U_ASSERT(toUCharsMapStart <= (srcIx+1));
+
+ // Set native indexing limit to be the current position.
+ // We are processing a non-ascii, non-native-indexing char now;
+@@ -1541,6 +1566,7 @@
+ U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
+ U_ASSERT(index<=ut->chunkNativeLimit);
+ int32_t mapIndex = index - u8b->toUCharsMapStart;
++ U_ASSERT(mapIndex < (int32_t)UGLY_SIZEOF_MAPTOUCHARS);
+ int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
+ U_ASSERT(offset>=0 && offset<=ut->chunkLength);
+ return offset;
+diff -ur icu.org/source/test/intltest/utxttest.cpp icu/source/test/intltest/utxttest.cpp
+--- icu.org/source/test/intltest/utxttest.cpp 2017-04-24 22:57:56.335754344 +0200
++++ icu/source/test/intltest/utxttest.cpp 2017-04-24 23:05:21.475965671 +0200
+@@ -61,6 +61,8 @@
+ if (exec) Ticket10562(); break;
+ case 6: name = "Ticket10983";
+ if (exec) Ticket10983(); break;
++ case 7: name = "Ticket12888";
++ if (exec) Ticket12888(); break;
+ default: name = ""; break;
+ }
+ }
+@@ -1501,3 +1503,63 @@
+
+ utext_close(ut);
+ }
++
++// Ticket 12888: bad handling of illegal utf-8 containing many instances of the archaic, now illegal,
++// six byte utf-8 forms. Original implementation had an assumption that
++// there would be at most three utf-8 bytes per UTF-16 code unit.
++// The five and six byte sequences map to a single replacement character.
++
++void UTextTest::Ticket12888() {
++ const char *badString =
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80";
++
++ UErrorCode status = U_ZERO_ERROR;
++ LocalUTextPointer ut(utext_openUTF8(NULL, badString, -1, &status));
++ TEST_SUCCESS(status);
++ for (;;) {
++ UChar32 c = utext_next32(ut.getAlias());
++ if (c == U_SENTINEL) {
++ break;
++ }
++ }
++ int32_t endIdx = utext_getNativeIndex(ut.getAlias());
++ if (endIdx != (int32_t)strlen(badString)) {
++ errln("%s:%d expected=%d, actual=%d", __FILE__, __LINE__, strlen(badString), endIdx);
++ return;
++ }
++
++ for (int32_t prevIndex = endIdx; prevIndex>0;) {
++ UChar32 c = utext_previous32(ut.getAlias());
++ int32_t currentIndex = utext_getNativeIndex(ut.getAlias());
++ if (c != 0xfffd) {
++ errln("%s:%d (expected, actual, index) = (%d, %d, %d)\n",
++ __FILE__, __LINE__, 0xfffd, c, currentIndex);
++ break;
++ }
++ if (currentIndex != prevIndex - 6) {
++ errln("%s:%d: wrong index. Expected, actual = %d, %d",
++ __FILE__, __LINE__, prevIndex - 6, currentIndex);
++ break;
++ }
++ prevIndex = currentIndex;
++ }
++}
+diff -ur icu.org/source/test/intltest/utxttest.h icu/source/test/intltest/utxttest.h
+--- icu.org/source/test/intltest/utxttest.h 2017-04-24 22:57:56.335754344 +0200
++++ icu/source/test/intltest/utxttest.h 2017-04-24 23:03:52.648731566 +0200
+@@ -1,5 +1,5 @@
+ /********************************************************************
+- * COPYRIGHT:
++ * COPYRIGHT:
+ * Copyright (c) 2005-2014, International Business Machines Corporation and
+ * others. All Rights Reserved.
+ ********************************************************************/
+@@ -35,6 +35,7 @@
+ void Ticket6847();
+ void Ticket10562();
+ void Ticket10983();
++ void Ticket12888();
+
+ private:
+ struct m { // Map between native indices & code points.
+@@ -49,9 +50,9 @@
+ void TestCopyMove(const UnicodeString &us, UText *ut, UBool move,
+ int32_t nativeStart, int32_t nativeLimit, int32_t nativeDest,
+ int32_t u16Start, int32_t u16Limit, int32_t u16Dest);
+- void TestReplace(const UnicodeString &us, // reference UnicodeString in which to do the replace
++ void TestReplace(const UnicodeString &us, // reference UnicodeString in which to do the replace
+ UText *ut, // UnicodeText object under test.
+- int32_t nativeStart, // Range to be replaced, in UText native units.
++ int32_t nativeStart, // Range to be replaced, in UText native units.
+ int32_t nativeLimit,
+ int32_t u16Start, // Range to be replaced, in UTF-16 units
+ int32_t u16Limit, // for use in the reference UnicodeString.
commit d63deac17cb076e66adf1fc4548f0e02e7452aea
Author: Andras Timar <andras.timar at collabora.com>
Date: Mon Apr 24 22:53:31 2017 +0200
Revert to ICU 56, because Windows MSP cannot be generated otherwise
Change-Id: Idd958f8293618924058934188aaa3bd8d189a513
diff --git a/configure.ac b/configure.ac
index 2ebb0ec7194d..6b003c4fc545 100644
--- a/configure.ac
+++ b/configure.ac
@@ -9023,7 +9023,7 @@ SYSTEM_GENBRK=
SYSTEM_GENCCODE=
SYSTEM_GENCMN=
-ICU_MAJOR=57
+ICU_MAJOR=56
ICU_MINOR=1
ICU_RECLASSIFIED_PREPEND_SET_EMPTY="TRUE"
ICU_RECLASSIFIED_CONDITIONAL_JAPANESE_STARTER="TRUE"
diff --git a/download.lst b/download.lst
index ebbba25d9080..53fc12297f5b 100755
--- a/download.lst
+++ b/download.lst
@@ -63,7 +63,7 @@ export HARFBUZZ_TARBALL := harfbuzz-0.9.40.tar.bz2
export HSQLDB_TARBALL := 17410483b5b5f267aa18b7e00b65e6e0-hsqldb_1_8_0.zip
export HUNSPELL_TARBALL := 4967da60b23413604c9e563beacc63b4-hunspell-1.3.3.tar.gz
export HYPHEN_TARBALL := 5ade6ae2a99bc1e9e57031ca88d36dad-hyphen-2.8.8.tar.gz
-export ICU_TARBALL := 976734806026a4ef8bdd17937c8898b9-icu4c-57_1-src.tgz
+export ICU_TARBALL := c4a2d71ff56aec5ebfab2a3f059be99d-icu4c-56_1-src.tgz
export JFREEREPORT_FLOW_ENGINE_TARBALL := ba2930200c9f019c2d93a8c88c651a0f-flow-engine-0.9.4.zip
export JFREEREPORT_FLUTE_TARBALL := d8bd5eed178db6e2b18eeed243f85aa8-flute-1.1.6.zip
export JFREEREPORT_LIBBASE_TARBALL := eeb2c7ddf0d302fba4bfc6e97eac9624-libbase-1.1.6.zip
diff --git a/external/icu/UnpackedTarball_icu.mk b/external/icu/UnpackedTarball_icu.mk
index 86369ef8e43b..ad28fcea0746 100644
--- a/external/icu/UnpackedTarball_icu.mk
+++ b/external/icu/UnpackedTarball_icu.mk
@@ -24,13 +24,8 @@ $(eval $(call gb_UnpackedTarball_add_patches,icu,\
external/icu/icu4c-icu11100.patch.1 \
external/icu/icu4c-scriptrun.patch \
external/icu/rtti.patch.0 \
- external/icu/clang-cl.patch.0 \
$(if $(filter-out ANDROID,$(OS)),external/icu/icu4c-icudata-stdlibs.diff) \
$(if $(filter EMSCRIPTEN,$(OS)),external/icu/icu4c-emscripten.patch.1) \
- external/icu/khmerbreakengine.patch \
- external/icu/icu4c-changeset-39671.patch.1 \
))
-$(eval $(call gb_UnpackedTarball_add_file,icu,source/data/brkitr/khmerdict.dict,external/icu/khmerdict.dict))
-
# vim: set noet sw=4 ts=4:
diff --git a/external/icu/clang-cl.patch.0 b/external/icu/clang-cl.patch.0
deleted file mode 100644
index 4df5d0f56c83..000000000000
--- a/external/icu/clang-cl.patch.0
+++ /dev/null
@@ -1,26 +0,0 @@
---- source/config/mh-cygwin-msvc
-+++ source/config/mh-cygwin-msvc
-@@ -51,8 +51,8 @@
- LDFLAGS+=-nologo
-
- # Commands to compile
--COMPILE.c= $(CC) $(CPPFLAGS) $(DEFS) $(CFLAGS) -c
-+COMPILE.c= true && $(CC) $(CPPFLAGS) $(DEFS) $(CFLAGS) -c
--COMPILE.cc= $(CXX) $(CPPFLAGS) $(DEFS) $(CXXFLAGS) -c
-+COMPILE.cc= true && $(CXX) $(CPPFLAGS) $(DEFS) $(CXXFLAGS) -c
-
- # Commands to link
- LINK.c= LINK.EXE -subsystem:console $(LDFLAGS)
---- source/runConfigureICU
-+++ source/runConfigureICU
-@@ -259,8 +259,8 @@
- Cygwin/MSVC)
- THE_OS="Windows with Cygwin"
- THE_COMP="Microsoft Visual C++"
-- CC=cl; export CC
-- CXX=cl; export CXX
-+ CC=${CC-cl}; export CC
-+ CXX=${CXX-cl}; export CXX
- RELEASE_CFLAGS='-Gy -MD'
- RELEASE_CXXFLAGS='-Gy -MD'
- DEBUG_CFLAGS='-Zi -MDd'
diff --git a/external/icu/icu4c-changeset-39671.patch.1 b/external/icu/icu4c-changeset-39671.patch.1
deleted file mode 100644
index b8ac1385364e..000000000000
--- a/external/icu/icu4c-changeset-39671.patch.1
+++ /dev/null
@@ -1,189 +0,0 @@
-diff -ur icu.org/source/common/utext.cpp icu/source/common/utext.cpp
---- icu.org/source/common/utext.cpp 2016-06-15 20:58:17.000000000 +0200
-+++ icu/source/common/utext.cpp 2017-04-21 16:38:15.993398034 +0200
-@@ -847,9 +847,15 @@
- //------------------------------------------------------------------------------
-
- // Chunk size.
--// Must be less than 85, because of byte mapping from UChar indexes to native indexes.
--// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes
--// to two UChars.)
-+// Must be less than 42 (256/6), because of byte mapping from UChar indexes to native indexes.
-+// Worst case there are six UTF-8 bytes per UChar.
-+// obsolete 6 byte form fd + 5 trails maps to fffd
-+// obsolete 5 byte form fc + 4 trails maps to fffd
-+// non-shortest 4 byte forms maps to fffd
-+// normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit
-+// mapToUChars array size must allow for the worst case, 6.
-+// This could be brought down to 4, by treating fd and fc as pure illegal,
-+// rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros.
- //
- enum { UTF8_TEXT_CHUNK_SIZE=32 };
-
-@@ -867,6 +873,15 @@
- // pair. Doing this is simpler than checking for the edge case.
- //
-
-+// erAck: older MSVC used on libreoffice-5-3 and 5-2 bails out with
-+// error C2070: 'unknown': illegal sizeof operand
-+// for sizeof(UTF8Buf::mapToUChars)
-+// so have an ugly workaround:
-+// First define a macro of the original size expression, so a follow-up patch
-+// on the original code would fail..
-+#define UGLY_MAPTOUCHARS_SIZE (UTF8_TEXT_CHUNK_SIZE*6+6)
-+#define UGLY_SIZEOF_MAPTOUCHARS (sizeof(uint8_t)*(UGLY_MAPTOUCHARS_SIZE))
-+
- struct UTF8Buf {
- int32_t bufNativeStart; // Native index of first char in UChar buf
- int32_t bufNativeLimit; // Native index following last char in buf.
-@@ -889,7 +904,7 @@
- // Requires two extra slots,
- // one for a supplementary starting in the last normal position,
- // and one for an entry for the buffer limit position.
-- uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to
-+ uint8_t mapToUChars[UGLY_MAPTOUCHARS_SIZE]; // Map native offset from bufNativeStart to
- // correspoding offset in filled part of buf.
- int32_t align;
- };
-@@ -1032,6 +1047,7 @@
- // Requested index is in this buffer.
- u8b = (UTF8Buf *)ut->p; // the current buffer
- mapIndex = ix - u8b->toUCharsMapStart;
-+ U_ASSERT(mapIndex < (int32_t)UGLY_SIZEOF_MAPTOUCHARS);
- ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
- return TRUE;
-
-@@ -1298,6 +1314,10 @@
- // Can only do this if the incoming index is somewhere in the interior of the string.
- // If index is at the end, there is no character there to look at.
- if (ix != ut->b) {
-+ // Note: this function will only move the index back if it is on a trail byte
-+ // and there is a preceding lead byte and the sequence from the lead
-+ // through this trail could be part of a valid UTF-8 sequence
-+ // Otherwise the index remains unchanged.
- U8_SET_CP_START(s8, 0, ix);
- }
-
-@@ -1311,7 +1331,10 @@
- UChar *buf = u8b->buf;
- uint8_t *mapToNative = u8b->mapToNative;
- uint8_t *mapToUChars = u8b->mapToUChars;
-- int32_t toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1);
-+ int32_t toUCharsMapStart = ix - UGLY_SIZEOF_MAPTOUCHARS + 1;
-+ // Note that toUCharsMapStart can be negative. Happens when the remaining
-+ // text from current position to the beginning is less than the buffer size.
-+ // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry.
- int32_t destIx = UTF8_TEXT_CHUNK_SIZE+2; // Start in the overflow region
- // at end of buffer to leave room
- // for a surrogate pair at the
-@@ -1338,6 +1361,7 @@
- if (c<0x80) {
- // Special case ASCII range for speed.
- buf[destIx] = (UChar)c;
-+ U_ASSERT(toUCharsMapStart <= srcIx);
- mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx;
- mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
- } else {
-@@ -1367,6 +1391,7 @@
- do {
- mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx;
- } while (sIx >= srcIx);
-+ U_ASSERT(toUCharsMapStart <= (srcIx+1));
-
- // Set native indexing limit to be the current position.
- // We are processing a non-ascii, non-native-indexing char now;
-@@ -1541,6 +1566,7 @@
- U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
- U_ASSERT(index<=ut->chunkNativeLimit);
- int32_t mapIndex = index - u8b->toUCharsMapStart;
-+ U_ASSERT(mapIndex < (int32_t)UGLY_SIZEOF_MAPTOUCHARS);
- int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
- U_ASSERT(offset>=0 && offset<=ut->chunkLength);
- return offset;
-diff -ur icu.org/source/test/intltest/utxttest.cpp icu/source/test/intltest/utxttest.cpp
---- icu.org/source/test/intltest/utxttest.cpp 2016-06-15 20:58:17.000000000 +0200
-+++ icu/source/test/intltest/utxttest.cpp 2017-04-21 16:14:57.383814739 +0200
-@@ -67,6 +67,8 @@
- if (exec) Ticket10983(); break;
- case 7: name = "Ticket12130";
- if (exec) Ticket12130(); break;
-+ case 8: name = "Ticket12888";
-+ if (exec) Ticket12888(); break;
- default: name = ""; break;
- }
- }
-@@ -1583,3 +1585,63 @@
- }
- utext_close(&ut);
- }
-+
-+// Ticket 12888: bad handling of illegal utf-8 containing many instances of the archaic, now illegal,
-+// six byte utf-8 forms. Original implementation had an assumption that
-+// there would be at most three utf-8 bytes per UTF-16 code unit.
-+// The five and six byte sequences map to a single replacement character.
-+
-+void UTextTest::Ticket12888() {
-+ const char *badString =
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
-+ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80";
-+
-+ UErrorCode status = U_ZERO_ERROR;
-+ LocalUTextPointer ut(utext_openUTF8(NULL, badString, -1, &status));
-+ TEST_SUCCESS(status);
-+ for (;;) {
-+ UChar32 c = utext_next32(ut.getAlias());
-+ if (c == U_SENTINEL) {
-+ break;
-+ }
-+ }
-+ int32_t endIdx = utext_getNativeIndex(ut.getAlias());
-+ if (endIdx != (int32_t)strlen(badString)) {
-+ errln("%s:%d expected=%d, actual=%d", __FILE__, __LINE__, strlen(badString), endIdx);
-+ return;
-+ }
-+
-+ for (int32_t prevIndex = endIdx; prevIndex>0;) {
-+ UChar32 c = utext_previous32(ut.getAlias());
-+ int32_t currentIndex = utext_getNativeIndex(ut.getAlias());
-+ if (c != 0xfffd) {
-+ errln("%s:%d (expected, actual, index) = (%d, %d, %d)\n",
-+ __FILE__, __LINE__, 0xfffd, c, currentIndex);
-+ break;
-+ }
-+ if (currentIndex != prevIndex - 6) {
-+ errln("%s:%d: wrong index. Expected, actual = %d, %d",
-+ __FILE__, __LINE__, prevIndex - 6, currentIndex);
-+ break;
-+ }
-+ prevIndex = currentIndex;
-+ }
-+}
-diff -ur icu.org/source/test/intltest/utxttest.h icu/source/test/intltest/utxttest.h
---- icu.org/source/test/intltest/utxttest.h 2016-06-15 20:58:17.000000000 +0200
-+++ icu/source/test/intltest/utxttest.h 2017-04-21 16:14:57.383814739 +0200
-@@ -38,6 +38,7 @@
- void Ticket10562();
- void Ticket10983();
- void Ticket12130();
-+ void Ticket12888();
-
- private:
- struct m { // Map between native indices & code points.
diff --git a/external/icu/khmerbreakengine.patch b/external/icu/khmerbreakengine.patch
deleted file mode 100644
index 8f81f315da3e..000000000000
--- a/external/icu/khmerbreakengine.patch
+++ /dev/null
@@ -1,1110 +0,0 @@
-diff --git a/source/common/dictbe.cpp b/source/common/dictbe.cpp
-index f1c874d..3ad1b3f 100644
---- misc/icu/source/common/dictbe.cpp
-+++ build/icu/source/common/dictbe.cpp
-@@ -27,8 +27,17 @@ U_NAMESPACE_BEGIN
- ******************************************************************
- */
-
--DictionaryBreakEngine::DictionaryBreakEngine(uint32_t breakTypes) {
-+DictionaryBreakEngine::DictionaryBreakEngine(uint32_t breakTypes) :
-+ clusterLimit(3)
-+{
-+ UErrorCode status = U_ZERO_ERROR;
- fTypes = breakTypes;
-+ fViramaSet.applyPattern(UNICODE_STRING_SIMPLE("[[:ccc=VR:]]"), status);
-+
-+ // note Skip Sets contain fIgnoreSet characters too.
-+ fSkipStartSet.applyPattern(UNICODE_STRING_SIMPLE("[[:lb=OP:][:lb=QU:]\\u200C\\u200D\\u2060]"), status);
-+ fSkipEndSet.applyPattern(UNICODE_STRING_SIMPLE("[[:lb=CP:][:lb=QU:][:lb=EX:][:lb=CL:]\\u200C\\u200D\\u2060]"), status);
-+ fNBeforeSet.applyPattern(UNICODE_STRING_SIMPLE("[[:lb=CR:][:lb=LF:][:lb=NL:][:lb=SP:][:lb=ZW:][:lb=IS:][:lb=BA:][:lb=NS:]]"), status);
- }
-
- DictionaryBreakEngine::~DictionaryBreakEngine() {
-@@ -90,7 +99,7 @@ DictionaryBreakEngine::findBreaks( UText *text,
- result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks);
- utext_setNativeIndex(text, current);
- }
--
-+
- return result;
- }
-
-@@ -101,6 +110,169 @@ DictionaryBreakEngine::setCharacters( const UnicodeSet &set ) {
- fSet.compact();
- }
-
-+bool
-+DictionaryBreakEngine::scanBeforeStart(UText *text, int32_t& start, bool &doBreak) const {
-+ UErrorCode status = U_ZERO_ERROR;
-+ UText* ut = utext_clone(NULL, text, false, true, &status);
-+ utext_setNativeIndex(ut, start);
-+ UChar32 c = utext_current32(ut);
-+ bool res = false;
-+ doBreak = true;
-+ while (start >= 0) {
-+ if (!fSkipStartSet.contains(c)) {
-+ res = (c == ZWSP);
-+ break;
-+ }
-+ --start;
-+ c = utext_previous32(ut);
-+ doBreak = false;
-+ }
-+ utext_close(ut);
-+ return res;
-+}
-+
-+bool
-+DictionaryBreakEngine::scanAfterEnd(UText *text, int32_t textEnd, int32_t& end, bool &doBreak) const {
-+ UErrorCode status = U_ZERO_ERROR;
-+ UText* ut = utext_clone(NULL, text, false, true, &status);
-+ utext_setNativeIndex(ut, end);
-+ UChar32 c = utext_current32(ut);
-+ bool res = false;
-+ doBreak = !fNBeforeSet.contains(c);
-+ while (end < textEnd) {
-+ if (!fSkipEndSet.contains(c)) {
-+ res = (c == ZWSP);
-+ break;
-+ }
-+ ++end;
-+ c = utext_next32(ut);
-+ doBreak = false;
-+ }
-+ utext_close(ut);
-+ return res;
-+}
-+
-+void
-+DictionaryBreakEngine::scanBackClusters(UText *text, int32_t textStart, int32_t& start) const {
-+ UChar32 c = 0;
-+ start = utext_getNativeIndex(text);
-+ while (start > textStart) {
-+ c = utext_previous32(text);
-+ --start;
-+ if (!fSkipEndSet.contains(c))
-+ break;
-+ }
-+ for (int i = 0; i < clusterLimit; ++i) { // scan backwards clusterLimit clusters
-+ while (start > textStart) {
-+ while (fIgnoreSet.contains(c))
-+ c = utext_previous32(text);
-+ if (!fMarkSet.contains(c)) {
-+ if (fBaseSet.contains(c)) {
-+ c = utext_previous32(text);
-+ if (!fViramaSet.contains(c)) { // Virama (e.g. coeng) preceding base. Treat sequence as a mark
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ break;
-+ } else {
-+ --start;
-+ }
-+ } else {
-+ break;
-+ }
-+ }
-+ c = utext_previous32(text);
-+ --start;
-+ }
-+ if (!fBaseSet.contains(c) || start < textStart) { // not a cluster start so finish
-+ break;
-+ }
-+ c = utext_previous32(text);
-+ --start; // go round again
-+ } // ignore hitting previous inhibitor since scanning for it should have found us!
-+ ++start; // counteract --before
-+}
-+
-+void
-+DictionaryBreakEngine::scanFwdClusters(UText *text, int32_t textEnd, int32_t& end) const {
-+ UChar32 c = utext_current32(text);
-+ end = utext_getNativeIndex(text);
-+ while (end < textEnd) {
-+ if (!fSkipStartSet.contains(c))
-+ break;
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ ++end;
-+ }
-+ for (int i = 0; i < clusterLimit; ++i) { // scan forwards clusterLimit clusters
-+ while (fIgnoreSet.contains(c)) {
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ }
-+ if (fBaseSet.contains(c)) {
-+ while (end < textEnd) {
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ ++end;
-+ if (!fMarkSet.contains(c))
-+ break;
-+ else if (fViramaSet.contains(c)) { // handle coeng + base as mark
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ ++end;
-+ if (!fBaseSet.contains(c))
-+ break;
-+ }
-+ }
-+ } else {
-+ --end; // bad char so break after char before it
-+ break;
-+ }
-+ }
-+}
-+
-+bool
-+DictionaryBreakEngine::scanWJ(UText *text, int32_t &start, int32_t end, int32_t &before, int32_t &after) const {
-+ UErrorCode status = U_ZERO_ERROR;
-+ UText* ut = utext_clone(NULL, text, false, true, &status);
-+ int32_t nat = start;
-+ utext_setNativeIndex(ut, nat);
-+ bool foundFirst = true;
-+ int32_t curr = start;
-+ while (nat < end) {
-+ UChar32 c = utext_current32(ut);
-+ if (c == ZWSP || c == WJ) {
-+ curr = nat + 1;
-+ if (foundFirst) // only scan backwards for first inhibitor
-+ scanBackClusters(ut, start, before);
-+ foundFirst = false; // don't scan backwards if we go around again. Also marks found something
-+
-+ utext_next32(ut);
-+ scanFwdClusters(ut, end, after);
-+ nat = after + 1;
-+
-+ if (c == ZWSP || c == WJ) { // did we hit another one?
-+ continue;
-+ } else {
-+ break;
-+ }
-+ }
-+
-+ ++nat; // keep hunting
-+ utext_next32(ut);
-+ }
-+
-+ utext_close(ut);
-+
-+ if (nat >= end && foundFirst) {
-+ start = before = after = nat;
-+ return false; // failed to find anything
-+ }
-+ else {
-+ start = curr;
-+ }
-+ return true; // yup hit one
-+}
-+
- /*
- ******************************************************************
- * PossibleWord
-@@ -128,35 +302,35 @@ private:
- public:
- PossibleWord() : count(0), prefix(0), offset(-1), mark(0), current(0) {};
- ~PossibleWord() {};
--
-+
- // Fill the list of candidates if needed, select the longest, and return the number found
-- int32_t candidates( UText *text, DictionaryMatcher *dict, int32_t rangeEnd );
--
-+ int32_t candidates( UText *text, DictionaryMatcher *dict, int32_t rangeEnd, UnicodeSet const *ignoreSet = NULL, int32_t minLength = 0 );
-+
- // Select the currently marked candidate, point after it in the text, and invalidate self
- int32_t acceptMarked( UText *text );
--
-+
- // Back up from the current candidate to the next shorter one; return TRUE if that exists
- // and point the text after it
- UBool backUp( UText *text );
--
-+
- // Return the longest prefix this candidate location shares with a dictionary word
- // Return value is in code points.
- int32_t longestPrefix() { return prefix; };
--
-+
- // Mark the current candidate as the one we like
- void markCurrent() { mark = current; };
--
-+
- // Get length in code points of the marked word.
- int32_t markedCPLength() { return cpLengths[mark]; };
- };
-
-
--int32_t PossibleWord::candidates( UText *text, DictionaryMatcher *dict, int32_t rangeEnd ) {
-+int32_t PossibleWord::candidates( UText *text, DictionaryMatcher *dict, int32_t rangeEnd, UnicodeSet const *ignoreSet, int32_t minLength) {
- // TODO: If getIndex is too slow, use offset < 0 and add discardAll()
- int32_t start = (int32_t)utext_getNativeIndex(text);
- if (start != offset) {
- offset = start;
-- count = dict->matches(text, rangeEnd-start, UPRV_LENGTHOF(cuLengths), cuLengths, cpLengths, NULL, &prefix);
-+ count = dict->matches(text, rangeEnd-start, UPRV_LENGTHOF(cuLengths), cuLengths, cpLengths, NULL, &prefix, ignoreSet, minLength);
- // Dictionary leaves text after longest prefix, not longest word. Back up.
- if (count <= 0) {
- utext_setNativeIndex(text, start);
-@@ -828,51 +1002,28 @@ foundBest:
- * KhmerBreakEngine
- */
-
--// How many words in a row are "good enough"?
--static const int32_t KHMER_LOOKAHEAD = 3;
--
--// Will not combine a non-word with a preceding dictionary word longer than this
--static const int32_t KHMER_ROOT_COMBINE_THRESHOLD = 3;
--
--// Will not combine a non-word that shares at least this much prefix with a
--// dictionary word, with a preceding word
--static const int32_t KHMER_PREFIX_COMBINE_THRESHOLD = 3;
--
--// Minimum word size
--static const int32_t KHMER_MIN_WORD = 2;
--
--// Minimum number of characters for two words
--static const int32_t KHMER_MIN_WORD_SPAN = KHMER_MIN_WORD * 2;
--
- KhmerBreakEngine::KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status)
- : DictionaryBreakEngine((1 << UBRK_WORD) | (1 << UBRK_LINE)),
- fDictionary(adoptDictionary)
- {
-- fKhmerWordSet.applyPattern(UNICODE_STRING_SIMPLE("[[:Khmr:]&[:LineBreak=SA:]]"), status);
-+
-+ clusterLimit = 3;
-+
-+ fKhmerWordSet.applyPattern(UNICODE_STRING_SIMPLE("[[:Khmr:]\\u2060\\u200C\\u200D]"), status);
- if (U_SUCCESS(status)) {
- setCharacters(fKhmerWordSet);
- }
- fMarkSet.applyPattern(UNICODE_STRING_SIMPLE("[[:Khmr:]&[:LineBreak=SA:]&[:M:]]"), status);
-- fMarkSet.add(0x0020);
-- fEndWordSet = fKhmerWordSet;
-- fBeginWordSet.add(0x1780, 0x17B3);
-- //fBeginWordSet.add(0x17A3, 0x17A4); // deprecated vowels
-- //fEndWordSet.remove(0x17A5, 0x17A9); // Khmer independent vowels that can't end a word
-- //fEndWordSet.remove(0x17B2); // Khmer independent vowel that can't end a word
-- fEndWordSet.remove(0x17D2); // KHMER SIGN COENG that combines some following characters
-- //fEndWordSet.remove(0x17B6, 0x17C5); // Remove dependent vowels
--// fEndWordSet.remove(0x0E31); // MAI HAN-AKAT
--// fEndWordSet.remove(0x0E40, 0x0E44); // SARA E through SARA AI MAIMALAI
--// fBeginWordSet.add(0x0E01, 0x0E2E); // KO KAI through HO NOKHUK
--// fBeginWordSet.add(0x0E40, 0x0E44); // SARA E through SARA AI MAIMALAI
--// fSuffixSet.add(THAI_PAIYANNOI);
--// fSuffixSet.add(THAI_MAIYAMOK);
-+ fIgnoreSet.add(0x2060); // WJ
-+ fIgnoreSet.add(0x200C, 0x200D); // ZWJ, ZWNJ
-+ fBaseSet.applyPattern(UNICODE_STRING_SIMPLE("[[:Khmr:]&[:lb=SA:]&[:^M:]]"), status);
-+ fPuncSet.applyPattern(UNICODE_STRING_SIMPLE("[\\u17D4\\u17D5\\u17D6\\u17D7\\u17D9:]"), status);
-
- // Compact for caching.
- fMarkSet.compact();
-- fEndWordSet.compact();
-- fBeginWordSet.compact();
--// fSuffixSet.compact();
-+ fIgnoreSet.compact();
-+ fBaseSet.compact();
-+ fPuncSet.compact();
- }
-
- KhmerBreakEngine::~KhmerBreakEngine() {
-@@ -884,180 +1036,204 @@ KhmerBreakEngine::divideUpDictionaryRange( UText *text,
- int32_t rangeStart,
- int32_t rangeEnd,
- UStack &foundBreaks ) const {
-- if ((rangeEnd - rangeStart) < KHMER_MIN_WORD_SPAN) {
-- return 0; // Not enough characters for two words
-+ uint32_t wordsFound = foundBreaks.size();
-+ UErrorCode status = U_ZERO_ERROR;
-+ int32_t before = 0;
-+ int32_t after = 0;
-+ int32_t finalBefore = 0;
-+ int32_t initAfter = 0;
-+ int32_t scanStart = rangeStart;
-+ int32_t scanEnd = rangeEnd;
-+
-+ bool startZwsp = false;
-+ bool breakStart = false;
-+ bool breakEnd = false;
-+
-+ if (rangeStart > 0) {
-+ --scanStart;
-+ startZwsp = scanBeforeStart(text, scanStart, breakStart);
-+ }
-+ utext_setNativeIndex(text, rangeStart);
-+ scanFwdClusters(text, rangeEnd, initAfter);
-+ bool endZwsp = scanAfterEnd(text, utext_nativeLength(text), scanEnd, breakEnd);
-+ utext_setNativeIndex(text, rangeEnd - 1);
-+ scanBackClusters(text, rangeStart, finalBefore);
-+ if (finalBefore < initAfter) { // the whole run is tented so no breaks
-+ if (breakStart || fTypes < UBRK_LINE)
-+ foundBreaks.push(rangeStart, status);
-+ if (breakEnd || fTypes < UBRK_LINE)
-+ foundBreaks.push(rangeEnd, status);
-+ return foundBreaks.size() - wordsFound;
- }
-
-- uint32_t wordsFound = 0;
-- int32_t cpWordLength = 0;
-- int32_t cuWordLength = 0;
-- int32_t current;
-- UErrorCode status = U_ZERO_ERROR;
-- PossibleWord words[KHMER_LOOKAHEAD];
-+ scanStart = rangeStart;
-+ scanWJ(text, scanStart, rangeEnd, before, after);
-+ if (startZwsp || initAfter >= before) {
-+ after = initAfter;
-+ before = 0;
-+ }
-+ if (!endZwsp && after > finalBefore && after < rangeEnd)
-+ endZwsp = true;
-+ if (endZwsp && before > finalBefore)
-+ before = finalBefore;
-
- utext_setNativeIndex(text, rangeStart);
-+ int32_t numCodePts = rangeEnd - rangeStart;
-+ // bestSnlp[i] is the snlp of the best segmentation of the first i
-+ // code points in the range to be matched.
-+ UVector32 bestSnlp(numCodePts + 1, status);
-+ bestSnlp.addElement(0, status);
-+ for(int32_t i = 1; i <= numCodePts; i++) {
-+ bestSnlp.addElement(kuint32max, status);
-+ }
-
-- while (U_SUCCESS(status) && (current = (int32_t)utext_getNativeIndex(text)) < rangeEnd) {
-- cuWordLength = 0;
-- cpWordLength = 0;
-+ // prev[i] is the index of the last code point in the previous word in
-+ // the best segmentation of the first i characters. Note negative implies
-+ // that the code point is part of an unknown word.
-+ UVector32 prev(numCodePts + 1, status);
-+ for(int32_t i = 0; i <= numCodePts; i++) {
-+ prev.addElement(kuint32max, status);
-+ }
-
-- // Look for candidate words at the current position
-- int32_t candidates = words[wordsFound%KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd);
-+ const int32_t maxWordSize = 20;
-+ UVector32 values(maxWordSize, status);
-+ values.setSize(maxWordSize);
-+ UVector32 lengths(maxWordSize, status);
-+ lengths.setSize(maxWordSize);
-
-- // If we found exactly one, use that
-- if (candidates == 1) {
-- cuWordLength = words[wordsFound % KHMER_LOOKAHEAD].acceptMarked(text);
-- cpWordLength = words[wordsFound % KHMER_LOOKAHEAD].markedCPLength();
-- wordsFound += 1;
-- }
-+ // Dynamic programming to find the best segmentation.
-
-- // If there was more than one, see which one can take us forward the most words
-- else if (candidates > 1) {
-- // If we're already at the end of the range, we're done
-- if ((int32_t)utext_getNativeIndex(text) >= rangeEnd) {
-- goto foundBest;
-- }
-- do {
-- int32_t wordsMatched = 1;
-- if (words[(wordsFound + 1) % KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd) > 0) {
-- if (wordsMatched < 2) {
-- // Followed by another dictionary word; mark first word as a good candidate
-- words[wordsFound % KHMER_LOOKAHEAD].markCurrent();
-- wordsMatched = 2;
-- }
-+ // In outer loop, i is the code point index,
-+ // ix is the corresponding string (code unit) index.
-+ // They differ when the string contains supplementary characters.
-+ int32_t ix = rangeStart;
-+ for (int32_t i = 0; i < numCodePts; ++i, utext_setNativeIndex(text, ++ix)) {
-+ if ((uint32_t)bestSnlp.elementAti(i) == kuint32max) {
-+ continue;
-+ }
-
-- // If we're already at the end of the range, we're done
-- if ((int32_t)utext_getNativeIndex(text) >= rangeEnd) {
-- goto foundBest;
-- }
-+ int32_t count;
-+ count = fDictionary->matches(text, numCodePts - i, maxWordSize,
-+ NULL, lengths.getBuffer(), values.getBuffer(), NULL, &fIgnoreSet, 2);
-+ // Note: lengths is filled with code point lengths
-+ // The NULL parameter is the ignored code unit lengths.
-
-- // See if any of the possible second words is followed by a third word
-- do {
-- // If we find a third word, stop right away
-- if (words[(wordsFound + 2) % KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd)) {
-- words[wordsFound % KHMER_LOOKAHEAD].markCurrent();
-- goto foundBest;
-- }
-- }
-- while (words[(wordsFound + 1) % KHMER_LOOKAHEAD].backUp(text));
-- }
-+ for (int32_t j = 0; j < count; j++) {
-+ int32_t ln = lengths.elementAti(j);
-+ if (ln + i >= numCodePts)
-+ continue;
-+ utext_setNativeIndex(text, ln+ix);
-+ int32_t c = utext_current32(text);
-+ if (fMarkSet.contains(c) || c == 0x17D2) { // Coeng
-+ lengths.removeElementAt(j);
-+ values.removeElementAt(j);
-+ --j;
-+ --count;
- }
-- while (words[wordsFound % KHMER_LOOKAHEAD].backUp(text));
--foundBest:
-- cuWordLength = words[wordsFound % KHMER_LOOKAHEAD].acceptMarked(text);
-- cpWordLength = words[wordsFound % KHMER_LOOKAHEAD].markedCPLength();
-- wordsFound += 1;
- }
--
-- // We come here after having either found a word or not. We look ahead to the
-- // next word. If it's not a dictionary word, we will combine it with the word we
-- // just found (if there is one), but only if the preceding word does not exceed
-- // the threshold.
-- // The text iterator should now be positioned at the end of the word we found.
-- if ((int32_t)utext_getNativeIndex(text) < rangeEnd && cpWordLength < KHMER_ROOT_COMBINE_THRESHOLD) {
-- // if it is a dictionary word, do nothing. If it isn't, then if there is
-- // no preceding word, or the non-word shares less than the minimum threshold
-- // of characters with a dictionary word, then scan to resynchronize
-- if (words[wordsFound % KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd) <= 0
-- && (cuWordLength == 0
-- || words[wordsFound % KHMER_LOOKAHEAD].longestPrefix() < KHMER_PREFIX_COMBINE_THRESHOLD)) {
-- // Look for a plausible word boundary
-- int32_t remaining = rangeEnd - (current+cuWordLength);
-- UChar32 pc;
-- UChar32 uc;
-- int32_t chars = 0;
-- for (;;) {
-- int32_t pcIndex = (int32_t)utext_getNativeIndex(text);
-- pc = utext_next32(text);
-- int32_t pcSize = (int32_t)utext_getNativeIndex(text) - pcIndex;
-- chars += pcSize;
-- remaining -= pcSize;
-- if (remaining <= 0) {
-+ if (count == 0) {
-+ utext_setNativeIndex(text, ix);
-+ int32_t c = utext_current32(text);
-+ if (fPuncSet.contains(c) || fIgnoreSet.contains(c) || c == ZWSP) {
-+ values.setElementAt(0, count);
-+ lengths.setElementAt(1, count++);
-+ } else if (fBaseSet.contains(c)) {
-+ int32_t currix = utext_getNativeIndex(text);
-+ do {
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ if (utext_getNativeIndex(text) >= rangeEnd)
- break;
-- }
-- uc = utext_current32(text);
-- if (fEndWordSet.contains(pc) && fBeginWordSet.contains(uc)) {
-- // Maybe. See if it's in the dictionary.
-- int32_t candidates = words[(wordsFound + 1) % KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd);
-- utext_setNativeIndex(text, current+cuWordLength+chars);
-- if (candidates > 0) {
-+ if (c == 0x17D2) { // Coeng
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ if (!fBaseSet.contains(c) || utext_getNativeIndex(text) >= rangeEnd) {
- break;
-+ } else {
-+ utext_next32(text);
-+ c = utext_current32(text);
-+ if (utext_getNativeIndex(text) >= rangeEnd)
-+ break;
- }
- }
-- }
--
-- // Bump the word count if there wasn't already one
-- if (cuWordLength <= 0) {
-- wordsFound += 1;
-- }
-+ } while (fMarkSet.contains(c) || fIgnoreSet.contains(c));
-+ values.setElementAt(BADSNLP, count);
-+ lengths.setElementAt(utext_getNativeIndex(text) - currix, count++);
-+ } else {
-+ values.setElementAt(BADSNLP, count);
-+ lengths.setElementAt(1, count++);
-+ }
-+ }
-
-- // Update the length with the passed-over characters
-- cuWordLength += chars;
-+ for (int32_t j = 0; j < count; j++) {
-+ uint32_t v = values.elementAti(j);
-+ int32_t newSnlp = bestSnlp.elementAti(i) + v;
-+ int32_t ln = lengths.elementAti(j);
-+ utext_setNativeIndex(text, ln+ix);
-+ int32_t c = utext_current32(text);
-+ while ((fPuncSet.contains(c) || fIgnoreSet.contains(c)) && ln + i < numCodePts) {
-+ ++ln;
-+ utext_next32(text);
-+ c = utext_current32(text);
- }
-- else {
-- // Back up to where we were for next iteration
-- utext_setNativeIndex(text, current+cuWordLength);
-+ int32_t ln_j_i = ln + i; // yes really i!
-+ if (newSnlp < bestSnlp.elementAti(ln_j_i)) {
-+ if (v == BADSNLP) {
-+ int32_t p = prev.elementAti(i);
-+ if (p < 0)
-+ prev.setElementAt(p, ln_j_i);
-+ else
-+ prev.setElementAt(-i, ln_j_i);
-+ }
-+ else
-+ prev.setElementAt(i, ln_j_i);
-+ bestSnlp.setElementAt(newSnlp, ln_j_i);
- }
- }
-+ }
-+ // Start pushing the optimal offset index into t_boundary (t for tentative).
-+ // prev[numCodePts] is guaranteed to be meaningful.
-+ // We'll first push in the reverse order, i.e.,
-+ // t_boundary[0] = numCodePts, and afterwards do a swap.
-+ UVector32 t_boundary(numCodePts+1, status);
-
-- // Never stop before a combining mark.
-- int32_t currPos;
-- while ((currPos = (int32_t)utext_getNativeIndex(text)) < rangeEnd && fMarkSet.contains(utext_current32(text))) {
-- utext_next32(text);
-- cuWordLength += (int32_t)utext_getNativeIndex(text) - currPos;
-+ int32_t numBreaks = 0;
-+ // No segmentation found, set boundary to end of range
-+ while (numCodePts >= 0 && (uint32_t)bestSnlp.elementAti(numCodePts) == kuint32max) {
-+ --numCodePts;
-+ }
-+ if (numCodePts < 0) {
-+ t_boundary.addElement(numCodePts, status);
-+ numBreaks++;
-+ } else {
-+ for (int32_t i = numCodePts; (uint32_t)i != kuint32max; i = prev.elementAti(i)) {
-+ if (i < 0) i = -i;
-+ t_boundary.addElement(i, status);
-+ numBreaks++;
- }
-+ U_ASSERT(prev.elementAti(t_boundary.elementAti(numBreaks - 1)) == 0);
-+ }
-
-- // Look ahead for possible suffixes if a dictionary word does not follow.
-- // We do this in code rather than using a rule so that the heuristic
-- // resynch continues to function. For example, one of the suffix characters
-- // could be a typo in the middle of a word.
--// if ((int32_t)utext_getNativeIndex(text) < rangeEnd && wordLength > 0) {
--// if (words[wordsFound%KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd) <= 0
--// && fSuffixSet.contains(uc = utext_current32(text))) {
--// if (uc == KHMER_PAIYANNOI) {
--// if (!fSuffixSet.contains(utext_previous32(text))) {
--// // Skip over previous end and PAIYANNOI
--// utext_next32(text);
--// utext_next32(text);
--// wordLength += 1; // Add PAIYANNOI to word
--// uc = utext_current32(text); // Fetch next character
--// }
--// else {
--// // Restore prior position
--// utext_next32(text);
--// }
--// }
--// if (uc == KHMER_MAIYAMOK) {
--// if (utext_previous32(text) != KHMER_MAIYAMOK) {
--// // Skip over previous end and MAIYAMOK
--// utext_next32(text);
--// utext_next32(text);
--// wordLength += 1; // Add MAIYAMOK to word
--// }
--// else {
--// // Restore prior position
--// utext_next32(text);
--// }
--// }
--// }
--// else {
--// utext_setNativeIndex(text, current+wordLength);
--// }
--// }
--
-- // Did we find a word on this iteration? If so, push it on the break stack
-- if (cuWordLength > 0) {
-- foundBreaks.push((current+cuWordLength), status);
-+ // Now that we're done, convert positions in t_boundary[] (indices in
-+ // the normalized input string) back to indices in the original input UText
-+ // while reversing t_boundary and pushing values to foundBreaks.
-+ for (int32_t i = numBreaks-1; i >= 0; i--) {
-+ int32_t cpPos = t_boundary.elementAti(i);
-+ if (cpPos == 0 && !breakStart && fTypes >= UBRK_LINE) continue;
-+ int32_t utextPos = cpPos + rangeStart;
-+ while (utextPos > after && scanWJ(text, utextPos, scanEnd, before, after));
-+ if (utextPos < before) {
-+ // Boundaries are added to foundBreaks output in ascending order.
-+ U_ASSERT(foundBreaks.size() == 0 ||foundBreaks.peeki() < utextPos);
-+ foundBreaks.push(utextPos, status);
- }
- }
--
-+
- // Don't return a break for the end of the dictionary range if there is one there.
-- if (foundBreaks.peeki() >= rangeEnd) {
-+ if (!breakEnd && fTypes >= UBRK_LINE && foundBreaks.peeki() >= rangeEnd) {
- (void) foundBreaks.popi();
-- wordsFound -= 1;
- }
--
-- return wordsFound;
-+ return foundBreaks.size() - wordsFound;
- }
-
- #if !UCONFIG_NO_NORMALIZATION
-diff --git a/source/common/dictbe.h b/source/common/dictbe.h
-index d3488cd..26caa75 100644
---- misc/icu/source/common/dictbe.h
-+++ build/icu/source/common/dictbe.h
-@@ -32,6 +32,15 @@ class Normalizer2;
- */
- class DictionaryBreakEngine : public LanguageBreakEngine {
- private:
-+
-+ /**
-+ * <p>Default constructor.</p>
-+ *
-+ */
-+ DictionaryBreakEngine();
-+
-+ protected:
-+
- /**
- * The set of characters handled by this engine
- * @internal
-@@ -46,11 +55,63 @@ class DictionaryBreakEngine : public LanguageBreakEngine {
-
- uint32_t fTypes;
-
-+ const int32_t WJ = 0x2060;
-+ const int32_t ZWSP = 0x200B;
-+
- /**
-- * <p>Default constructor.</p>
-- *
-+ * A Unicode set of all viramas
-+ * @internal
- */
-- DictionaryBreakEngine();
-+ UnicodeSet fViramaSet;
-+
-+ /**
-+ * A Unicode set of all base characters
-+ * @internal
-+ */
-+ UnicodeSet fBaseSet;
-+
-+ /**
-+ * A Unicode set of all marks
-+ * @internal
-+ */
-+ UnicodeSet fMarkSet;
-+
-+ /**
-+ * A Unicode set of all characters ignored ignored in dictionary matching
-+ * @internal
-+ */
-+ UnicodeSet fIgnoreSet;
-+
-+ /**
-+ * A Unicode set of all characters ignored ignored in dictionary matching
-+ * @internal
-+ */
-+ UnicodeSet fSkipStartSet;
-+
-+ /**
-+ * A Unicode set of all characters ignored ignored in dictionary matching
-+ * @internal
-+ */
-+ UnicodeSet fSkipEndSet;
-+
-+ /**
-+ * A Unicode set of all characters that should not be broken before
-+ * @internal
-+ */
-+ UnicodeSet fNBeforeSet;
-+
-+ /**
-+ * The number of clusters within which breaks are inhibited
-+ * @internal
-+ */
-+ int32_t clusterLimit;
-+
-+ bool scanWJ(UText *text, int32_t &start, int32_t end, int32_t &before, int32_t &after) const;
-+
-+ bool scanBeforeStart(UText *text, int32_t& start, bool &doBreak) const;
-+ bool scanAfterEnd(UText *text, int32_t rangeEnd, int32_t& end, bool &doBreak) const;
-+ void scanBackClusters(UText *text, int32_t textStart, int32_t& start) const;
-+ void scanFwdClusters(UText *text, int32_t textEnd, int32_t& end) const;
-
- public:
-
-@@ -81,7 +142,7 @@ class DictionaryBreakEngine : public LanguageBreakEngine {
- * <p>Find any breaks within a run in the supplied text.</p>
- *
- * @param text A UText representing the text. The iterator is left at
-- * the end of the run of characters which the engine is capable of handling
-+ * the end of the run of characters which the engine is capable of handling
- * that starts from the first (or last) character in the range.
- * @param startPos The start of the run within the supplied text.
- * @param endPos The end of the run within the supplied text.
-@@ -243,118 +304,120 @@ class LaoBreakEngine : public DictionaryBreakEngine {
-
- };
-
--/*******************************************************************
-- * BurmeseBreakEngine
-- */
--
--/**
-- * <p>BurmeseBreakEngine is a kind of DictionaryBreakEngine that uses a
-- * DictionaryMatcher and heuristics to determine Burmese-specific breaks.</p>
-- *
-- * <p>After it is constructed a BurmeseBreakEngine may be shared between
-- * threads without synchronization.</p>
-- */
--class BurmeseBreakEngine : public DictionaryBreakEngine {
-- private:
-- /**
-- * The set of characters handled by this engine
-- * @internal
-- */
--
-- UnicodeSet fBurmeseWordSet;
-- UnicodeSet fEndWordSet;
-- UnicodeSet fBeginWordSet;
-- UnicodeSet fMarkSet;
-- DictionaryMatcher *fDictionary;
--
-- public:
--
-- /**
-- * <p>Default constructor.</p>
-- *
-- * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
-- * engine is deleted.
-- */
-- BurmeseBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
--
-- /**
-- * <p>Virtual destructor.</p>
-- */
-- virtual ~BurmeseBreakEngine();
--
-- protected:
-- /**
-- * <p>Divide up a range of known dictionary characters.</p>
-- *
-- * @param text A UText representing the text
-- * @param rangeStart The start of the range of dictionary characters
-- * @param rangeEnd The end of the range of dictionary characters
-- * @param foundBreaks Output of C array of int32_t break positions, or 0
-- * @return The number of breaks found
-- */
-- virtual int32_t divideUpDictionaryRange( UText *text,
-- int32_t rangeStart,
-- int32_t rangeEnd,
-- UStack &foundBreaks ) const;
--
--};
--
--/*******************************************************************
-- * KhmerBreakEngine
-- */
--
--/**
-- * <p>KhmerBreakEngine is a kind of DictionaryBreakEngine that uses a
-- * DictionaryMatcher and heuristics to determine Khmer-specific breaks.</p>
-- *
-- * <p>After it is constructed a KhmerBreakEngine may be shared between
-- * threads without synchronization.</p>
-- */
--class KhmerBreakEngine : public DictionaryBreakEngine {
-- private:
-- /**
-- * The set of characters handled by this engine
-- * @internal
-- */
--
-- UnicodeSet fKhmerWordSet;
-- UnicodeSet fEndWordSet;
-- UnicodeSet fBeginWordSet;
-- UnicodeSet fMarkSet;
-- DictionaryMatcher *fDictionary;
--
-- public:
--
-- /**
-- * <p>Default constructor.</p>
-- *
-- * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
-- * engine is deleted.
-- */
-- KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
--
-- /**
-- * <p>Virtual destructor.</p>
-- */
-- virtual ~KhmerBreakEngine();
--
-- protected:
-- /**
-- * <p>Divide up a range of known dictionary characters.</p>
-- *
-- * @param text A UText representing the text
-- * @param rangeStart The start of the range of dictionary characters
-- * @param rangeEnd The end of the range of dictionary characters
-- * @param foundBreaks Output of C array of int32_t break positions, or 0
-- * @return The number of breaks found
-- */
-- virtual int32_t divideUpDictionaryRange( UText *text,
-- int32_t rangeStart,
-- int32_t rangeEnd,
-- UStack &foundBreaks ) const;
--
--};
--
-+/*******************************************************************
-+ * BurmeseBreakEngine
-+ */
-+
-+/**
-+ * <p>BurmeseBreakEngine is a kind of DictionaryBreakEngine that uses a
-+ * DictionaryMatcher and heuristics to determine Burmese-specific breaks.</p>
-+ *
-+ * <p>After it is constructed a BurmeseBreakEngine may be shared between
-+ * threads without synchronization.</p>
-+ */
-+class BurmeseBreakEngine : public DictionaryBreakEngine {
-+ private:
-+ /**
-+ * The set of characters handled by this engine
-+ * @internal
-+ */
-+
-+ UnicodeSet fBurmeseWordSet;
-+ UnicodeSet fEndWordSet;
-+ UnicodeSet fBeginWordSet;
-+ UnicodeSet fMarkSet;
-+ DictionaryMatcher *fDictionary;
-+
-+ public:
-+
-+ /**
-+ * <p>Default constructor.</p>
-+ *
-+ * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
-+ * engine is deleted.
-+ */
-+ BurmeseBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
-+
-+ /**
-+ * <p>Virtual destructor.</p>
-+ */
-+ virtual ~BurmeseBreakEngine();
-+
-+ protected:
-+ /**
-+ * <p>Divide up a range of known dictionary characters.</p>
-+ *
-+ * @param text A UText representing the text
-+ * @param rangeStart The start of the range of dictionary characters
-+ * @param rangeEnd The end of the range of dictionary characters
-+ * @param foundBreaks Output of C array of int32_t break positions, or 0
-+ * @return The number of breaks found
-+ */
-+ virtual int32_t divideUpDictionaryRange( UText *text,
-+ int32_t rangeStart,
-+ int32_t rangeEnd,
-+ UStack &foundBreaks ) const;
-+
-+};
-+
-+/*******************************************************************
-+ * KhmerBreakEngine
-+ */
-+
-+/**
-+ * <p>KhmerBreakEngine is a kind of DictionaryBreakEngine that uses a
-+ * DictionaryMatcher and heuristics to determine Khmer-specific breaks.</p>
-+ *
-+ * <p>After it is constructed a KhmerBreakEngine may be shared between
-+ * threads without synchronization.</p>
-+ */
-+class KhmerBreakEngine : public DictionaryBreakEngine {
-+ private:
-+ /**
-+ * The set of characters handled by this engine
-+ * @internal
-+ */
-+
-+ UnicodeSet fKhmerWordSet;
-+ UnicodeSet fBeginWordSet;
-+ UnicodeSet fPuncSet;
-+ DictionaryMatcher *fDictionary;
-+
-+ const uint32_t BADSNLP = 256 * 20;
-+ const uint32_t kuint32max = 0x7FFFFFFF;
-+
-+ public:
-+
-+ /**
-+ * <p>Default constructor.</p>
-+ *
-+ * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
-+ * engine is deleted.
-+ */
-+ KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
-+
-+ /**
-+ * <p>Virtual destructor.</p>
-+ */
-+ virtual ~KhmerBreakEngine();
-+
-+ protected:
-+ /**
-+ * <p>Divide up a range of known dictionary characters.</p>
-+ *
-+ * @param text A UText representing the text
-+ * @param rangeStart The start of the range of dictionary characters
-+ * @param rangeEnd The end of the range of dictionary characters
-+ * @param foundBreaks Output of C array of int32_t break positions, or 0
-+ * @return The number of breaks found
-+ */
-+ virtual int32_t divideUpDictionaryRange( UText *text,
-+ int32_t rangeStart,
-+ int32_t rangeEnd,
-+ UStack &foundBreaks ) const;
-+
-+};
-+
- #if !UCONFIG_NO_NORMALIZATION
-
- /*******************************************************************
-diff --git a/source/common/dictionarydata.cpp b/source/common/dictionarydata.cpp
-index cb594c6..82f2e77 100644
---- misc/icu/source/common/dictionarydata.cpp
-+++ build/icu/source/common/dictionarydata.cpp
-@@ -42,7 +42,7 @@ int32_t UCharsDictionaryMatcher::getType() const {
-
- int32_t UCharsDictionaryMatcher::matches(UText *text, int32_t maxLength, int32_t limit,
- int32_t *lengths, int32_t *cpLengths, int32_t *values,
-- int32_t *prefix) const {
-+ int32_t *prefix, UnicodeSet const* ignoreSet, int32_t minLength) const {
-
- UCharsTrie uct(characters);
- int32_t startingTextIndex = (int32_t)utext_getNativeIndex(text);
-@@ -53,7 +53,13 @@ int32_t UCharsDictionaryMatcher::matches(UText *text, int32_t maxLength, int32_t
- UStringTrieResult result = (codePointsMatched == 0) ? uct.first(c) : uct.next(c);
- int32_t lengthMatched = (int32_t)utext_getNativeIndex(text) - startingTextIndex;
- codePointsMatched += 1;
-+ if (ignoreSet != NULL && ignoreSet->contains(c)) {
-+ continue;
-+ }
- if (USTRINGTRIE_HAS_VALUE(result)) {
-+ if (codePointsMatched < minLength) {
-+ continue;
-+ }
- if (wordCount < limit) {
- if (values != NULL) {
- values[wordCount] = uct.getValue();
-@@ -110,7 +116,7 @@ int32_t BytesDictionaryMatcher::getType() const {
-
- int32_t BytesDictionaryMatcher::matches(UText *text, int32_t maxLength, int32_t limit,
- int32_t *lengths, int32_t *cpLengths, int32_t *values,
-- int32_t *prefix) const {
-+ int32_t *prefix, UnicodeSet const* ignoreSet, int32_t minLength) const {
- BytesTrie bt(characters);
- int32_t startingTextIndex = (int32_t)utext_getNativeIndex(text);
- int32_t wordCount = 0;
-@@ -120,7 +126,13 @@ int32_t BytesDictionaryMatcher::matches(UText *text, int32_t maxLength, int32_t
- UStringTrieResult result = (codePointsMatched == 0) ? bt.first(transform(c)) : bt.next(transform(c));
- int32_t lengthMatched = (int32_t)utext_getNativeIndex(text) - startingTextIndex;
- codePointsMatched += 1;
-+ if (ignoreSet != NULL && ignoreSet->contains(c)) {
-+ continue;
-+ }
- if (USTRINGTRIE_HAS_VALUE(result)) {
-+ if (codePointsMatched < minLength) {
-+ continue;
-+ }
- if (wordCount < limit) {
- if (values != NULL) {
- values[wordCount] = bt.getValue();
-diff --git a/source/common/dictionarydata.h b/source/common/dictionarydata.h
-index 0216ab0..ee9e571 100644
---- misc/icu/source/common/dictionarydata.h
-+++ build/icu/source/common/dictionarydata.h
-@@ -19,6 +19,7 @@
- #include "unicode/utext.h"
- #include "unicode/udata.h"
- #include "udataswp.h"
-+#include "unicode/uniset.h"
- #include "unicode/uobject.h"
- #include "unicode/ustringtrie.h"
-
-@@ -90,7 +91,7 @@ public:
- */
- virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit,
- int32_t *lengths, int32_t *cpLengths, int32_t *values,
-- int32_t *prefix) const = 0;
-+ int32_t *prefix, UnicodeSet const* ignoreSet = NULL, int32_t minLength = 0) const = 0;
-
- /** @return DictionaryData::TRIE_TYPE_XYZ */
- virtual int32_t getType() const = 0;
-@@ -105,7 +106,7 @@ public:
- virtual ~UCharsDictionaryMatcher();
- virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit,
- int32_t *lengths, int32_t *cpLengths, int32_t *values,
-- int32_t *prefix) const;
-+ int32_t *prefix, UnicodeSet const* ignoreSet = NULL, int32_t minLength = 0) const;
- virtual int32_t getType() const;
- private:
- const UChar *characters;
-@@ -123,7 +124,7 @@ public:
- virtual ~BytesDictionaryMatcher();
- virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit,
- int32_t *lengths, int32_t *cpLengths, int32_t *values,
-- int32_t *prefix) const;
-+ int32_t *prefix, UnicodeSet const* ignoreSet = NULL, int32_t minLength = 0) const;
- virtual int32_t getType() const;
- private:
- UChar32 transform(UChar32 c) const;
-diff --git a/source/data/Makefile.in b/source/data/Makefile.in
-index 816c82d..c637d70 100644
---- misc/icu/source/data/Makefile.in
-+++ build/icu/source/data/Makefile.in
-@@ -181,7 +181,7 @@ endif
- endif
- endif
-
--packagedata: icupkg.inc $(PKGDATA_LIST) build-local
-+packagedata: icupkg.inc $(PKGDATA_LIST) build-local $(MAINBUILDDIR)/khmerdict.stamp
- ifneq ($(ENABLE_STATIC),)
- ifeq ($(PKGDATA_MODE),dll)
- $(PKGDATA_INVOKE) $(PKGDATA) -e $(ICUDATA_ENTRY_POINT) -T $(OUTTMPDIR) -p $(ICUDATA_NAME) $(PKGDATA_LIBSTATICNAME) -m static $(PKGDATA_VERSIONING) $(PKGDATA_LIST)
-@@ -564,8 +564,14 @@ $(BRKBLDDIR)/burmesedict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
- $(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x1000 -c -i $(BUILDDIR) $(DICTSRCDIR)/burmesedict.txt $(BRKBLDDIR)/burmesedict.dict
-
- # TODO: figure out why combining characters are here?
--$(BRKBLDDIR)/khmerdict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
-- $(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x1780 -c -i $(BUILDDIR) $(DICTSRCDIR)/khmerdict.txt $(BRKBLDDIR)/khmerdict.dict
-+#$(BRKBLDDIR)/khmerdict.dict: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(DAT_FILES)
-+# $(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x1780 -c -i $(BUILDDIR) $(DICTSRCDIR)/khmerdict.txt $(BRKBLDDIR)/khmerdict.dict
-+
-+#$(MAINBUILDDIR)/khmerdict.stamp: $(TOOLBINDIR)/gendict$(TOOLEXEEXT) $(BRKSRCDIR)/khmerdict.txt build-local
-+# $(INVOKE) $(TOOLBINDIR)/gendict --bytes --transform offset-0x1780 -c -i $(BUILDDIR) $(DICTSRCDIR)/khmerdict.txt $(BRKBLDDIR)/khmerdict.dict
-+$(MAINBUILDDIR)/khmerdict.stamp: $(BRKSRCDIR)/khmerdict.dict build-local
-+ cp $< $(BRKBLDDIR)
-+ echo "timestamp" > $@
-
- #################################################### CFU
- # CFU FILES
-
diff --git a/external/icu/khmerdict.dict b/external/icu/khmerdict.dict
deleted file mode 100644
index 52605b65469d..000000000000
Binary files a/external/icu/khmerdict.dict and /dev/null differ
diff --git a/i18npool/source/breakiterator/breakiterator_unicode.cxx b/i18npool/source/breakiterator/breakiterator_unicode.cxx
index ea3f689731d4..a55149152daa 100644
--- a/i18npool/source/breakiterator/breakiterator_unicode.cxx
+++ b/i18npool/source/breakiterator/breakiterator_unicode.cxx
@@ -123,7 +123,7 @@ void SAL_CALL BreakIterator_Unicode::loadICUBreakIterator(const css::lang::Local
OUStringToOString(breakRules[breakType], RTL_TEXTENCODING_ASCII_US).getStr(), &status), status);
}
//use icu's breakiterator for Thai, Tibetan and Dzongkha
- else if (rLocale.Language != "th" && rLocale.Language != "lo" && rLocale.Language != "bo" && rLocale.Language != "dz" && rLocale.Language != "km")
+ else if (rLocale.Language != "th" && rLocale.Language != "lo" && rLocale.Language != "bo" && rLocale.Language != "dz")
{
status = U_ZERO_ERROR;
OStringBuffer aUDName(64);
More information about the Libreoffice-commits
mailing list