Files
tubestation/intl/icu/source/common/normalizer2.cpp
Alexandru Marc 52a93f69fa Backed out 19 changesets (bug 1927706) for causing reftest failures @ space-cluster-2.html
Backed out changeset 3877f1fa62f5 (bug 1927706)
Backed out changeset 233c6ebf84a2 (bug 1927706)
Backed out changeset 07e5871d5fa3 (bug 1927706)
Backed out changeset 84ef75087931 (bug 1927706)
Backed out changeset f89b916619e1 (bug 1927706)
Backed out changeset b82d9d622315 (bug 1927706)
Backed out changeset b0d2c5711865 (bug 1927706)
Backed out changeset 9529dda25bd9 (bug 1927706)
Backed out changeset 40b7907d7fc8 (bug 1927706)
Backed out changeset c549655dbd73 (bug 1927706)
Backed out changeset c5cc289771b3 (bug 1927706)
Backed out changeset 8ef66f7822c4 (bug 1927706)
Backed out changeset dff6d37fb2fe (bug 1927706)
Backed out changeset 083a0b3da643 (bug 1927706)
Backed out changeset 06649ac72a19 (bug 1927706)
Backed out changeset 019f7533abbc (bug 1927706)
Backed out changeset f1539604c459 (bug 1927706)
Backed out changeset 578667f1f0d4 (bug 1927706)
Backed out changeset 8ed1e7e7d4ab (bug 1927706)
2024-10-30 11:19:58 +02:00

573 lines
18 KiB
C++

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
* Copyright (C) 2009-2016, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: normalizer2.cpp
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* created on: 2009nov22
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_NORMALIZATION
#include "unicode/edits.h"
#include "unicode/normalizer2.h"
#include "unicode/stringoptions.h"
#include "unicode/unistr.h"
#include "unicode/unorm.h"
#include "cstring.h"
#include "mutex.h"
#include "norm2allmodes.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "ucln_cmn.h"
using icu::Normalizer2Impl;
#if NORM2_HARDCODE_NFC_DATA
// NFC/NFD data machine-generated by gennorm2 --csource
#define INCLUDED_FROM_NORMALIZER2_CPP
#include "norm2_nfc_data.h"
#endif
U_NAMESPACE_BEGIN
// Public API dispatch via Normalizer2 subclasses -------------------------- ***
Normalizer2::~Normalizer2() {}
void
Normalizer2::normalizeUTF8(uint32_t /*options*/, StringPiece src, ByteSink &sink,
Edits *edits, UErrorCode &errorCode) const {
if (U_FAILURE(errorCode)) {
return;
}
if (edits != nullptr) {
errorCode = U_UNSUPPORTED_ERROR;
return;
}
UnicodeString src16 = UnicodeString::fromUTF8(src);
normalize(src16, errorCode).toUTF8(sink);
}
UBool
Normalizer2::getRawDecomposition(UChar32, UnicodeString &) const {
return false;
}
UChar32
Normalizer2::composePair(UChar32, UChar32) const {
return U_SENTINEL;
}
uint8_t
Normalizer2::getCombiningClass(UChar32 /*c*/) const {
return 0;
}
UBool
Normalizer2::isNormalizedUTF8(StringPiece s, UErrorCode &errorCode) const {
return U_SUCCESS(errorCode) && isNormalized(UnicodeString::fromUTF8(s), errorCode);
}
// Normalizer2 implementation for the old UNORM_NONE.
class NoopNormalizer2 : public Normalizer2 {
virtual ~NoopNormalizer2();
virtual UnicodeString &
normalize(const UnicodeString &src,
UnicodeString &dest,
UErrorCode &errorCode) const override {
if(U_SUCCESS(errorCode)) {
if(&dest!=&src) {
dest=src;
} else {
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
}
return dest;
}
virtual void
normalizeUTF8(uint32_t options, StringPiece src, ByteSink &sink,
Edits *edits, UErrorCode &errorCode) const override {
if(U_SUCCESS(errorCode)) {
if (edits != nullptr) {
if ((options & U_EDITS_NO_RESET) == 0) {
edits->reset();
}
edits->addUnchanged(src.length());
}
if ((options & U_OMIT_UNCHANGED_TEXT) == 0) {
sink.Append(src.data(), src.length());
}
sink.Flush();
}
}
virtual UnicodeString &
normalizeSecondAndAppend(UnicodeString &first,
const UnicodeString &second,
UErrorCode &errorCode) const override {
if(U_SUCCESS(errorCode)) {
if(&first!=&second) {
first.append(second);
} else {
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
}
return first;
}
virtual UnicodeString &
append(UnicodeString &first,
const UnicodeString &second,
UErrorCode &errorCode) const override {
if(U_SUCCESS(errorCode)) {
if(&first!=&second) {
first.append(second);
} else {
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
}
return first;
}
virtual UBool
getDecomposition(UChar32, UnicodeString &) const override {
return false;
}
// No need to override the default getRawDecomposition().
virtual UBool
isNormalized(const UnicodeString &, UErrorCode &errorCode) const override {
return U_SUCCESS(errorCode);
}
virtual UBool
isNormalizedUTF8(StringPiece, UErrorCode &errorCode) const override {
return U_SUCCESS(errorCode);
}
virtual UNormalizationCheckResult
quickCheck(const UnicodeString &, UErrorCode &) const override {
return UNORM_YES;
}
virtual int32_t
spanQuickCheckYes(const UnicodeString &s, UErrorCode &) const override {
return s.length();
}
virtual UBool hasBoundaryBefore(UChar32) const override { return true; }
virtual UBool hasBoundaryAfter(UChar32) const override { return true; }
virtual UBool isInert(UChar32) const override { return true; }
};
NoopNormalizer2::~NoopNormalizer2() {}
Normalizer2WithImpl::~Normalizer2WithImpl() {}
DecomposeNormalizer2::~DecomposeNormalizer2() {}
ComposeNormalizer2::~ComposeNormalizer2() {}
FCDNormalizer2::~FCDNormalizer2() {}
// instance cache ---------------------------------------------------------- ***
U_CDECL_BEGIN
static UBool U_CALLCONV uprv_normalizer2_cleanup();
U_CDECL_END
static Normalizer2 *noopSingleton;
static icu::UInitOnce noopInitOnce {};
static void U_CALLCONV initNoopSingleton(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) {
return;
}
noopSingleton=new NoopNormalizer2;
if(noopSingleton==nullptr) {
errorCode=U_MEMORY_ALLOCATION_ERROR;
return;
}
ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup);
}
const Normalizer2 *Normalizer2Factory::getNoopInstance(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return nullptr; }
umtx_initOnce(noopInitOnce, &initNoopSingleton, errorCode);
return noopSingleton;
}
const Normalizer2Impl *
Normalizer2Factory::getImpl(const Normalizer2 *norm2) {
return &((Normalizer2WithImpl *)norm2)->impl;
}
Norm2AllModes::~Norm2AllModes() {
delete impl;
}
Norm2AllModes *
Norm2AllModes::createInstance(Normalizer2Impl *impl, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) {
delete impl;
return nullptr;
}
Norm2AllModes *allModes=new Norm2AllModes(impl);
if(allModes==nullptr) {
errorCode=U_MEMORY_ALLOCATION_ERROR;
delete impl;
return nullptr;
}
return allModes;
}
#if NORM2_HARDCODE_NFC_DATA
Norm2AllModes *
Norm2AllModes::createNFCInstance(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) {
return nullptr;
}
Normalizer2Impl *impl=new Normalizer2Impl;
if(impl==nullptr) {
errorCode=U_MEMORY_ALLOCATION_ERROR;
return nullptr;
}
impl->init(norm2_nfc_data_indexes, &norm2_nfc_data_trie,
norm2_nfc_data_extraData, norm2_nfc_data_smallFCD);
return createInstance(impl, errorCode);
}
static Norm2AllModes *nfcSingleton;
static icu::UInitOnce nfcInitOnce {};
static void U_CALLCONV initNFCSingleton(UErrorCode &errorCode) {
nfcSingleton=Norm2AllModes::createNFCInstance(errorCode);
ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup);
}
const Norm2AllModes *
Norm2AllModes::getNFCInstance(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return nullptr; }
umtx_initOnce(nfcInitOnce, &initNFCSingleton, errorCode);
return nfcSingleton;
}
const Normalizer2 *
Normalizer2::getNFCInstance(UErrorCode &errorCode) {
const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
return allModes!=nullptr ? &allModes->comp : nullptr;
}
const Normalizer2 *
Normalizer2::getNFDInstance(UErrorCode &errorCode) {
const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
return allModes!=nullptr ? &allModes->decomp : nullptr;
}
const Normalizer2 *Normalizer2Factory::getFCDInstance(UErrorCode &errorCode) {
const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
return allModes!=nullptr ? &allModes->fcd : nullptr;
}
const Normalizer2 *Normalizer2Factory::getFCCInstance(UErrorCode &errorCode) {
const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
return allModes!=nullptr ? &allModes->fcc : nullptr;
}
const Normalizer2Impl *
Normalizer2Factory::getNFCImpl(UErrorCode &errorCode) {
const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
return allModes!=nullptr ? allModes->impl : nullptr;
}
#endif // NORM2_HARDCODE_NFC_DATA
U_CDECL_BEGIN
static UBool U_CALLCONV uprv_normalizer2_cleanup() {
delete noopSingleton;
noopSingleton = nullptr;
noopInitOnce.reset();
#if NORM2_HARDCODE_NFC_DATA
delete nfcSingleton;
nfcSingleton = nullptr;
nfcInitOnce.reset();
#endif
return true;
}
U_CDECL_END
U_NAMESPACE_END
// C API ------------------------------------------------------------------- ***
U_NAMESPACE_USE
U_CAPI const UNormalizer2 * U_EXPORT2
unorm2_getNFCInstance(UErrorCode *pErrorCode) {
return (const UNormalizer2 *)Normalizer2::getNFCInstance(*pErrorCode);
}
U_CAPI const UNormalizer2 * U_EXPORT2
unorm2_getNFDInstance(UErrorCode *pErrorCode) {
return (const UNormalizer2 *)Normalizer2::getNFDInstance(*pErrorCode);
}
U_CAPI void U_EXPORT2
unorm2_close(UNormalizer2 *norm2) {
delete (Normalizer2 *)norm2;
}
U_CAPI int32_t U_EXPORT2
unorm2_normalize(const UNormalizer2 *norm2,
const char16_t *src, int32_t length,
char16_t *dest, int32_t capacity,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return 0;
}
if( (src==nullptr ? length!=0 : length<-1) ||
(dest==nullptr ? capacity!=0 : capacity<0) ||
(src==dest && src!=nullptr)
) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UnicodeString destString(dest, 0, capacity);
// length==0: Nothing to do, and n2wi->normalize(nullptr, nullptr, buffer, ...) would crash.
if(length!=0) {
const Normalizer2 *n2=(const Normalizer2 *)norm2;
const Normalizer2WithImpl *n2wi=dynamic_cast<const Normalizer2WithImpl *>(n2);
if(n2wi!=nullptr) {
// Avoid duplicate argument checking and support NUL-terminated src.
ReorderingBuffer buffer(n2wi->impl, destString);
if(buffer.init(length, *pErrorCode)) {
n2wi->normalize(src, length>=0 ? src+length : nullptr, buffer, *pErrorCode);
}
} else {
UnicodeString srcString(length<0, src, length);
n2->normalize(srcString, destString, *pErrorCode);
}
}
return destString.extract(dest, capacity, *pErrorCode);
}
static int32_t
normalizeSecondAndAppend(const UNormalizer2 *norm2,
char16_t *first, int32_t firstLength, int32_t firstCapacity,
const char16_t *second, int32_t secondLength,
UBool doNormalize,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return 0;
}
if( (second==nullptr ? secondLength!=0 : secondLength<-1) ||
(first==nullptr ? (firstCapacity!=0 || firstLength!=0) :
(firstCapacity<0 || firstLength<-1)) ||
(first==second && first!=nullptr)
) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UnicodeString firstString(first, firstLength, firstCapacity);
firstLength=firstString.length(); // In case it was -1.
// secondLength==0: Nothing to do, and n2wi->normalizeAndAppend(nullptr, nullptr, buffer, ...) would crash.
if(secondLength!=0) {
const Normalizer2 *n2=(const Normalizer2 *)norm2;
const Normalizer2WithImpl *n2wi=dynamic_cast<const Normalizer2WithImpl *>(n2);
if(n2wi!=nullptr) {
// Avoid duplicate argument checking and support NUL-terminated src.
UnicodeString safeMiddle;
{
ReorderingBuffer buffer(n2wi->impl, firstString);
if(buffer.init(firstLength+secondLength+1, *pErrorCode)) { // destCapacity>=-1
n2wi->normalizeAndAppend(second, secondLength>=0 ? second+secondLength : nullptr,
doNormalize, safeMiddle, buffer, *pErrorCode);
}
} // The ReorderingBuffer destructor finalizes firstString.
if(U_FAILURE(*pErrorCode) || firstString.length()>firstCapacity) {
// Restore the modified suffix of the first string.
// This does not restore first[] array contents between firstLength and firstCapacity.
// (That might be uninitialized memory, as far as we know.)
if(first!=nullptr) { /* don't dereference nullptr */
safeMiddle.extract(0, 0x7fffffff, first+firstLength-safeMiddle.length());
if(firstLength<firstCapacity) {
first[firstLength]=0; // NUL-terminate in case it was originally.
}
}
}
} else {
UnicodeString secondString(secondLength<0, second, secondLength);
if(doNormalize) {
n2->normalizeSecondAndAppend(firstString, secondString, *pErrorCode);
} else {
n2->append(firstString, secondString, *pErrorCode);
}
}
}
return firstString.extract(first, firstCapacity, *pErrorCode);
}
U_CAPI int32_t U_EXPORT2
unorm2_normalizeSecondAndAppend(const UNormalizer2 *norm2,
char16_t *first, int32_t firstLength, int32_t firstCapacity,
const char16_t *second, int32_t secondLength,
UErrorCode *pErrorCode) {
return normalizeSecondAndAppend(norm2,
first, firstLength, firstCapacity,
second, secondLength,
true, pErrorCode);
}
U_CAPI int32_t U_EXPORT2
unorm2_append(const UNormalizer2 *norm2,
char16_t *first, int32_t firstLength, int32_t firstCapacity,
const char16_t *second, int32_t secondLength,
UErrorCode *pErrorCode) {
return normalizeSecondAndAppend(norm2,
first, firstLength, firstCapacity,
second, secondLength,
false, pErrorCode);
}
U_CAPI int32_t U_EXPORT2
unorm2_getDecomposition(const UNormalizer2 *norm2,
UChar32 c, char16_t *decomposition, int32_t capacity,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return 0;
}
if(decomposition==nullptr ? capacity!=0 : capacity<0) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UnicodeString destString(decomposition, 0, capacity);
if(reinterpret_cast<const Normalizer2 *>(norm2)->getDecomposition(c, destString)) {
return destString.extract(decomposition, capacity, *pErrorCode);
} else {
return -1;
}
}
U_CAPI int32_t U_EXPORT2
unorm2_getRawDecomposition(const UNormalizer2 *norm2,
UChar32 c, char16_t *decomposition, int32_t capacity,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return 0;
}
if(decomposition==nullptr ? capacity!=0 : capacity<0) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UnicodeString destString(decomposition, 0, capacity);
if(reinterpret_cast<const Normalizer2 *>(norm2)->getRawDecomposition(c, destString)) {
return destString.extract(decomposition, capacity, *pErrorCode);
} else {
return -1;
}
}
U_CAPI UChar32 U_EXPORT2
unorm2_composePair(const UNormalizer2 *norm2, UChar32 a, UChar32 b) {
return reinterpret_cast<const Normalizer2 *>(norm2)->composePair(a, b);
}
U_CAPI uint8_t U_EXPORT2
unorm2_getCombiningClass(const UNormalizer2 *norm2, UChar32 c) {
return reinterpret_cast<const Normalizer2 *>(norm2)->getCombiningClass(c);
}
U_CAPI UBool U_EXPORT2
unorm2_isNormalized(const UNormalizer2 *norm2,
const char16_t *s, int32_t length,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return 0;
}
if((s==nullptr && length!=0) || length<-1) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UnicodeString sString(length<0, s, length);
return ((const Normalizer2 *)norm2)->isNormalized(sString, *pErrorCode);
}
U_CAPI UNormalizationCheckResult U_EXPORT2
unorm2_quickCheck(const UNormalizer2 *norm2,
const char16_t *s, int32_t length,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return UNORM_NO;
}
if((s==nullptr && length!=0) || length<-1) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return UNORM_NO;
}
UnicodeString sString(length<0, s, length);
return ((const Normalizer2 *)norm2)->quickCheck(sString, *pErrorCode);
}
U_CAPI int32_t U_EXPORT2
unorm2_spanQuickCheckYes(const UNormalizer2 *norm2,
const char16_t *s, int32_t length,
UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return 0;
}
if((s==nullptr && length!=0) || length<-1) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
UnicodeString sString(length<0, s, length);
return ((const Normalizer2 *)norm2)->spanQuickCheckYes(sString, *pErrorCode);
}
U_CAPI UBool U_EXPORT2
unorm2_hasBoundaryBefore(const UNormalizer2 *norm2, UChar32 c) {
return ((const Normalizer2 *)norm2)->hasBoundaryBefore(c);
}
U_CAPI UBool U_EXPORT2
unorm2_hasBoundaryAfter(const UNormalizer2 *norm2, UChar32 c) {
return ((const Normalizer2 *)norm2)->hasBoundaryAfter(c);
}
U_CAPI UBool U_EXPORT2
unorm2_isInert(const UNormalizer2 *norm2, UChar32 c) {
return ((const Normalizer2 *)norm2)->isInert(c);
}
// Some properties APIs ---------------------------------------------------- ***
U_CAPI uint8_t U_EXPORT2
u_getCombiningClass(UChar32 c) {
UErrorCode errorCode=U_ZERO_ERROR;
const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
if(U_SUCCESS(errorCode)) {
return nfd->getCombiningClass(c);
} else {
return 0;
}
}
U_CFUNC uint16_t
unorm_getFCD16(UChar32 c) {
UErrorCode errorCode=U_ZERO_ERROR;
const Normalizer2Impl *impl=Normalizer2Factory::getNFCImpl(errorCode);
if(U_SUCCESS(errorCode)) {
return impl->getFCD16(c);
} else {
return 0;
}
}
#endif // !UCONFIG_NO_NORMALIZATION