Attachment #405783 for bug #215058

View Details Raw Unified Return to Bug 215058

| Differences between



2020-08-01  Yusuke Suzuki  <ysuzuki@apple.com>

        [JSC] Implement Intl Language Tag Parser
        https://bugs.webkit.org/show_bug.cgi?id=215058

        Reviewed by NOBODY (OOPS!).

        * runtime/IntlLocale.cpp:
        (JSC::LocaleIDBuilder::initialize):
        (JSC::IntlLocale::initializeLocale):
        * runtime/IntlObject.cpp:
        (JSC::canonicalizeLocaleList):
        (JSC::parseVariantCode):
        (JSC::convertToUnicodeSingletonIndex):
        (JSC::isUnicodeExtensionAttribute):
        (JSC::isUnicodeExtensionKey):
        (JSC::isUnicodeExtensionTypeComponent):
        (JSC::isUnicodePUExtensionValue):
        (JSC::isUnicodeOtherExtensionValue):
        (JSC::isUnicodeTKey):
        (JSC::isUnicodeTValue):
        (JSC::LanguageTagParser::LanguageTagParser):
        (JSC::LanguageTagParser::isEOS):
        (JSC::LanguageTagParser::next):
        (JSC::LanguageTagParser::parseUnicodeLocaleId):
        (JSC::LanguageTagParser::parseUnicodeLanguageId):
        (JSC::LanguageTagParser::parseUnicodeExtensionAfterPrefix):
        (JSC::LanguageTagParser::parseTransformedExtensionAfterPrefix):
        (JSC::LanguageTagParser::parseOtherExtensionAfterPrefix):
        (JSC::LanguageTagParser::parsePUExtensionAfterPrefix):
        (JSC::LanguageTagParser::parseExtensionsAndPUExtensions):
        (JSC::isStructurallyValidLanguageTag):
        (JSC::isUnicodeLanguageId):
        * runtime/IntlObject.h:

2020-07-31  Keith Miller  <keith_miller@apple.com>

        Move Options setter to where we allow access to the Options object




class LocaleIDBuilder final {
public:
    bool initialize(const String&);
    CString toCanonical();

    void overrideLanguageScriptRegion(StringView language, StringView script, StringView region);

    Vector<char, 32> m_buffer;
};

bool LocaleIDBuilder::initialize(const String& tag)
{
    if (!isStructurallyValidLanguageTag(tag))
        return false;
    ASSERT(tag.isAllASCII());
    m_buffer = localeIDBufferForLanguageTag(tag.ascii());
    return m_buffer.size();
}


    }

    LocaleIDBuilder localeID;
    if (!localeID.initialize(tag)) {
        throwRangeError(globalObject, scope, "invalid language tag"_s);
        return;
    }



    return true;
}

// https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
// https://tc39.es/ecma402/#sec-canonicalizeunicodelocaleid
static String canonicalizeLanguageTag(const CString& tag)
{

                return { };
            }

            String tag;
            if (kValue.inherits<IntlLocale>(vm))
                tag = jsCast<IntlLocale*>(kValue)->toString();
            else {
                JSString* string = kValue.toString(globalObject);
                RETURN_IF_EXCEPTION(scope, Vector<String>());

                tag = string->value(globalObject);
                RETURN_IF_EXCEPTION(scope, Vector<String>());

                rawTag = tagValue.tryGetUtf8();
            }
            if (!rawTag) {
                if (rawTag.error() == UTF8ConversionError::OutOfMemory)
                    throwOutOfMemoryError(globalObject, scope);
                return { };
            }

            if (isStructurallyValidLanguageTag(tag)) {
                ASSERT(tag.isAllASCII());
                String canonicalizedTag = canonicalizeLanguageTag(tag.ascii());
                if (!canonicalizedTag.isNull()) {
                    if (seenSet.add(canonicalizedTag).isNewEntry)
                        seen.append(canonicalizedTag);
                    continue;
                }
                throwException(globalObject, scope, createRangeError(globalObject, errorMessage));
                return { };
            }

            String errorMessage = tryMakeString("invalid language tag: ", tag);
            if (UNLIKELY(!errorMessage)) {
                throwException(globalObject, scope, createOutOfMemoryError(globalObject));
                return { };
            }
            throwException(globalObject, scope, createRangeError(globalObject, errorMessage));
            return { };
        }
    }


    return length == 4 && isASCIIDigit(string[0]) && string.substring(1).isAllSpecialCharacters<isASCIIAlphanumeric>();
}

using VariantCode = uint64_t;
static VariantCode parseVariantCode(StringView string)
//     | unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
// https://github.com/tc39/proposal-intl-displaynames/issues/79
bool isUnicodeLanguageId(StringView string)
{
    ASSERT(isUnicodeVariantSubtag(string));
    ASSERT(string.isAllASCII());
    ASSERT(string.length() <= 8);
    ASSERT(string.length() >= 1);
    struct Code {
        LChar characters[8] { };
    };
    static_assert(std::is_unsigned_v<LChar>);
    static_assert(sizeof(VariantCode) == sizeof(Code));
    Code code { };
    for (unsigned index = 0; index < string.length(); ++index)
        code.characters[index] = string[index];
    VariantCode result = bitwise_cast<VariantCode>(code);
    ASSERT(!result); // Not possible since some characters exist.
    ASSERT(result != static_cast<VariantCode>(-1)); // Not possible since all characters are ASCII (not Latin-1).
    return result;
}

static unsigned convertToUnicodeSingletonIndex(UChar singleton)
{
    ASSERT(isASCIIAlphanumeric(singleton));
    singleton = toASCIILower(singleton);
    // 0 - 9 => numeric
    // 10 - 35 => alpha
    if (isASCIIDigit(singleton))
        return singleton - '0';
    return (singleton - 'a') + 10;
}
static constexpr unsigned numberOfUnicodeSingletons = 10 + 26; // Digits + Alphabets.

static bool isUnicodeExtensionAttribute(StringView string)
{
    auto length = string.length();
    if (length >= 3 && length <= 8)
        return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
    return false;
}

static bool isUnicodeExtensionKey(StringView string)
{
    return string.length() == 2 && isASCIIAlphanumeric(string[0]) && isASCIIAlpha(string[1]);
}

static bool isUnicodeExtensionTypeComponent(StringView string)
{
    auto length = string.length();
    if (length >= 3 && length <= 8)
        return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
    return false;
}

static bool isUnicodePUExtensionValue(StringView string)
{
    auto length = string.length();
    if (length >= 1 && length <= 8)
        return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
    return false;
}

static bool isUnicodeOtherExtensionValue(StringView string)
{
    auto length = string.length();
    if (length >= 2 && length <= 8)
        return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
    return false;
}

static bool isUnicodeTKey(StringView string)
{
    return string.length() == 2 && isASCIIAlpha(string[0]) && isASCIIDigit(string[1]);
}

static bool isUnicodeTValue(StringView string)
{
    auto length = string.length();
    if (length >= 3 && length <= 8)
        return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
    return false;
}

// The IsStructurallyValidLanguageTag abstract operation verifies that the locale argument (which must be a String value)
//
//     represents a well-formed "Unicode BCP 47 locale identifier" as specified in Unicode Technical Standard 35 section 3.2,
//     does not include duplicate variant subtags, and
//     does not include duplicate singleton subtags.
//
//  The abstract operation returns true if locale can be generated from the EBNF grammar in section 3.2 of the Unicode Technical Standard 35,
//  starting with unicode_locale_id, and does not contain duplicate variant or singleton subtags (other than as a private use subtag).
//  It returns false otherwise. Terminal value characters in the grammar are interpreted as the Unicode equivalents of the ASCII octet values given.
//
// https://unicode.org/reports/tr35/#Unicode_locale_identifier
class LanguageTagParser {
public:
    LanguageTagParser(StringView tag)
        : m_range(tag.splitAllowingEmptyEntries('-'))
        , m_cursor(m_range.begin())
    {
        ASSERT(m_cursor != m_range.end());
        m_current = *m_cursor;
    }

    bool parseUnicodeLocaleId();
    bool parseUnicodeLanguageId();

    bool isEOS()
    {
        return m_cursor == m_range.end();
    }

    bool next()
    {
        if (isEOS())
            return false;

        ++m_cursor;
        if (isEOS()) {
            m_current = StringView();
            return false;
        }
        m_current = *m_cursor;
        return true;
    }

private:
    bool parseExtensionsAndPUExtensions();

    bool parseUnicodeExtensionAfterPrefix();
    bool parseTransformedExtensionAfterPrefix();
    bool parseOtherExtensionAfterPrefix();
    bool parsePUExtensionAfterPrefix();

    StringView::SplitResult m_range;
    StringView::SplitResult::Iterator m_cursor;
    StringView m_current;
};

bool LanguageTagParser::parseUnicodeLocaleId()
{
    // unicode_locale_id    = unicode_language_id
    //                        extensions*
    //                        pu_extensions? ;
    ASSERT(!isEOS());
    if (!parseUnicodeLanguageId())
        return false;
    if (isEOS())
        return true;
    if (!parseExtensionsAndPUExtensions())
        return false;
    return true;
}

bool LanguageTagParser::parseUnicodeLanguageId()
{
    // unicode_language_id  = unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
    ASSERT(!isEOS());
    if (!isUnicodeLanguageSubtag(m_current))
        return false;
    if (!next())
        return true;

    if (isUnicodeScriptSubtag(m_current)) {
        if (!next())
            return true;
    }

    if (isUnicodeRegionSubtag(m_current)) {
        if (!next())
            return true;
    }

    HashSet<VariantCode> variantCodes;
    while (true) {
        if (!isUnicodeVariantSubtag(m_current))
            return true;
        // https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
        // does not include duplicate variant subtags
        if (!variantCodes.add(parseVariantCode(m_current)).isNewEntry)
            return false;
        if (!next())
            return true;
    }
}

bool LanguageTagParser::parseUnicodeExtensionAfterPrefix()
{
    // ((sep keyword)+ | (sep attribute)+ (sep keyword)*) ;
    //
    // keyword = key (sep type)? ;
    // key = alphanum alpha ;
    // type = alphanum{3,8} (sep alphanum{3,8})* ;
    // attribute = alphanum{3,8} ;
    ASSERT(!isEOS());
    bool isAttributeOrKeyword = false;
    if (isUnicodeExtensionAttribute(m_current)) {
        isAttributeOrKeyword = true;
        while (true) {
            if (!isUnicodeExtensionAttribute(m_current))
                break;
            if (!next())
                return true;
        }
    }

    if (isUnicodeExtensionKey(m_current)) {
        isAttributeOrKeyword = true;
        while (true) {
            if (!isUnicodeExtensionKey(m_current))
                break;
            if (!next())
                return true;
            while (true) {
                if (!isUnicodeExtensionTypeComponent(m_current))
                    break;
                if (!next())
                    return true;
            }
        }
    }

    if (!isAttributeOrKeyword)
        return false;
    return true;
}

bool LanguageTagParser::parseTransformedExtensionAfterPrefix()
{
    // ((sep tlang (sep tfield)*) | (sep tfield)+) ;
    //
    // tlang = unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
    // tfield = tkey tvalue;
    // tkey = alpha digit ;
    // tvalue = (sep alphanum{3,8})+ ;
    ASSERT(!isEOS());
    bool isTLangOrTField = false;
    if (isUnicodeLanguageSubtag(m_current)) {
        isTLangOrTField = true;
        if (!parseUnicodeLanguageId())
            return false;
        if (isEOS())
            return true;
    }

    if (isUnicodeTKey(m_current)) {
        isTLangOrTField = true;
        while (true) {
            if (!isUnicodeTKey(m_current))
                break;
            if (!next())
                return false;
            if (!isUnicodeTValue(m_current))
                return false;
            if (!next())
                return true;
            while (true) {
                if (!isUnicodeTValue(m_current))
                    break;
                if (!next())
                    return true;
            }
        }
    }

    if (!isTLangOrTField)
        return false;
    return true;
}

bool LanguageTagParser::parseOtherExtensionAfterPrefix()
{
    // (sep alphanum{2,8})+ ;
    ASSERT(!isEOS());
    if (!isUnicodeOtherExtensionValue(m_current))
        return false;
    if (!next())
        return true;
    if (isUnicodeScriptSubtag(subtags[cursor]))
        ++cursor;

    while (true) {
        if (!isUnicodeOtherExtensionValue(m_current))
            return true;
        if (!next())
            return true;
    }
}

bool LanguageTagParser::parsePUExtensionAfterPrefix()
{
    // (sep alphanum{1,8})+ ;
    ASSERT(!isEOS());
    if (!isUnicodePUExtensionValue(m_current))
        return false;
    if (!next())
        return true;
    if (isUnicodeRegionSubtag(subtags[cursor]))
        ++cursor;

    while (true) {
        if (!isUnicodePUExtensionValue(m_current))
            return true;
        if (!next())
            return true;
    }
}

bool LanguageTagParser::parseExtensionsAndPUExtensions()
{
    // unicode_locale_id    = unicode_language_id
    //                        extensions*
    //                        pu_extensions? ;
    //
    // extensions = unicode_locale_extensions
    //            | transformed_extensions
    //            | other_extensions ;
    //
    // pu_extensions = sep [xX] (sep alphanum{1,8})+ ;
    ASSERT(!isEOS());
    Bitmap<numberOfUnicodeSingletons> singletonsSet { };
    while (true) {
        if (m_current.length() != 1)
            return true;
        UChar prefixCode = m_current[0];
        if (!isASCIIAlphanumeric(prefixCode))
            return true;

        // https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
        // does not include duplicate singleton subtags.
        //
        // https://unicode.org/reports/tr35/#Unicode_locale_identifier
        // As is often the case, the complete syntactic constraints are not easily captured by ABNF,
        // so there is a further condition: There cannot be more than one extension with the same singleton (-a-, …, -t-, -u-, …).
        // Note that the private use extension (-x-) must come after all other extensions.
        if (singletonsSet.get(convertToUnicodeSingletonIndex(prefixCode)))
            return false;
        singletonsSet.set(convertToUnicodeSingletonIndex(prefixCode), true);

        switch (prefixCode) {
        case 'u':
        case 'U': {
            // unicode_locale_extensions = sep [uU] ((sep keyword)+ | (sep attribute)+ (sep keyword)*) ;
            if (!next())
                return false;
            if (!parseUnicodeExtensionAfterPrefix())
                return false;
            if (isEOS())
                return true;
            break; // Next extension.
        }
        case 't':
        case 'T': {
            // transformed_extensions = sep [tT] ((sep tlang (sep tfield)*) | (sep tfield)+) ;
            if (!next())
                return false;
            if (!parseTransformedExtensionAfterPrefix())
                return false;
            if (isEOS())
                return true;
            break; // Next extension.
        }
        case 'x':
        case 'X': {
            // pu_extensions = sep [xX] (sep alphanum{1,8})+ ;
            if (!next())
                return false;
            if (!parsePUExtensionAfterPrefix())
                return false;
            return true; // If pu_extensions appear, no extensions can follow after that. This must be the end of unicode_locale_id.
        }
        default: {
            // other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ;
            if (!next())
                return false;
            if (!parseOtherExtensionAfterPrefix())
                return false;
            if (isEOS())
                return true;
            break; // Next extension.
        }
        }
    }
}

// https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
bool isStructurallyValidLanguageTag(StringView string)
{
    LanguageTagParser parser(string);
    if (!parser.parseUnicodeLocaleId())
        return false;
    if (!parser.isEOS())
        return false;
    return true;
}

// unicode_language_id, but intersection of BCP47 and UTS35.
// unicode_language_id =
//     | unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
// https://github.com/tc39/proposal-intl-displaynames/issues/79
bool isUnicodeLanguageId(StringView string)
{
    LanguageTagParser parser(string);
    if (!parser.parseUnicodeLanguageId())
        return false;
    if (!parser.isEOS())
        return false;
    return true;
}

bool isWellFormedCurrencyCode(StringView currency)
{
    return currency.length() == 3 && currency.isAllSpecialCharacters<isASCIIAlpha>();

Lines 117-122 bool isUnicodeScriptSubtag(StringView); a/Source/JavaScriptCore/runtime/IntlObject.h_sec1
- a/Source/JavaScriptCore/runtime/IntlObject.h +1 lines
117	bool isUnicodeRegionSubtag(StringView);	117	bool isUnicodeRegionSubtag(StringView);
118	bool isUnicodeVariantSubtag(StringView);	118	bool isUnicodeVariantSubtag(StringView);
119	bool isUnicodeLanguageId(StringView);	119	bool isUnicodeLanguageId(StringView);
		120	bool isStructurallyValidLanguageTag(StringView);
120		121
121	bool isWellFormedCurrencyCode(StringView);	122	bool isWellFormedCurrencyCode(StringView);
122		123

Return to Bug 215058

Lines 68-74 void IntlLocale::visitChildren(JSCell* cell, SlotVisitor& visitor) a/Source/JavaScriptCore/runtime/IntlLocale.cpp_sec1
- a/Source/JavaScriptCore/runtime/IntlLocale.cpp -4 / +7 lines
68		68
69	class LocaleIDBuilder final {	69	class LocaleIDBuilder final {
70	public:	70	public:
71	bool initialize(const CString&);	71	bool initialize(const String&);
72	CString toCanonical();	72	CString toCanonical();
73		73
74	void overrideLanguageScriptRegion(StringView language, StringView script, StringView region);	74	void overrideLanguageScriptRegion(StringView language, StringView script, StringView region);
Lines 78-86 class LocaleIDBuilder final { a/Source/JavaScriptCore/runtime/IntlLocale.cpp_sec2
78	Vector<char, 32> m_buffer;	78	Vector<char, 32> m_buffer;
79	};	79	};
80		80
81	bool LocaleIDBuilder::initialize(const CString& tag)	81	bool LocaleIDBuilder::initialize(const String& tag)
82	{	82	{
83	m_buffer = localeIDBufferForLanguageTag(tag);	83	if (!isStructurallyValidLanguageTag(tag))
		84	return false;
		85	ASSERT(tag.isAllASCII());
		86	m_buffer = localeIDBufferForLanguageTag(tag.ascii());
84	return m_buffer.size();	87	return m_buffer.size();
85	}	88	}
86		89
Lines 216-222 void IntlLocale::initializeLocale(JSGlobalObject* globalObject, const String& ta a/Source/JavaScriptCore/runtime/IntlLocale.cpp_sec3
216	}	219	}
217		220
218	LocaleIDBuilder localeID;	221	LocaleIDBuilder localeID;
219	if (!localeID.initialize(tag.utf8())) {	222	if (!localeID.initialize(tag)) {
220	throwRangeError(globalObject, scope, "invalid language tag"_s);	223	throwRangeError(globalObject, scope, "invalid language tag"_s);
221	return;	224	return;
222	}	225	}

Lines 1-3 a/Source/JavaScriptCore/ChangeLog_sec1
- a/Source/JavaScriptCore/ChangeLog +35 lines
		1	2020-08-01 Yusuke Suzuki <ysuzuki@apple.com>
		2
		3	[JSC] Implement Intl Language Tag Parser
		4	https://bugs.webkit.org/show_bug.cgi?id=215058
		5
		6	Reviewed by NOBODY (OOPS!).
		7
		8	* runtime/IntlLocale.cpp:
		9	(JSC::LocaleIDBuilder::initialize):
		10	(JSC::IntlLocale::initializeLocale):
		11	* runtime/IntlObject.cpp:
		12	(JSC::canonicalizeLocaleList):
		13	(JSC::parseVariantCode):
		14	(JSC::convertToUnicodeSingletonIndex):
		15	(JSC::isUnicodeExtensionAttribute):
		16	(JSC::isUnicodeExtensionKey):
		17	(JSC::isUnicodeExtensionTypeComponent):
		18	(JSC::isUnicodePUExtensionValue):
		19	(JSC::isUnicodeOtherExtensionValue):
		20	(JSC::isUnicodeTKey):
		21	(JSC::isUnicodeTValue):
		22	(JSC::LanguageTagParser::LanguageTagParser):
		23	(JSC::LanguageTagParser::isEOS):
		24	(JSC::LanguageTagParser::next):
		25	(JSC::LanguageTagParser::parseUnicodeLocaleId):
		26	(JSC::LanguageTagParser::parseUnicodeLanguageId):
		27	(JSC::LanguageTagParser::parseUnicodeExtensionAfterPrefix):
		28	(JSC::LanguageTagParser::parseTransformedExtensionAfterPrefix):
		29	(JSC::LanguageTagParser::parseOtherExtensionAfterPrefix):
		30	(JSC::LanguageTagParser::parsePUExtensionAfterPrefix):
		31	(JSC::LanguageTagParser::parseExtensionsAndPUExtensions):
		32	(JSC::isStructurallyValidLanguageTag):
		33	(JSC::isUnicodeLanguageId):
		34	* runtime/IntlObject.h:
		35
1	2020-07-31 Keith Miller <keith_miller@apple.com>	36	2020-07-31 Keith Miller <keith_miller@apple.com>
2		37
3	Move Options setter to where we allow access to the Options object	38	Move Options setter to where we allow access to the Options object

Lines 393-399 bool isUnicodeLocaleIdentifierType(StringView string) a/Source/JavaScriptCore/runtime/IntlObject.cpp_sec1
- a/Source/JavaScriptCore/runtime/IntlObject.cpp -43 / +410 lines
393	return true;	393	return true;
394	}	394	}
395		395
396	// https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
397	// https://tc39.es/ecma402/#sec-canonicalizeunicodelocaleid	396	// https://tc39.es/ecma402/#sec-canonicalizeunicodelocaleid
398	static String canonicalizeLanguageTag(const CString& tag)	397	static String canonicalizeLanguageTag(const CString& tag)
399	{	398	{
Lines 454-490 Vector<String> canonicalizeLocaleList(JSGlobalObject* globalObject, JSValue loca a/Source/JavaScriptCore/runtime/IntlObject.cpp_sec2
454	return { };	453	return { };
455	}	454	}
456		455
457	Expected<CString, UTF8ConversionError> rawTag;	456	String tag;
458	if (kValue.inherits<IntlLocale>(vm))	457	if (kValue.inherits<IntlLocale>(vm))
459	rawTag = jsCast<IntlLocale*>(kValue)->toString().tryGetUtf8();	458	tag = jsCast<IntlLocale*>(kValue)->toString();
460	else {	459	else {
461	JSString* tag = kValue.toString(globalObject);	460	JSString* string = kValue.toString(globalObject);
462	RETURN_IF_EXCEPTION(scope, Vector<String>());	461	RETURN_IF_EXCEPTION(scope, Vector<String>());
463		462
464	auto tagValue = tag->value(globalObject);	463	tag = string->value(globalObject);
465	RETURN_IF_EXCEPTION(scope, Vector<String>());	464	RETURN_IF_EXCEPTION(scope, Vector<String>());
466
467	rawTag = tagValue.tryGetUtf8();
468	}
469	if (!rawTag) {
470	if (rawTag.error() == UTF8ConversionError::OutOfMemory)
471	throwOutOfMemoryError(globalObject, scope);
472	return { };
473	}	465	}
474		466
475	String canonicalizedTag = canonicalizeLanguageTag(rawTag.value());	467	if (isStructurallyValidLanguageTag(tag)) {
476	if (canonicalizedTag.isNull()) {	468	ASSERT(tag.isAllASCII());
477	String errorMessage = tryMakeString("invalid language tag: ", rawTag->data());	469	String canonicalizedTag = canonicalizeLanguageTag(tag.ascii());
478	if (UNLIKELY(!errorMessage)) {	470	if (!canonicalizedTag.isNull()) {
479	throwException(globalObject, scope, createOutOfMemoryError(globalObject));	471	if (seenSet.add(canonicalizedTag).isNewEntry)
480	return { };	472	seen.append(canonicalizedTag);
		473	continue;
481	}	474	}
482	throwException(globalObject, scope, createRangeError(globalObject, errorMessage));
483	return { };
484	}	475	}
485		476
486	if (seenSet.add(canonicalizedTag).isNewEntry)	477	String errorMessage = tryMakeString("invalid language tag: ", tag);
487	seen.append(canonicalizedTag);	478	if (UNLIKELY(!errorMessage)) {
		479	throwException(globalObject, scope, createOutOfMemoryError(globalObject));
		480	return { };
		481	}
		482	throwException(globalObject, scope, createRangeError(globalObject, errorMessage));
		483	return { };
488	}	484	}
489	}	485	}
490		486
Lines 843-883 bool isUnicodeVariantSubtag(StringView string) a/Source/JavaScriptCore/runtime/IntlObject.cpp_sec3
843	return length == 4 && isASCIIDigit(string[0]) && string.substring(1).isAllSpecialCharacters<isASCIIAlphanumeric>();	839	return length == 4 && isASCIIDigit(string[0]) && string.substring(1).isAllSpecialCharacters<isASCIIAlphanumeric>();
844	}	840	}
845		841
846	// unicode_language_id, but intersection of BCP47 and UTS35.	842	using VariantCode = uint64_t;
847	// unicode_language_id =	843	static VariantCode parseVariantCode(StringView string)
848	// \| unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
849	// https://github.com/tc39/proposal-intl-displaynames/issues/79
850	bool isUnicodeLanguageId(StringView string)
851	{	844	{
852	Vector<StringView, 4> subtags;	845	ASSERT(isUnicodeVariantSubtag(string));
853	for (auto subtag : string.splitAllowingEmptyEntries('-'))	846	ASSERT(string.isAllASCII());
854	subtags.append(subtag);	847	ASSERT(string.length() <= 8);
		848	ASSERT(string.length() >= 1);
		849	struct Code {
		850	LChar characters[8] { };
		851	};
		852	static_assert(std::is_unsigned_v<LChar>);
		853	static_assert(sizeof(VariantCode) == sizeof(Code));
		854	Code code { };
		855	for (unsigned index = 0; index < string.length(); ++index)
		856	code.characters[index] = string[index];
		857	VariantCode result = bitwise_cast<VariantCode>(code);
		858	ASSERT(!result); // Not possible since some characters exist.
		859	ASSERT(result != static_cast<VariantCode>(-1)); // Not possible since all characters are ASCII (not Latin-1).
		860	return result;
		861	}
		862
		863	static unsigned convertToUnicodeSingletonIndex(UChar singleton)
		864	{
		865	ASSERT(isASCIIAlphanumeric(singleton));
		866	singleton = toASCIILower(singleton);
		867	// 0 - 9 => numeric
		868	// 10 - 35 => alpha
		869	if (isASCIIDigit(singleton))
		870	return singleton - '0';
		871	return (singleton - 'a') + 10;
		872	}
		873	static constexpr unsigned numberOfUnicodeSingletons = 10 + 26; // Digits + Alphabets.
		874
		875	static bool isUnicodeExtensionAttribute(StringView string)
		876	{
		877	auto length = string.length();
		878	if (length >= 3 && length <= 8)
		879	return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
		880	return false;
		881	}
		882
		883	static bool isUnicodeExtensionKey(StringView string)
		884	{
		885	return string.length() == 2 && isASCIIAlphanumeric(string[0]) && isASCIIAlpha(string[1]);
		886	}
		887
		888	static bool isUnicodeExtensionTypeComponent(StringView string)
		889	{
		890	auto length = string.length();
		891	if (length >= 3 && length <= 8)
		892	return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
		893	return false;
		894	}
		895
		896	static bool isUnicodePUExtensionValue(StringView string)
		897	{
		898	auto length = string.length();
		899	if (length >= 1 && length <= 8)
		900	return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
		901	return false;
		902	}
		903
		904	static bool isUnicodeOtherExtensionValue(StringView string)
		905	{
		906	auto length = string.length();
		907	if (length >= 2 && length <= 8)
		908	return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
		909	return false;
		910	}
		911
912	static bool isUnicodeTKey(StringView string)
913	{
914	return string.length() == 2 && isASCIIAlpha(string[0]) && isASCIIDigit(string[1]);
915	}
916
917	static bool isUnicodeTValue(StringView string)
918	{
919	auto length = string.length();
920	if (length >= 3 && length <= 8)
921	return string.isAllSpecialCharacters<isASCIIAlphanumeric>();
922	return false;
923	}
924
925	// The IsStructurallyValidLanguageTag abstract operation verifies that the locale argument (which must be a String value)
926	//
927	// represents a well-formed "Unicode BCP 47 locale identifier" as specified in Unicode Technical Standard 35 section 3.2,
928	// does not include duplicate variant subtags, and
929	// does not include duplicate singleton subtags.
930	//
931	// The abstract operation returns true if locale can be generated from the EBNF grammar in section 3.2 of the Unicode Technical Standard 35,
932	// starting with unicode_locale_id, and does not contain duplicate variant or singleton subtags (other than as a private use subtag).
933	// It returns false otherwise. Terminal value characters in the grammar are interpreted as the Unicode equivalents of the ASCII octet values given.
934	//
935	// https://unicode.org/reports/tr35/#Unicode_locale_identifier
936	class LanguageTagParser {
937	public:
938	LanguageTagParser(StringView tag)
939	: m_range(tag.splitAllowingEmptyEntries('-'))
940	, m_cursor(m_range.begin())
941	{
942	ASSERT(m_cursor != m_range.end());
943	m_current = *m_cursor;
944	}
945
946	bool parseUnicodeLocaleId();
947	bool parseUnicodeLanguageId();
948
949	bool isEOS()
950	{
951	return m_cursor == m_range.end();
952	}
953
954	bool next()
955	{
956	if (isEOS())
957	return false;
958
959	++m_cursor;
960	if (isEOS()) {
961	m_current = StringView();
962	return false;
963	}
964	m_current = *m_cursor;
965	return true;
966	}
967
968	private:
969	bool parseExtensionsAndPUExtensions();
970
971	bool parseUnicodeExtensionAfterPrefix();
972	bool parseTransformedExtensionAfterPrefix();
973	bool parseOtherExtensionAfterPrefix();
974	bool parsePUExtensionAfterPrefix();
975
976	StringView::SplitResult m_range;
977	StringView::SplitResult::Iterator m_cursor;
978	StringView m_current;
979	};
980
981	bool LanguageTagParser::parseUnicodeLocaleId()
982	{
983	// unicode_locale_id = unicode_language_id
984	// extensions*
985	// pu_extensions? ;
986	ASSERT(!isEOS());
987	if (!parseUnicodeLanguageId())
988	return false;
989	if (isEOS())
990	return true;
991	if (!parseExtensionsAndPUExtensions())
992	return false;
993	return true;
994	}
995
996	bool LanguageTagParser::parseUnicodeLanguageId()
997	{
998	// unicode_language_id = unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
999	ASSERT(!isEOS());
1000	if (!isUnicodeLanguageSubtag(m_current))
1001	return false;
1002	if (!next())
1003	return true;
1004
1005	if (isUnicodeScriptSubtag(m_current)) {
1006	if (!next())
1007	return true;
1008	}
1009
1010	if (isUnicodeRegionSubtag(m_current)) {
1011	if (!next())
1012	return true;
1013	}
1014
1015	HashSet<VariantCode> variantCodes;
1016	while (true) {
1017	if (!isUnicodeVariantSubtag(m_current))
1018	return true;
1019	// https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
1020	// does not include duplicate variant subtags
1021	if (!variantCodes.add(parseVariantCode(m_current)).isNewEntry)
1022	return false;
1023	if (!next())
1024	return true;
1025	}
1026	}
1027
1028	bool LanguageTagParser::parseUnicodeExtensionAfterPrefix()
1029	{
1030	// ((sep keyword)+ \| (sep attribute)+ (sep keyword)*) ;
1031	//
1032	// keyword = key (sep type)? ;
1033	// key = alphanum alpha ;
1034	// type = alphanum{3,8} (sep alphanum{3,8})* ;
1035	// attribute = alphanum{3,8} ;
1036	ASSERT(!isEOS());
1037	bool isAttributeOrKeyword = false;
1038	if (isUnicodeExtensionAttribute(m_current)) {
1039	isAttributeOrKeyword = true;
1040	while (true) {
1041	if (!isUnicodeExtensionAttribute(m_current))
1042	break;
1043	if (!next())
1044	return true;
1045	}
1046	}
1047
1048	if (isUnicodeExtensionKey(m_current)) {
1049	isAttributeOrKeyword = true;
1050	while (true) {
1051	if (!isUnicodeExtensionKey(m_current))
1052	break;
1053	if (!next())
1054	return true;
1055	while (true) {
1056	if (!isUnicodeExtensionTypeComponent(m_current))
1057	break;
1058	if (!next())
1059	return true;
1060	}
1061	}
1062	}
1063
1064	if (!isAttributeOrKeyword)
1065	return false;
1066	return true;
1067	}
1068
1069	bool LanguageTagParser::parseTransformedExtensionAfterPrefix()
1070	{
1071	// ((sep tlang (sep tfield)*) \| (sep tfield)+) ;
1072	//
1073	// tlang = unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
1074	// tfield = tkey tvalue;
1075	// tkey = alpha digit ;
1076	// tvalue = (sep alphanum{3,8})+ ;
1077	ASSERT(!isEOS());
1078	bool isTLangOrTField = false;
1079	if (isUnicodeLanguageSubtag(m_current)) {
1080	isTLangOrTField = true;
1081	if (!parseUnicodeLanguageId())
1082	return false;
1083	if (isEOS())
1084	return true;
1085	}
855		1086
856	ASSERT(subtags.size() >= 1);	1087	if (isUnicodeTKey(m_current)) {
857	unsigned cursor = 0;	1088	isTLangOrTField = true;
858	if (!isUnicodeLanguageSubtag(subtags[cursor]))	1089	while (true) {
		1090	if (!isUnicodeTKey(m_current))
		1091	break;
		1092	if (!next())
		1093	return false;
		1094	if (!isUnicodeTValue(m_current))
		1095	return false;
		1096	if (!next())
		1097	return true;
		1098	while (true) {
		1099	if (!isUnicodeTValue(m_current))
		1100	break;
		1101	if (!next())
		1102	return true;
		1103	}
		1104	}
		1105	}
		1106
		1107	if (!isTLangOrTField)
859	return false;	1108	return false;
860	++cursor;	1109	return true;
		1110	}
861		1111
862	if (cursor == subtags.size())	1112	bool LanguageTagParser::parseOtherExtensionAfterPrefix()
		1113	{
		1114	// (sep alphanum{2,8})+ ;
		1115	ASSERT(!isEOS());
		1116	if (!isUnicodeOtherExtensionValue(m_current))
		1117	return false;
		1118	if (!next())
863	return true;	1119	return true;
864	if (isUnicodeScriptSubtag(subtags[cursor]))
865	++cursor;
866		1120
867	if (cursor == subtags.size())	1121	while (true) {
		1122	if (!isUnicodeOtherExtensionValue(m_current))
		1123	return true;
		1124	if (!next())
		1125	return true;
		1126	}
		1127	}
		1128
		1129	bool LanguageTagParser::parsePUExtensionAfterPrefix()
		1130	{
		1131	// (sep alphanum{1,8})+ ;
		1132	ASSERT(!isEOS());
		1133	if (!isUnicodePUExtensionValue(m_current))
		1134	return false;
		1135	if (!next())
868	return true;	1136	return true;
869	if (isUnicodeRegionSubtag(subtags[cursor]))
870	++cursor;
871		1137
872	while (true) {	1138	while (true) {
873	if (cursor == subtags.size())	1139	if (!isUnicodePUExtensionValue(m_current))
		1140	return true;
		1141	if (!next())
874	return true;	1142	return true;
875	if (!isUnicodeVariantSubtag(subtags[cursor]))	1143	}
		1144	}
		1145
		1146	bool LanguageTagParser::parseExtensionsAndPUExtensions()
		1147	{
		1148	// unicode_locale_id = unicode_language_id
		1149	// extensions*
		1150	// pu_extensions? ;
		1151	//
		1152	// extensions = unicode_locale_extensions
		1153	// \| transformed_extensions
		1154	// \| other_extensions ;
		1155	//
		1156	// pu_extensions = sep [xX] (sep alphanum{1,8})+ ;
		1157	ASSERT(!isEOS());
		1158	Bitmap<numberOfUnicodeSingletons> singletonsSet { };
		1159	while (true) {
		1160	if (m_current.length() != 1)
		1161	return true;
		1162	UChar prefixCode = m_current[0];
		1163	if (!isASCIIAlphanumeric(prefixCode))
		1164	return true;
		1165
		1166	// https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
		1167	// does not include duplicate singleton subtags.
		1168	//
		1169	// https://unicode.org/reports/tr35/#Unicode_locale_identifier
		1170	// As is often the case, the complete syntactic constraints are not easily captured by ABNF,
		1171	// so there is a further condition: There cannot be more than one extension with the same singleton (-a-, …, -t-, -u-, …).
		1172	// Note that the private use extension (-x-) must come after all other extensions.
		1173	if (singletonsSet.get(convertToUnicodeSingletonIndex(prefixCode)))
876	return false;	1174	return false;
877	++cursor;	1175	singletonsSet.set(convertToUnicodeSingletonIndex(prefixCode), true);
		1176
		1177	switch (prefixCode) {
		1178	case 'u':
		1179	case 'U': {
		1180	// unicode_locale_extensions = sep [uU] ((sep keyword)+ \| (sep attribute)+ (sep keyword)*) ;
		1181	if (!next())
		1182	return false;
		1183	if (!parseUnicodeExtensionAfterPrefix())
		1184	return false;
		1185	if (isEOS())
		1186	return true;
		1187	break; // Next extension.
		1188	}
		1189	case 't':
		1190	case 'T': {
		1191	// transformed_extensions = sep [tT] ((sep tlang (sep tfield)*) \| (sep tfield)+) ;
		1192	if (!next())
		1193	return false;
		1194	if (!parseTransformedExtensionAfterPrefix())
		1195	return false;
		1196	if (isEOS())
		1197	return true;
		1198	break; // Next extension.
		1199	}
		1200	case 'x':
		1201	case 'X': {
		1202	// pu_extensions = sep [xX] (sep alphanum{1,8})+ ;
		1203	if (!next())
		1204	return false;
		1205	if (!parsePUExtensionAfterPrefix())
		1206	return false;
		1207	return true; // If pu_extensions appear, no extensions can follow after that. This must be the end of unicode_locale_id.
		1208	}
		1209	default: {
		1210	// other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ;
		1211	if (!next())
		1212	return false;
		1213	if (!parseOtherExtensionAfterPrefix())
		1214	return false;
		1215	if (isEOS())
		1216	return true;
		1217	break; // Next extension.
		1218	}
		1219	}
878	}	1220	}
879	}	1221	}
880		1222
		1223	// https://tc39.es/ecma402/#sec-isstructurallyvalidlanguagetag
		1224	bool isStructurallyValidLanguageTag(StringView string)
		1225	{
		1226	LanguageTagParser parser(string);
		1227	if (!parser.parseUnicodeLocaleId())
		1228	return false;
		1229	if (!parser.isEOS())
		1230	return false;
		1231	return true;
		1232	}
		1233
		1234	// unicode_language_id, but intersection of BCP47 and UTS35.
		1235	// unicode_language_id =
		1236	// \| unicode_language_subtag (sep unicode_script_subtag)? (sep unicode_region_subtag)? (sep unicode_variant_subtag)* ;
		1237	// https://github.com/tc39/proposal-intl-displaynames/issues/79
		1238	bool isUnicodeLanguageId(StringView string)
		1239	{
		1240	LanguageTagParser parser(string);
		1241	if (!parser.parseUnicodeLanguageId())
		1242	return false;
		1243	if (!parser.isEOS())
		1244	return false;
		1245	return true;
		1246	}
		1247
881	bool isWellFormedCurrencyCode(StringView currency)	1248	bool isWellFormedCurrencyCode(StringView currency)
882	{	1249	{
883	return currency.length() == 3 && currency.isAllSpecialCharacters<isASCIIAlpha>();	1250	return currency.length() == 3 && currency.isAllSpecialCharacters<isASCIIAlpha>();