std.uni

import std.uni;
void main()
{
    // initialize code point sets using script/block or property name
    // now 'set' contains code points from both scripts.
    auto set = unicode("Cyrillic") | unicode("Armenian");
    // same thing but simpler and checked at compile-time
    auto ascii = unicode.ASCII;
    auto currency = unicode.Currency_Symbol;

    // easy set ops
    auto a = set & ascii;
    assert(a.empty); // as it has no intersection with ascii
    a = set | ascii;
    auto b = currency - a; // subtract all ASCII, Cyrillic and Armenian

    // some properties of code point sets
    assert(b.length > 45); // 46 items in Unicode 6.1, even more in 6.2
    // testing presence of a code point in a set
    // is just fine, it is O(logN)
    assert(!b['$']);
    assert(!b['\u058F']); // Armenian dram sign
    assert(b['¥']);

    // building fast lookup tables, these guarantee O(1) complexity
    // 1-level Trie lookup table essentially a huge bit-set ~262Kb
    auto oneTrie = toTrie!1(b);
    // 2-level far more compact but typically slightly slower
    auto twoTrie = toTrie!2(b);
    // 3-level even smaller, and a bit slower yet
    auto threeTrie = toTrie!3(b);
    assert(oneTrie['£']);
    assert(twoTrie['£']);
    assert(threeTrie['£']);

    // build the trie with the most sensible trie level
    // and bind it as a functor
    auto cyrillicOrArmenian = toDelegate(set);
    auto balance = find!(cyrillicOrArmenian)("Hello ընկեր!");
    assert(balance == "ընկեր!");
    // compatible with bool delegate(dchar)
    bool delegate(dchar) bindIt = cyrillicOrArmenian;

    // Normalization
    string s = "Plain ascii (and not only), is always normalized!";
    assert(s is normalize(s));// is the same string

    string nonS = "A\u0308ffin"; // A ligature
    auto nS = normalize(nonS); // to NFC, the W3C endorsed standard
    assert(nS == "Äffin");
    assert(nS != nonS);
    string composed = "Äffin";

    assert(normalize!NFD(composed) == "A\u0308ffin");
    // to NFKD, compatibility decomposition useful for fuzzy matching/searching
    assert(normalize!NFKD("2¹⁰") == "210");
}

auto elemsPerPage = (2 ^^ bits_per_page) / Value.sizeOfInBits;
pages[index[n >> bits_per_page]][n & (elemsPerPage - 1)];

enum dchar lineSep;

Constant code point (0x2028) - line separator.

enum dchar paraSep;

Constant code point (0x2029) - paragraph separator.

enum dchar nelSep;

Constant code point (0x0085) - next line.

template isCodepointSet(T)

Tests if T is some kind a set of code points. Intended for template constraints.

enum auto isIntegralPair(T, V = uint);

Tests if T is a pair of integers that implicitly convert to V. The following code must compile for any pair T:

(T x){ V a = x[0]; V b = x[1];}

The following must not compile:

(T x){ V c = x[2];}

alias CodepointSet = InversionList!(GcPolicy).InversionList;

The recommended default type for set of code points. For details, see the current implementation: InversionList.

struct CodepointInterval;

The recommended type of std.typecons.Tuple to represent [a, b) intervals of code points. As used in InversionList. Any interval type should pass isIntegralPair trait.

struct InversionList(SP = GcPolicy);

InversionList is a set of code points represented as an array of open-right [a, b) intervals (see CodepointInterval above). The name comes from the way the representation reads left to right. For instance a set of all values [10, 50), [80, 90), plus a singular value 60 looks like this:

10, 50, 60, 61, 80, 90

The way to read this is: start with negative meaning that all numbers smaller then the next one are not present in this set (and positive - the contrary). Then switch positive/negative after each number passed from left to right.

This way negative spans until 10, then positive until 50, then negative until 60, then positive until 61, and so on. As seen this provides a space-efficient storage of highly redundant data that comes in long runs. A description which Unicode character properties fit nicely. The technique itself could be seen as a variation on RLE encoding.

Sets are value types (just like int is) thus they are never aliased.

Example:

auto a = CodepointSet('a', 'z'+1);
auto b = CodepointSet('A', 'Z'+1);
auto c = a;
a = a | b;
assert(a == CodepointSet('A', 'Z'+1, 'a', 'z'+1));
assert(a != c);

See also unicode for simpler construction of sets from predefined ones.

Memory usage is 8 bytes per each contiguous interval in a set. The value semantics are achieved by using the COW technique and thus it's not safe to cast this type to shared.

Note:

It's not recommended to rely on the template parameters or the exact type of a current code point set in std.uni. The type and parameters may change when the standard allocators design is finalized. Use isCodepointSet with templates or just stick with the default alias CodepointSet throughout the whole code base.

pure this(Set)(Set set)
if (isCodepointSet!Set);

Construct from another code point set of any type.

pure this(Range)(Range intervals)
if (isForwardRange!Range && isIntegralPair!(ElementType!Range));

Construct a set from a forward range of code point intervals.

this()(uint[] intervals...);

Construct a set from plain values of code point intervals.

Examples:

import std.algorithm.comparison : equal;

auto set = CodepointSet('a', 'z'+1, 'а', 'я'+1);
foreach(v; 'a'..'z'+1)
    assert(set[v]);
// Cyrillic lowercase interval
foreach(v; 'а'..'я'+1)
    assert(set[v]);
//specific order is not required, intervals may interesect
auto set2 = CodepointSet('а', 'я'+1, 'a', 'd', 'b', 'z'+1);
//the same end result
assert(set2.byInterval.equal(set.byInterval));

@property auto byInterval();

Get range that spans all of the code point intervals in this InversionList.

Example:

import std.algorithm.comparison : equal;
import std.typecons : tuple;

auto set = CodepointSet('A', 'D'+1, 'a', 'd'+1);

assert(set.byInterval.equal([tuple('A','E'), tuple('a','e')]));

const bool opIndex(uint val);

Tests the presence of code point val in this set.

Examples:

auto gothic = unicode.Gothic;
// Gothic letter ahsa
assert(gothic['\U00010330']);
// no ascii in Gothic obviously
assert(!gothic['$']);

@property size_t length();

Number of code points in this set

This opBinary(string op, U)(U rhs)
if (isCodepointSet!U || is(U : dchar));

Sets support natural syntax for set algebra, namely:

Operator	Math notation	Description
&	a ∩ b	intersection
|	a ∪ b	union
-	a ∖ b	subtraction
~	a ~ b	symmetric set difference i.e. (a ∪ b) \ (a ∩ b)

Examples:

import std.algorithm.comparison : equal;
import std.range : iota;

auto lower = unicode.LowerCase;
auto upper = unicode.UpperCase;
auto ascii = unicode.ASCII;

assert((lower & upper).empty); // no intersection
auto lowerASCII = lower & ascii;
assert(lowerASCII.byCodepoint.equal(iota('a', 'z'+1)));
// throw away all of the lowercase ASCII
assert((ascii - lower).length == 128 - 26);

auto onlyOneOf = lower ~ ascii;
assert(!onlyOneOf['Δ']); // not ASCII and not lowercase
assert(onlyOneOf['$']); // ASCII and not lowercase
assert(!onlyOneOf['a']); // ASCII and lowercase
assert(onlyOneOf['я']); // not ASCII but lowercase

// throw away all cased letters from ASCII
auto noLetters = ascii - (lower | upper);
assert(noLetters.length == 128 - 26*2);

ref This opOpAssign(string op, U)(U rhs)
if (isCodepointSet!U || is(U : dchar));

The 'op=' versions of the above overloaded operators.

const bool opBinaryRight(string op : "in", U)(U ch)
if (is(U : dchar));

Tests the presence of codepoint ch in this set, the same as opIndex.

Examples:

assert('я' in unicode.Cyrillic);
assert(!('z' in unicode.Cyrillic));

auto opUnary(string op : "!")();

Obtains a set that is the inversion of this set.

See Also:

inverted

@property auto byCodepoint();

A range that spans each code point in this set.

Examples:

import std.algorithm.comparison : equal;
import std.range : iota;

auto set = unicode.ASCII;
set.byCodepoint.equal(iota(0, 0x80));

void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt);

Obtain a textual representation of this InversionList in form of open-right intervals.

The formatting flag is applied individually to each value, for example:

%s and %d format the intervals as a [low..high) range of integrals

%x formats the intervals as a [low..high) range of lowercase hex characters

%X formats the intervals as a [low..high) range of uppercase hex characters

Examples:

import std.conv : to;
import std.format : format;
import std.uni : unicode;

assert(unicode.Cyrillic.to!string ==
    "[1024..1157) [1159..1320) [7467..7468) [7544..7545) [11744..11776) [42560..42648) [42655..42656)");

// The specs '%s' and '%d' are equivalent to the to!string call above.
assert(format("%d", unicode.Cyrillic) == unicode.Cyrillic.to!string);

assert(format("%#x", unicode.Cyrillic) ==
    "[0x400..0x485) [0x487..0x528) [0x1d2b..0x1d2c) [0x1d78..0x1d79) [0x2de0..0x2e00) [0xa640..0xa698) [0xa69f..0xa6a0)");

assert(format("%#X", unicode.Cyrillic) ==
    "[0X400..0X485) [0X487..0X528) [0X1D2B..0X1D2C) [0X1D78..0X1D79) [0X2DE0..0X2E00) [0XA640..0XA698) [0XA69F..0XA6A0)");

ref auto add()(uint a, uint b);

Add an interval [a, b) to this set.

Examples:

CodepointSet someSet;
someSet.add('0', '5').add('A','Z'+1);
someSet.add('5', '9'+1);
assert(someSet['0']);
assert(someSet['5']);
assert(someSet['9']);
assert(someSet['Z']);

@property auto inverted();

Obtains a set that is the inversion of this set.

See the '!' opUnary for the same but using operators.

Examples:

auto set = unicode.ASCII;
// union with the inverse gets all of the code points in the Unicode
assert((set | set.inverted).length == 0x110000);
// no intersection with the inverse
assert((set & set.inverted).empty);

string toSourceCode(string funcName = "");

Generates string with D source code of unary function with name of funcName taking a single dchar argument. If funcName is empty the code is adjusted to be a lambda function.

The function generated tests if the code point passed belongs to this set or not. The result is to be used with string mixin. The intended usage area is aggressive optimization via meta programming in parser generators and the like.

Note: Use with care for relatively small or regular sets. It could end up being slower then just using multi-staged tables.

Example:

import std.stdio;

// construct set directly from [a, b$RPAREN intervals
auto set = CodepointSet(10, 12, 45, 65, 100, 200);
writeln(set);
writeln(set.toSourceCode("func"));

The above outputs something along the lines of:

bool func(dchar ch)  @safe pure nothrow @nogc
{
    if(ch < 45)
    {
        if(ch == 10 || ch == 11) return true;
        return false;
    }
    else if (ch < 65) return true;
    else
    {
        if(ch < 100) return false;
        if(ch < 200) return true;
        return false;
    }
}

const @property bool empty();

True if this set doesn't contain any code points.

Examples:

CodepointSet emptySet;
assert(emptySet.length == 0);
assert(emptySet.empty);

template codepointSetTrie(sizes...) if (sumOfIntegerTuple!sizes == 21)

A shorthand for creating a custom multi-level fixed Trie from a CodepointSet. sizes are numbers of bits per level, with the most significant bits used first.

Note: The sum of sizes must be equal 21.

See Also:

toTrie, which is even simpler.

Example:

{
    import std.stdio;
    auto set = unicode("Number");
    auto trie = codepointSetTrie!(8, 5, 8)(set);
    writeln("Input code points to test:");
    foreach(line; stdin.byLine)
    {
        int count=0;
        foreach(dchar ch; line)
            if(trie[ch])// is number
                count++;
        writefln("Contains %d number code points.", count);
    }
}

template CodepointSetTrie(sizes...) if (sumOfIntegerTuple!sizes == 21)

Type of Trie generated by codepointSetTrie function.

template codepointTrie(T, sizes...) if (sumOfIntegerTuple!sizes == 21)

A slightly more general tool for building fixed Trie for the Unicode data.

Specifically unlike codepointSetTrie it's allows creating mappings of dchar to an arbitrary type T.

Note: Overload taking CodepointSets will naturally convert only to bool mapping Tries.

template CodepointTrie(T, sizes...) if (sumOfIntegerTuple!sizes == 21)

Type of Trie as generated by codepointTrie function.

struct MatcherConcept;

Conceptual type that outlines the common properties of all UTF Matchers.

Note: For illustration purposes only, every method call results in assertion failure. Use utfMatcher to obtain a concrete matcher for UTF-8 or UTF-16 encodings.

bool match(Range)(ref Range inp)
if (isRandomAccessRange!Range && is(ElementType!Range : char));

bool skip(Range)(ref Range inp)
if (isRandomAccessRange!Range && is(ElementType!Range : char));

bool test(Range)(ref Range inp)
if (isRandomAccessRange!Range && is(ElementType!Range : char));

Perform a semantic equivalent 2 operations: decoding a code point at front of inp and testing if it belongs to the set of code points of this matcher.

The effect on inp depends on the kind of function called:

Match. If the codepoint is found in the set then range inp is advanced by its size in code units, otherwise the range is not modifed.

Skip. The range is always advanced by the size of the tested code point regardless of the result of test.

Test. The range is left unaffected regardless of the result of test.

Examples:

string truth = "2² = 4";
auto m = utfMatcher!char(unicode.Number);
assert(m.match(truth)); // '2' is a number all right
assert(truth == "² = 4"); // skips on match
assert(m.match(truth)); // so is the superscript '2'
assert(!m.match(truth)); // space is not a number
assert(truth == " = 4"); // unaffected on no match
assert(!m.skip(truth)); // same test ...
assert(truth == "= 4"); // but skips a codepoint regardless
assert(!m.test(truth)); // '=' is not a number
assert(truth == "= 4"); // test never affects argument

enum auto isUtfMatcher(M, C);

Test if M is an UTF Matcher for ranges of Char.

@trusted auto utfMatcher(Char, Set)(Set set)
if (isCodepointSet!Set);

Constructs a matcher object to classify code points from the set for encoding that has Char as code unit.

See MatcherConcept for API outline.

auto toTrie(size_t level, Set)(Set set)
if (isCodepointSet!Set);

Convenience function to construct optimal configurations for packed Trie from any set of code points.

The parameter level indicates the number of trie levels to use, allowed values are: 1, 2, 3 or 4. Levels represent different trade-offs speed-size wise.

Level 1 is fastest and the most memory hungry (a bit array).

Level 4 is the slowest and has the smallest footprint.

See the Synopsis section for example.

Note: Level 4 stays very practical (being faster and more predictable) compared to using direct lookup on the set itself.

auto toDelegate(Set)(Set set)
if (isCodepointSet!Set);

Builds a Trie with typically optimal speed-size trade-off and wraps it into a delegate of the following type: bool delegate(dchar ch).

Effectively this creates a 'tester' lambda suitable for algorithms like std.algorithm.find that take unary predicates.

See the Synopsis section for example.

struct unicode;

A single entry point to lookup Unicode code point sets by name or alias of a block, script or general category.

It uses well defined standard rules of property name lookup. This includes fuzzy matching of names, so that 'White_Space', 'white-SpAce' and 'whitespace' are all considered equal and yield the same set of white space characters.

pure @property auto opDispatch(string name)();

Performs the lookup of set of code points with compile-time correctness checking. This short-cut version combines 3 searches: across blocks, scripts, and common binary properties.

Note that since scripts and blocks overlap the usual trick to disambiguate is used - to get a block use unicode.InBlockName, to search a script use unicode.ScriptName.

See Also:

block, script and (not included in this search) hangulSyllableType.

auto opCall(C)(in C[] name)
if (is(C : dchar));

The same lookup across blocks, scripts, or binary properties, but performed at run-time. This version is provided for cases where name is not known beforehand; otherwise compile-time checked opDispatch is typically a better choice.

See the table of properties for available sets.

struct block;

Narrows down the search for sets of code points to all Unicode blocks.

Note: Here block names are unambiguous as no scripts are searched and thus to search use simply unicode.block.BlockName notation.

See table of properties for available sets.

See Also:

table of properties.

Examples:

// use .block for explicitness
assert(unicode.block.Greek_and_Coptic == unicode.InGreek_and_Coptic);

struct script;

Narrows down the search for sets of code points to all Unicode scripts.

See the table of properties for available sets.

Examples:

auto arabicScript = unicode.script.arabic;
auto arabicBlock = unicode.block.arabic;
// there is an intersection between script and block
assert(arabicBlock['؁']);
assert(arabicScript['؁']);
// but they are different
assert(arabicBlock != arabicScript);
assert(arabicBlock == unicode.inArabic);
assert(arabicScript == unicode.arabic);

struct hangulSyllableType;

Fetch a set of code points that have the given hangul syllable type.

Other non-binary properties (once supported) follow the same notation - unicode.propertyName.propertyValue for compile-time checked access and unicode.propertyName(propertyValue) for run-time checked one.

See the table of properties for available sets.

Examples:

// L here is syllable type not Letter as in unicode.L short-cut
auto leadingVowel = unicode.hangulSyllableType("L");
// check that some leading vowels are present
foreach(vowel; '\u1110'..'\u115F')
    assert(leadingVowel[vowel]);
assert(leadingVowel == unicode.hangulSyllableType.L);

size_t graphemeStride(C)(in C[] input, size_t index)
if (is(C : dchar));

Computes the length of grapheme cluster starting at index. Both the resulting length and the index are measured in code units.

Parameters:

C	type that is implicitly convertible to dchars
C[] input	array of grapheme clusters
size_t index	starting index into input[]

Returns:

length of grapheme cluster

Examples:

assert(graphemeStride("  ", 1) == 1);
// A + combing ring above
string city = "A\u030Arhus";
size_t first = graphemeStride(city, 0);
assert(first == 3); //\u030A has 2 UTF-8 code units
assert(city[0..first] == "A\u030A");
assert(city[first..$] == "rhus");

Grapheme decodeGrapheme(Input)(ref Input inp)
if (isInputRange!Input && is(Unqual!(ElementType!Input) == dchar));

Reads one full grapheme cluster from an input range of dchar inp.

For examples see the Grapheme below.

Note: This function modifies inp and thus inp must be an L-value.

auto byGrapheme(Range)(Range range)
if (isInputRange!Range && is(Unqual!(ElementType!Range) == dchar));

Iterate a string by grapheme.

Useful for doing string manipulation that needs to be aware of graphemes.

See Also:

byCodePoint

Examples:

import std.conv;
import std.range;
import std.algorithm;
auto text = "noe\u0308l"; // noël using e + combining diaeresis
assert(text.walkLength == 5); // 5 code points

auto gText = text.byGrapheme;
assert(gText.walkLength == 4); // 4 graphemes

assert(gText.take(3).equal("noe\u0308".byGrapheme));
assert(gText.drop(3).equal("l".byGrapheme));

auto byCodePoint(Range)(Range range)
if (isInputRange!Range && is(Unqual!(ElementType!Range) == Grapheme));

Range byCodePoint(Range)(Range range)
if (isInputRange!Range && is(Unqual!(ElementType!Range) == dchar));

Lazily transform a range of Graphemes to a range of code points.

Useful for converting the result to a string after doing operations on graphemes.

Acts as the identity function when given a range of code points.

Examples:

import std.conv : text;
import std.range;

string s = "noe\u0308l"; // noël

// reverse it and convert the result to a string
string reverse = s.byGrapheme
    .array
    .retro
    .byCodePoint
    .text;

assert(reverse == "le\u0308on"); // lëon

struct Grapheme;

A structure designed to effectively pack characters of a grapheme cluster.

Grapheme has value semantics so 2 copies of a Grapheme always refer to distinct objects. In most actual scenarios a Grapheme fits on the stack and avoids memory allocation overhead for all but quite long clusters.

See Also:

decodeGrapheme, graphemeStride

const pure nothrow @nogc @trusted dchar opIndex(size_t index);

Gets a code point at the given index in this cluster.

pure nothrow @nogc @trusted void opIndexAssign(dchar ch, size_t index);

Writes a code point ch at given index in this cluster.

Warning: Use of this facility may invalidate grapheme cluster, see also Grapheme.valid.

Examples:

auto g = Grapheme("A\u0302");
assert(g[0] == 'A');
assert(g.valid);
g[1] = '~'; // ASCII tilda is not a combining mark
assert(g[1] == '~');
assert(!g.valid);

pure nothrow @nogc @system auto opSlice(size_t a, size_t b);

pure nothrow @nogc @system auto opSlice();

Random-access range over Grapheme's characters.

Warning: Invalidates when this Grapheme leaves the scope, attempts to use it then would lead to memory corruption.

const pure nothrow @nogc @property @trusted size_t length();

Grapheme cluster length in code points.

ref auto opOpAssign(string op)(dchar ch);

Append character ch to this grapheme.

Warning: Use of this facility may invalidate grapheme cluster, see also valid.

See Also:

Grapheme.valid

Examples:

import std.algorithm.comparison : equal;
auto g = Grapheme("A");
assert(g.valid);
g ~= '\u0301';
assert(g[].equal("A\u0301"));
assert(g.valid);
g ~= "B";
// not a valid grapheme cluster anymore
assert(!g.valid);
// still could be useful though
assert(g[].equal("A\u0301B"));

ref auto opOpAssign(string op, Input)(Input inp)
if (isInputRange!Input && is(ElementType!Input : dchar));

Append all characters from the input range inp to this Grapheme.

@property bool valid()();

True if this object contains valid extended grapheme cluster. Decoding primitives of this module always return a valid Grapheme.

Appending to and direct manipulation of grapheme's characters may render it no longer valid. Certain applications may chose to use Grapheme as a "small string" of any code points and ignore this property entirely.

int sicmp(S1, S2)(S1 str1, S2 str2)
if (isForwardRange!S1 && is(Unqual!(ElementType!S1) == dchar) && isForwardRange!S2 && is(Unqual!(ElementType!S2) == dchar));

Does basic case-insensitive comparison of strings str1 and str2. This function uses simpler comparison rule thus achieving better performance than icmp. However keep in mind the warning below.

Parameters:

S1 str1	a string or a ForwardRange of dchars
S2 str2	a string or a ForwardRange of dchars

Returns:

An int that is 0 if the strings match, <0 if str1 is lexicographically "less" than str2, >0 if str1 is lexicographically "greater" than str2

Warning: This function only handles 1:1 code point mapping and thus is not sufficient for certain alphabets like German, Greek and few others.

See Also:

icmp std.algorithm.comparison.cmp

Examples:

assert(sicmp("Август", "авгусТ") == 0);
// Greek also works as long as there is no 1:M mapping in sight
assert(sicmp("ΌΎ", "όύ") == 0);
// things like the following won't get matched as equal
// Greek small letter iota with dialytika and tonos
assert(sicmp("ΐ", "\u03B9\u0308\u0301") != 0);

// while icmp has no problem with that
assert(icmp("ΐ", "\u03B9\u0308\u0301") == 0);
assert(icmp("ΌΎ", "όύ") == 0);

int icmp(S1, S2)(S1 str1, S2 str2)
if (isForwardRange!S1 && is(Unqual!(ElementType!S1) == dchar) && isForwardRange!S2 && is(Unqual!(ElementType!S2) == dchar));

Does case insensitive comparison of str1 and str2. Follows the rules of full case-folding mapping. This includes matching as equal german ß with "ss" and other 1:M code point mappings unlike sicmp. The cost of icmp being pedantically correct is slightly worse performance.

Examples:

assert(icmp("Rußland", "Russland") == 0);
assert(icmp("ᾩ -> \u1F70\u03B9", "\u1F61\u03B9 -> ᾲ") == 0);

pure nothrow @nogc @safe ubyte combiningClass(dchar ch);

Returns the combining class of ch.

Examples:

// shorten the code
alias CC = combiningClass;

// combining tilda
assert(CC('\u0303') == 230);
// combining ring below
assert(CC('\u0325') == 220);
// the simple consequence is that  "tilda" should be
// placed after a "ring below" in a sequence

enum UnicodeDecomposition: int;

Unicode character decomposition type.

Canonical

Canonical decomposition. The result is canonically equivalent sequence.

Compatibility

Compatibility decomposition. The result is compatibility equivalent sequence.

Note: Compatibility decomposition is a lossy conversion, typically suitable only for fuzzy matching and internal processing.

pure nothrow @trusted dchar compose(dchar first, dchar second);

Try to canonically compose 2 characters. Returns the composed character if they do compose and dchar.init otherwise.

The assumption is that first comes before second in the original text, usually meaning that the first is a starter.

Note: Hangul syllables are not covered by this function. See composeJamo below.

Examples:

assert(compose('A','\u0308') == '\u00C4');
assert(compose('A', 'B') == dchar.init);
assert(compose('C', '\u0301') == '\u0106');
// note that the starter is the first one
// thus the following doesn't compose
assert(compose('\u0308', 'A') == dchar.init);

Grapheme decompose(UnicodeDecomposition decompType = Canonical)(dchar ch);

Returns a full Canonical (by default) or Compatibility decomposition of character ch. If no decomposition is available returns a Grapheme with the ch itself.

Note: This function also decomposes hangul syllables as prescribed by the standard.

See Also:

decomposeHangul for a restricted version that takes into account only hangul syllables but no other decompositions.

Examples:

assert(compose('A','\u0308') == '\u00C4');
assert(compose('A', 'B') == dchar.init);
assert(compose('C', '\u0301') == '\u0106');
// note that the starter is the first one
// thus the following doesn't compose
assert(compose('\u0308', 'A') == dchar.init);

assert(decompose('Ĉ')[].equalS("C\u0302"));
assert(decompose('D')[].equalS("D"));
assert(decompose('\uD4DC')[].equalS("\u1111\u1171\u11B7"));
assert(decompose!Compatibility('¹')[].equalS("1"));

@trusted Grapheme decomposeHangul(dchar ch);

Decomposes a Hangul syllable. If ch is not a composed syllable then this function returns Grapheme containing only ch as is.

Examples:

import std.algorithm;
assert(decomposeHangul('\uD4DB')[].equal("\u1111\u1171\u11B6"));

pure nothrow @nogc @trusted dchar composeJamo(dchar lead, dchar vowel, dchar trailing = (dchar).init);

Try to compose hangul syllable out of a leading consonant (lead), a vowel and optional trailing consonant jamos.

On success returns the composed LV or LVT hangul syllable.

If any of lead and vowel are not a valid hangul jamo of the respective character class returns dchar.init.

Examples:

assert(composeJamo('\u1111', '\u1171', '\u11B6') == '\uD4DB');
// leaving out T-vowel, or passing any codepoint
// that is not trailing consonant composes an LV-syllable
assert(composeJamo('\u1111', '\u1171') == '\uD4CC');
assert(composeJamo('\u1111', '\u1171', ' ') == '\uD4CC');
assert(composeJamo('\u1111', 'A') == dchar.init);
assert(composeJamo('A', '\u1171') == dchar.init);

enum NormalizationForm: int;

Enumeration type for normalization forms, passed as template parameter for functions like normalize.

NFC

NFD

NFKC

NFKD

Shorthand aliases from values indicating normalization forms.

inout(C)[] normalize(NormalizationForm norm = NFC, C)(inout(C)[] input);

Returns input string normalized to the chosen form. Form C is used by default.

For more information on normalization forms see the normalization section.

Note: In cases where the string in question is already normalized, it is returned unmodified and no memory allocation happens.

Examples:

// any encoding works
wstring greet = "Hello world";
assert(normalize(greet) is greet); // the same exact slice

// An example of a character with all 4 forms being different:
// Greek upsilon with acute and hook symbol (code point 0x03D3)
assert(normalize!NFC("ϓ") == "\u03D3");
assert(normalize!NFD("ϓ") == "\u03D2\u0301");
assert(normalize!NFKC("ϓ") == "\u038E");
assert(normalize!NFKD("ϓ") == "\u03A5\u0301");

bool allowedIn(NormalizationForm norm)(dchar ch);

Tests if dchar ch is always allowed (Quick_Check=YES) in normalization form norm.

Examples:

// e.g. Cyrillic is always allowed, so is ASCII
assert(allowedIn!NFC('я'));
assert(allowedIn!NFD('я'));
assert(allowedIn!NFKC('я'));
assert(allowedIn!NFKD('я'));
assert(allowedIn!NFC('Z'));

pure nothrow @nogc @safe bool isWhite(dchar c);

Whether or not c is a Unicode whitespace character. (general Unicode category: Part of C0(tab, vertical tab, form feed, carriage return, and linefeed characters), Zs, Zl, Zp, and NEL(U+0085))

pure nothrow @nogc @safe bool isLower(dchar c);

Return whether c is a Unicode lowercase character.

pure nothrow @nogc @safe bool isUpper(dchar c);

Return whether c is a Unicode uppercase character.

auto asLowerCase(Range)(Range str)
if (isInputRange!Range && isSomeChar!(ElementEncodingType!Range) && !isConvertibleToString!Range);

auto asUpperCase(Range)(Range str)
if (isInputRange!Range && isSomeChar!(ElementEncodingType!Range) && !isConvertibleToString!Range);

Convert input range or string to upper or lower case.

Does not allocate memory. Characters in UTF-8 or UTF-16 format that cannot be decoded are treated as std.utf.replacementDchar.

Parameters:

Range str

string or range of characters

Returns:

an InputRange of dchars

See Also:

toUpper, toLower

Examples:

import std.algorithm.comparison : equal;

assert("hEllo".asUpperCase.equal("HELLO"));

auto asCapitalized(Range)(Range str)
if (isInputRange!Range && isSomeChar!(ElementEncodingType!Range) && !isConvertibleToString!Range);

Capitalize input range or string, meaning convert the first character to upper case and subsequent characters to lower case.

Does not allocate memory. Characters in UTF-8 or UTF-16 format that cannot be decoded are treated as std.utf.replacementDchar.

Parameters:

Range str

string or range of characters

Returns:

an InputRange of dchars

See Also:

toUpper, toLower asUpperCase, asLowerCase

Examples:

import std.algorithm.comparison : equal;

assert("hEllo".asCapitalized.equal("Hello"));

pure @trusted void toLowerInPlace(C)(ref C[] s)
if (is(C == char) || is(C == wchar) || is(C == dchar));

Converts s to lowercase (by performing Unicode lowercase mapping) in place. For a few characters string length may increase after the transformation, in such a case the function reallocates exactly once. If s does not have any uppercase characters, then s is unaltered.

pure @trusted void toUpperInPlace(C)(ref C[] s)
if (is(C == char) || is(C == wchar) || is(C == dchar));

Converts s to uppercase (by performing Unicode uppercase mapping) in place. For a few characters string length may increase after the transformation, in such a case the function reallocates exactly once. If s does not have any lowercase characters, then s is unaltered.

pure nothrow @nogc @safe dchar toLower(dchar c);

If c is a Unicode uppercase character, then its lowercase equivalent is returned. Otherwise c is returned.

Warning: certain alphabets like German and Greek have no 1:1 upper-lower mapping. Use overload of toLower which takes full string instead.

pure @trusted S toLower(S)(S s)
if (isSomeString!S);

Returns a string which is identical to s except that all of its characters are converted to lowercase (by preforming Unicode lowercase mapping). If none of s characters were affected, then s itself is returned.

pure nothrow @nogc @safe dchar toUpper(dchar c);

If c is a Unicode lowercase character, then its uppercase equivalent is returned. Otherwise c is returned.

Warning: Certain alphabets like German and Greek have no 1:1 upper-lower mapping. Use overload of toUpper which takes full string instead.

toUpper can be used as an argument to std.algorithm.iteration.map to produce an algorithm that can convert a range of characters to upper case without allocating memory. A string can then be produced by using std.algorithm.mutation.copy to send it to an std.array.appender.

Examples:

import std.algorithm;
import std.uni;
import std.array;

auto abuf = appender!(char[])();
"hello".map!toUpper.copy(&abuf);
assert(abuf.data == "HELLO");

pure @trusted S toUpper(S)(S s)
if (isSomeString!S);

Returns a string which is identical to s except that all of its characters are converted to uppercase (by preforming Unicode uppercase mapping). If none of s characters were affected, then s itself is returned.

pure nothrow @nogc @safe bool isAlpha(dchar c);

Returns whether c is a Unicode alphabetic character (general Unicode category: Alphabetic).

pure nothrow @nogc @safe bool isMark(dchar c);

Returns whether c is a Unicode mark (general Unicode category: Mn, Me, Mc).

pure nothrow @nogc @safe bool isNumber(dchar c);

Returns whether c is a Unicode numerical character (general Unicode category: Nd, Nl, No).

pure nothrow @nogc @safe bool isPunctuation(dchar c);

Returns whether c is a Unicode punctuation character (general Unicode category: Pd, Ps, Pe, Pc, Po, Pi, Pf).

pure nothrow @nogc @safe bool isSymbol(dchar c);

Returns whether c is a Unicode symbol character (general Unicode category: Sm, Sc, Sk, So).

pure nothrow @nogc @safe bool isSpace(dchar c);

Returns whether c is a Unicode space character (general Unicode category: Zs)

Note: This doesn't include '\n', '\r', \t' and other non-space character. For commonly used less strict semantics see isWhite.

pure nothrow @nogc @safe bool isGraphical(dchar c);

Returns whether c is a Unicode graphical character (general Unicode category: L, M, N, P, S, Zs).

pure nothrow @nogc @safe bool isControl(dchar c);

Returns whether c is a Unicode control character (general Unicode category: Cc).

pure nothrow @nogc @safe bool isFormat(dchar c);

Returns whether c is a Unicode formatting character (general Unicode category: Cf).

pure nothrow @nogc @safe bool isPrivateUse(dchar c);

Returns whether c is a Unicode Private Use code point (general Unicode category: Co).

pure nothrow @nogc @safe bool isSurrogate(dchar c);

Returns whether c is a Unicode surrogate code point (general Unicode category: Cs).

pure nothrow @nogc @safe bool isSurrogateHi(dchar c);

Returns whether c is a Unicode high surrogate (lead surrogate).

pure nothrow @nogc @safe bool isSurrogateLo(dchar c);

Returns whether c is a Unicode low surrogate (trail surrogate).

pure nothrow @nogc @safe bool isNonCharacter(dchar c);

Returns whether c is a Unicode non-character i.e. a code point with no assigned abstract character. (general Unicode category: Cn)