dmd.lexer

static immutable ubyte[256] cmtable;

Do our own char maps

abstract class DiagnosticReporter;

Interface for diagnostic reporting.

abstract int errorCount();

Returns:

the number of errors that occurred during lexing or parsing.

abstract int warningCount();

Returns:

the number of warnings that occurred during lexing or parsing.

abstract int deprecationCount();

Returns:

the number of deprecations that occurred during lexing or parsing.

final void error(ref const Loc loc, const(char)* format, ...);

abstract void error(ref const Loc loc, const(char)* format, va_list args);

Reports an error message.

Parameters:

Loc loc	Location of error
const(char)* format	format string for error
...	format string arguments

final void errorSupplemental(ref const Loc loc, const(char)* format, ...);

abstract void errorSupplemental(ref const Loc loc, const(char)* format, va_list);

Reports additional details about an error message.

Parameters:

Loc loc	Location of error
const(char)* format	format string for supplemental message
...	format string arguments

final void warning(ref const Loc loc, const(char)* format, ...);

abstract void warning(ref const Loc loc, const(char)* format, va_list args);

Reports a warning message.

Parameters:

Loc loc	Location of warning
const(char)* format	format string for warning
...	format string arguments

final void warningSupplemental(ref const Loc loc, const(char)* format, ...);

abstract void warningSupplemental(ref const Loc loc, const(char)* format, va_list);

Reports additional details about a warning message.

Parameters:

Loc loc	Location of warning
const(char)* format	format string for supplemental message
...	format string arguments

final void deprecation(ref const Loc loc, const(char)* format, ...);

abstract void deprecation(ref const Loc loc, const(char)* format, va_list args);

Reports a deprecation message.

Parameters:

Loc loc	Location of the deprecation
const(char)* format	format string for the deprecation
...	format string arguments

final void deprecationSupplemental(ref const Loc loc, const(char)* format, ...);

abstract void deprecationSupplemental(ref const Loc loc, const(char)* format, va_list);

Reports additional details about a deprecation message.

Parameters:

Loc loc	Location of deprecation
const(char)* format	format string for supplemental message
...	format string arguments

class StderrDiagnosticReporter: dmd.lexer.DiagnosticReporter;

Diagnostic reporter which prints the diagnostic messages to stderr.

This is usually the default diagnostic reporter.

this(Diagnostic useDeprecated);

Initializes this object.

Parameters:

Diagnostic useDeprecated

indicates how deprecation diagnostics should be handled

class Lexer;

this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, bool doDocComment, bool commentToken, DiagnosticReporter diagnosticReporter);

Creates a Lexer for the source code base[begoffset..endoffset+1]. The last character, base[endoffset], must be null (0) or EOF (0x1A).

Parameters:

const(char)* filename	used for error messages
const(char)* base	source code, must be terminated by a null (0) or EOF (0x1A) character
size_t begoffset	starting offset into base[]
size_t endoffset	the last offset to read into base[]
bool doDocComment	handle documentation comments
bool commentToken	comments become TOK.comment's
DiagnosticReporter diagnosticReporter	the diagnostic reporter to use

final bool errors();

Returns:

true if any errors occurred during lexing or parsing.

pure nothrow @safe Token* allocateToken();

Returns:

a newly allocated Token.

pure nothrow @nogc @safe void releaseToken(Token* token);

Frees the given token by returning it to the freelist.

final TOK peekNext();

Look ahead at next token's value.

final TOK peekNext2();

Look 2 tokens ahead at value.

final void scan(Token* t);

Turn next token in buffer into a token.

final Token* peekPastParen(Token* tk);

tk is on the opening (. Look ahead and return token that is past the closing ).

final uint escapeSequence();

Parse escape sequence.

static dchar escapeSequence(ref const Loc loc, DiagnosticReporter handler, ref const(char)* sequence);

Parse the given string literal escape sequence into a single character.

Parameters:

Loc loc	the location of the current token
DiagnosticReporter handler	the diagnostic reporter object
const(char)* sequence	pointer to string with escape sequence to parse. this is a reference variable that is also used to return the position after the sequence

Returns:

the escaped sequence as a single character

final void wysiwygStringConstant(Token* result);

Lex a wysiwyg string. p must be pointing to the first character before the contents of the string literal. The character pointed to by p will be used as the terminating character (i.e. backtick or double-quote).

Parameters:

Token* result

pointer to the token that accepts the result

final TOK hexStringConstant(Token* t);

Lex hex strings: x"0A ae 34FE BD"

final void delimitedStringConstant(Token* result);

Lex a delimited string. Some examples of delimited strings are:

q"(foo(xxx))"      // "foo(xxx)"
q"[foo$(LPAREN)]"  // "foo$(LPAREN)"
q"/foo]/"          // "foo]"
q"HERE
foo
HERE"              // "foo\n"

It is assumed that p points to the opening double-quote '"'.

Parameters:

Token* result

pointer to the token that accepts the result

final void tokenStringConstant(Token* result);

Lex a token string. Some examples of token strings are:

q{ foo(xxx) }    // " foo(xxx) "
q{foo$(LPAREN)}  // "foo$(LPAREN)"
q{{foo}"}"}      // "{foo}"}""

It is assumed that p points to the opening curly-brace '{'.

Parameters:

Token* result

pointer to the token that accepts the result

final void escapeStringConstant(Token* t);

Scan a double-quoted string while building the processed string value by handling escape sequences. The result is returned in the given t token. This function assumes that p currently points to the opening double-quote of the string.

Parameters:

Token* t

the token to set the resulting string to

final TOK charConstant(Token* t);

final void stringPostfix(Token* t);

Get postfix of string literal.

final TOK number(Token* t);

Read in a number. If it's an integer, store it in tok.TKutok.Vlong. integers can be decimal, octal or hex Handle the suffixes U, UL, LU, L, etc. If it's double, store it in tok.TKutok.Vdouble.

Returns:

TKnum TKdouble,...

final TOK inreal(Token* t);

Read in characters, converting them to real.

Bugs:

Exponent overflow not detected. Too much requested precision is not detected.

final void poundLine();

parse: #line linnum [filespec] also allow __LINE__ for linnum, and __FILE__ for filespec

final uint decodeUTF();

Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.

final void getDocComment(Token* t, uint lineComment, bool newParagraph);

Parse doc comment embedded between t.ptr and p. Remove trailing blanks and tabs from lines. Replace all newlines with \n. Remove leading comment character from each line. Decide if it's a lineComment or a blockComment. Append to previous one for this token.

If newParagraph is true, an extra newline will be added between adjoining doc comments.

static const(char)* combineComments(const(char)* c1, const(char)* c2, bool newParagraph);

Combine two document comments into one, separated by an extra newline if newParagraph is true.