3. Lexical Analysis and Parsing
+
using C+
Bruno R. Preiss
B.A.Sc., M.A.Sc., Ph.D., P.Eng.
MMIV
4. Copyright c 2004 by Bruno R. Preiss.
All rights reserved. No part of this publication may be reproduced, stored in a retrieval system, or transmitted, in any form or by any means, electronic, mechanical,
photocopying, recording, or otherwise, without the prior written permission of the
author.
A
This book was prepared with L TEX and reproduced from camera-ready copy supplied by the author. The book is typeset using the Computer Modern fonts designed
by Donald E. Knuth with various additional glyphs designed by the author and implemented using
.
is a trademark of Addison Wesley Publishing Company.
TEX is a trademark of the American Mathematical Society.
UNIX is a registered trademark of AT&T Bell Laboratories.
19. 1.2. Lexis Grammar Specification
¶5
Lexis start symbol definition
1 file.
5
5
≡
This code is used in ¶54.
Lexis grammar productions 6 ≡
1 file =
2
languageDefinitionPart auxiliaryDefinitionsPart tokenDefinitionsPart
3
‘‘
4
file actions 29
5
’’.
¶6
This code is used in ¶54.
Lexis grammar productions 7 ≡
1 languageDefinitionPart =
2
‘‘
3
languageDefinitionPart case 1 actions
4
’’
5
| language colon identifier period
6
‘‘
7
languageDefinitionPart case 2 actions
8
’’
9
.
¶7
30
31
This code is used in ¶54.
Lexis grammar productions 8 ≡
1 auxiliaryDefinitionsPart =
2
| definitions colon auxiliaryDefinitionList.
¶8
This code is used in ¶54.
Lexis grammar productions 9 ≡
1 auxiliaryDefinitionList =
2
| auxiliaryDefinitionList auxiliaryDefinition.
¶9
This code is used in ¶54.
Lexis grammar productions 10 ≡
1 auxiliaryDefinition =
2
identifier equal expression period
3
‘‘
4
auxiliaryDefinition actions
5
’’.
This code is used in ¶54.
¶10
32
20. Chapter 1. Lexis Input Specifications
6
¶11
Lexis grammar productions 11 ≡
1 tokenDefinitionsPart =
2
‘‘
3
tokenDefinitionsPart case 1 actions
4
’’
5
| tokens colon tokenDefinitionList
6
‘‘
7
tokenDefinitionsPart case 2 actions
8
’’.
33
34
This code is used in ¶54.
¶12
Lexis grammar productions 12 ≡
1 tokenDefinitionList =
2
‘‘
3
tokenDefinitionList case 1 actions
4
’’
5
| tokenDefinitionList tokenDefinition
6
‘‘
7
tokenDefinitionList case 2 actions
8
’’.
This code is used in ¶54.
¶13
Lexis grammar productions 13 ≡
1 tokenDefinition =
2
equal expression period
3
‘‘
4
tokenDefinition case 1 actions 37
5
’’
6
| identifier equal expression period
7
‘‘
8
tokenDefinition case 2 actions 38
9
’’.
This code is used in ¶54.
35
36
21. 1.2. Lexis Grammar Specification
Lexis grammar productions 14 ≡
1 expression =
2
alternation
3
‘‘
4
expression case 1
5
’’
6
| circumflex alternation
7
‘‘
8
expression case 2
9
’’
10
| alternation dollar
11
‘‘
12
expression case 3
13
’’
14
| circumflex alternation
15
‘‘
16
expression case 4
17
’’.
7
¶14
actions
39
actions
40
actions
41
dollar
actions
42
This code is used in ¶54.
Lexis grammar productions 15 ≡
1 alternation =
2
alternative
3
‘‘
4
alternation case 1 actions 43
5
’’
6
| alternation verticalBar alternative
7
‘‘
8
alternation case 2 actions 44
9
’’.
¶15
This code is used in ¶54.
Lexis grammar productions 16 ≡
1 alternative =
2
‘‘
3
alternative case 1 actions
4
’’
5
| alternative item
6
‘‘
7
alternative case 2 actions
8
’’.
This code is used in ¶54.
¶16
45
46
22. Chapter 1. Lexis Input Specifications
8
¶17
Lexis grammar productions 17 ≡
1 item =
2
questionMark
3
‘‘
4
item case 1 actions 47
5
’’
6
| identifier
7
‘‘
8
item case 2 actions 48
9
’’
10
| stringT
11
‘‘
12
item case 3 actions 49
13
’’
14
| characterClass
15
‘‘
16
item case 4 actions 50
17
’’
18
| leftParenthesis expression rightParenthesis
19
‘‘
20
item case 5 actions 51
21
’’
22
| leftBracket expression rightBracket
23
‘‘
24
item case 6 actions 52
25
’’
26
| leftBrace expression rightBrace
27
‘‘
28
item case 7 actions 53
29
’’.
This code is used in ¶54.
1.2.1
¶18
ParserData Class
ParserData class declaration 18 ≡
1 class ParserData
2 {
3
ParserData field declarations 19
4
ParserData method declarations
5 };
20
This code is used in ¶22.
¶19
ParserData field declarations 19 ≡
1 protected: HashTable<string,NFA*> definitionsTable;
This code is used in ¶18.
23. 1.2. Lexis Grammar Specification
¶20
9
ParserData method declarations 20 ≡
1 protected: ParserData();
This code is used in ¶18.
ParserData class inline method definitions 21 ≡
1 inline ParserData::ParserData() :
2
definitionsTable()
3
{}
¶21
This code is used in ¶22.
File: Lexis/ParserData.h 22 ≡
1 #if !defined(_Lexis_ParserData_h)
2 #define _Lexis_ParserData_h
¶22
3
4
5
6
7
#include
#include
#include
#include
<string>
"Lexis/NFA.h"
"Toolbox/HashTable.h"
"Toolbox/String.h"
8
9
10
using namespace Toolbox;
using namespace Toolbox::String;
11
12
13
14
15
16
namespace Lexis
{
ParserData class declaration 18
ParserData class inline method definitions
}
21
17
18
#endif
1.2.2
ItemAttributes Class
ItemAttributes class declaration 23 ≡
1 class ItemAttributes
2 {
3
ItemAttributes field declarations 24
4
ItemAttributes method declarations
5 };
¶23
25
This code is used in ¶27.
ItemAttributes field declarations
1 public: NFA* nfa;
2 public: string language;
This code is used in ¶23.
24
≡
¶24
24. Chapter 1. Lexis Input Specifications
10
ItemAttributes method declarations
1 public: ItemAttributes();
25
≡
¶25
This code is used in ¶23.
¶26
ItemAttributes class inline method definitions 26 ≡
1 inline ItemAttributes::ItemAttributes() :
2
nfa(0),
3
language()
4
{}
This code is used in ¶27.
¶27
File: Lexis/ItemAttributes.h 27 ≡
1 #if !defined(_Lexis_ItemAttributes_h)
2 #define _Lexis_ItemAttributes_h
3
4
5
#include <string>
#include "Lexis/NFA.h"
6
7
8
using namespace Toolbox;
using namespace Toolbox::String;
9
10
11
12
13
14
namespace Lexis
{
ItemAttributes class declaration 23
ItemAttributes class inline method definitions
}
15
16
#endif
1.2.3
¶28
Actions
Lexis actions preamble 28 ≡
1 #include <stdint.h>
2 #include "Lexis/ParserData.h"
3 #include "Lexis/ItemAttributes.h"
4 #include "Toolbox/CharacterSet.h"
5 #include "Toolbox/EscapeSequence.h"
6
7
using Lexis::NFA;
This code is used in ¶54.
¶29
file actions 29 ≡
1 NFA* nfa = $tokenDefinitionsPart;
2 nfa->setLanguage($languageDefinitionPart);
3 $$ = nfa;
This code is used in ¶6.
26
25. 1.2. Lexis Grammar Specification
11
languageDefinitionPart case 1 actions
1 $$ = "UserDefined";
30
≡
¶30
31
≡
¶31
This code is used in ¶7.
languageDefinitionPart case 2 actions
1 $$ = $identifier.getLexeme();
This code is used in ¶7.
auxiliaryDefinition actions 32 ≡
1 HashTable<string,NFA*>::const_iterator ptr =
2
definitionsTable.find($identifier.getLexeme()), null;
3 if (ptr != null)
4
cerr << "Error: " << $identifier.getLexeme() <<
5
" multiply defined.n";
6 definitionsTable.add(
7
$identifier.getLexeme(), $expression);
¶32
This code is used in ¶10.
tokenDefinitionsPart case 1 actions
1 $$ = 0;
33
≡
¶33
34
≡
¶34
This code is used in ¶11.
tokenDefinitionsPart case 2 actions
1 $$ = $tokenDefinitionList;
This code is used in ¶11.
tokenDefinitionList case 1 actions
1 $$ = 0;
35
≡
¶35
This code is used in ¶12.
tokenDefinitionList case 2 actions 36 ≡
1 NFA* result = 0;
2 if ($tokenDefinitionList == 0)
3 {
4
result = $tokenDefinition;
5 }
6 else
7 {
8
assert($tokenDefinition != 0);
9
result = new NFA(
10
*$tokenDefinitionList | *$tokenDefinition);
11
delete $tokenDefinitionList;
12
delete $tokenDefinition;
13 }
14 $$ = result;
This code is used in ¶12.
¶36
26. Chapter 1. Lexis Input Specifications
12
¶37
tokenDefinition case 1 actions 37 ≡
1 $expression->setAction(Lexis::discard);
2 $$ = $expression;
This code is used in ¶13.
¶38
tokenDefinition case 2 actions 38 ≡
1 $expression->setAction(Lexis::accept);
2 $expression->setToken($identifier.getLexeme());
3 $$ = $expression;
This code is used in ¶13.
¶39
expression case 1 actions
1 $$ = $alternation;
39
≡
This code is used in ¶14.
¶40
expression case 2 actions 40 ≡
1 NFA nl("n");
2 NFA* result = new NFA(nl * *$alternation);
3 result->setAnchor(Lexis::head);
4 delete $alternation;
5 $$ = result;
This code is used in ¶14.
¶41
expression case 3 actions 41 ≡
1 NFA nl("n");
2 NFA* result = new NFA(*$alternation * nl);
3 result->setAnchor(Lexis::tail);
4 delete $alternation;
5 $$ = result;
This code is used in ¶14.
¶42
expression case 4 actions 42 ≡
1 NFA nl("n");
2 NFA* result = new NFA(nl * *$alternation * nl);
3 result->setAnchor(Lexis::both);
4 delete $alternation;
5 $$ = result;
This code is used in ¶14.
¶43
alternation case 1 actions
1 $$ = $alternative;
This code is used in ¶15.
43
≡
27. 1.2. Lexis Grammar Specification
¶44
13
alternation case 2 actions 44 ≡
1 NFA* result = new NFA(*$alternation | *$alternative);
2 delete $alternation;
3 delete $alternative;
4 $$ = result;
This code is used in ¶15.
alternative case 1 actions
1 $$ = new NFA();
45
≡
¶45
This code is used in ¶16.
alternative case 2 actions 46 ≡
1 NFA* result = new NFA(*$alternative * *$item);
2 delete $alternative;
3 delete $item;
4 $$ = result;
¶46
This code is used in ¶16.
item case 1 actions 47 ≡
1 CharacterSet set;
2 set += ’n’;
3 set = ~set;
4 $$ = new NFA(set);
¶47
This code is used in ¶17.
item case 2 actions 48 ≡
1 HashTable<string,NFA*>::const_iterator ptr =
2
definitionsTable.find($identifier.getLexeme()), null;
3 NFA* result = 0;
4 if (ptr == null)
5 {
6
cerr << "Error: " << $identifier.getLexeme() <<
7
" not defined.n";
8
result = new NFA();
9 }
10 else
11
result = new NFA(**ptr);
12 $$ = result;
¶48
This code is used in ¶17.
item case 3 actions 49 ≡
1 string src($stringT.getLexeme());
2
3
4
$$ = new NFA(EscapeSequence::decode(
src.substr(1, src.size()-2)));
This code is used in ¶17.
¶49
28. Chapter 1. Lexis Input Specifications
14
¶50
item case 4 actions 50 ≡
1 string src($characterClass.getLexeme());
2 bool complement = false;
3
4
5
6
7
8
9
10
11
assert(src.size() >= 2);
if (src[1U] == ’!’)
{
complement = true;
src = EscapeSequence::decode(src.substr(2, src.size()-3));
}
else
src = EscapeSequence::decode(src.substr(1, src.size()-2));
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
CharacterSet dst;
int16_t lastc = -1;
string::iterator p(src.begin()), lim(src.end());
while (p != lim)
{
char c = *p++;
if (c == ’-’ && lastc >= 0)
{
if (p != lim)
{
c = *p++;
for (uint8_t i = lastc + 1U;
i <= static_cast<uint8_t>(c); i++)
dst += i;
lastc = -1;
}
else
{
dst += ’-’;
lastc = ’-’;
}
}
else
{
dst += c;
lastc = static_cast<uint8_t>(c);
}
}
if (complement)
dst = ~dst;
$$ = new NFA(dst);
This code is used in ¶17.
¶51
item case 5 actions 51 ≡
1 $$ = $expression;
This code is used in ¶17.
29. 1.2. Lexis Grammar Specification
item case 6 actions 52 ≡
1 NFA* result = new NFA(!*$expression);
2 delete $expression;
3 $$ = result;
15
¶52
This code is used in ¶17.
item case 7 actions 53 ≡
1 NFA* result = new NFA(~*$expression);
2 delete $expression;
3 $$ = result;
¶53
This code is used in ¶17.
1.2.4
File
File: Lexis/Lexis.gtk 54 ≡
1 Language:
2
LexisGrammar.
3 Preamble:
4
‘‘
5
Lexis actions preamble 28
6
’’.
7 Terminals:
8
"lexistab.h".
9 Nonterminals:
10
Lexis non-terminal definitions 4
11 Start:
12
Lexis start symbol definition 5
13 Productions:
14
Lexis grammar productions 6–17
¶54
31. Chapter 2
State Class
2.1
Declarations
State class declaration 55 ≡
1 class State
2 {
3
State class field declarations 56
4
State class method declarations
5 };
¶55
57
This code is used in ¶71.
State class field declarations 56 ≡
1 protected: StateNumber number;
2 protected: Anchor anchor;
3 protected: Action action;
4 protected: string token;
¶56
This code is used in ¶55.
State class method declarations 57 ≡
1 public: State();
2 public: State(StateNumber, Anchor, Action, string const&);
3 public: State& operator=(State const&);
4 public: void setNumber(StateNumber);
5 public: void setAnchor(Anchor);
6 public: void setAction(Action);
7 public: void setToken(string const&);
8 public: StateNumber getNumber() const;
9 public: Anchor getAnchor() const;
10 public: Action getAction() const;
11 public: string const& getToken() const;
12 public: ostream& put(ostream&) const;
This code is used in ¶55.
¶57
32. Chapter 2. State Class
18
2.2
¶58
Definitions
State class inline method definitions
1 inline State::State() :
2
number(0),
3
anchor(none),
4
action(advance),
5
token()
6
{}
58
≡
This code is used in ¶71.
¶59
State class inline method definitions 59 ≡
1 inline State::State(StateNumber n, Anchor ancr,
2
Action actn, string const& s) :
3
number(n),
4
anchor(ancr),
5
action(actn),
6
token(s)
7
{}
This code is used in ¶71.
¶60
State class inline method definitions 60 ≡
1 inline StateNumber State::getNumber() const
2
{ return number; }
This code is used in ¶71.
¶61
State class inline method definitions 61 ≡
1 inline Anchor State::getAnchor() const
2
{ return anchor; }
This code is used in ¶71.
¶62
State class inline method definitions 62 ≡
1 inline Action State::getAction() const
2
{ return action; }
This code is used in ¶71.
¶63
State class inline method definitions 63 ≡
1 inline string const& State::getToken() const
2
{ return token; }
This code is used in ¶71.
¶64
State class inline method definitions 64 ≡
1 inline ostream& operator<<(ostream& stream, State const& state)
2
{ return state.put(stream); }
This code is used in ¶71.
33. 2.2. Definitions
State method definitions 65 ≡
1 State& State::operator=(State const& s)
2 {
3
anchor = s.anchor;
4
action = s.action;
5
token = s.token;
6
return *this;
7 }
19
¶65
This code is used in ¶72.
State method definitions 66 ≡
1 void State::setNumber(StateNumber n)
2
{ number = n; }
¶66
This code is used in ¶72.
State method definitions 67 ≡
1 void State::setAnchor(Anchor a)
2
{ anchor = a; }
¶67
This code is used in ¶72.
State method definitions 68 ≡
1 void State::setAction(Action a)
2
{ action = a; }
¶68
This code is used in ¶72.
State method definitions 69 ≡
1 void State::setToken(string const& s)
2
{ token = s; }
¶69
This code is used in ¶72.
State method definitions 70 ≡
1 ostream& State::put(ostream& s) const
2 {
3
s << "number = " << number;
4
s << ", anchor = " << anchor;
5
s << ", action = " << action;
6
return s;
7 }
This code is used in ¶72.
¶70
41. Chapter 4
NFA Class
4.1
Declarations
NFA class declaration 92 ≡
1 class NFA
2 {
3
NFA class field declarations 93
4
NFA class method declarations
5
NFA class friends 95
6 };
¶92
94
This code is used in ¶120.
NFA class field declarations 93 ≡
1 public: string language;
2 public: StateNumber const numberOfStates;
3 public: vector<NFAState> state;
This code is used in ¶92.
¶93
42. Chapter 4. NFA Class
28
NFA class method declarations 94 ≡
1 public: explicit NFA(StateNumber = 1);
2 public: explicit NFA(string const&);
3 public: explicit NFA(CharacterSet const&);
4 public: NFA(NFA const&);
5 public: ~NFA();
6 public: NFAState const& operator[](StateNumber) const;
7 public: NFAState& operator[](StateNumber);
8 public: StateNumber getNumberOfStates() const;
9 public: void setAnchor(Anchor);
10 public: void setAction(Action);
11 public: void setToken(string const&);
12 public: void setLanguage(string const&);
13 public: ostream& put(ostream&) const;
14 public: Set getEpsilonClosure(Set const&) const;
15 public: Set getMoveSet(Set const&, char) const;
16 public: Action getDominantAction(Set const&) const;
17 public: Anchor getDominantAnchor(Set const&) const;
18 public: string getDominantToken(Set const&) const;
19 public: string const& getLanguage() const;
¶94
This code is used in ¶92.
¶95
NFA class friends 95 ≡
1 friend NFA operator*(NFA
2 friend NFA operator|(NFA
3 friend NFA operator!(NFA
4 friend NFA operator~(NFA
const&, NFA const&);
const&, NFA const&);
const&);
const&);
This code is used in ¶92.
4.2
¶96
Definitions
NFA class inline method definitions 96 ≡
1 inline void NFA::setLanguage(string const& s)
2
{ language = s; }
This code is used in ¶120.
¶97
NFA class inline method definitions 97 ≡
1 inline string const& NFA::getLanguage() const
2
{ return language; }
This code is used in ¶120.
¶98
NFA class inline method definitions 98 ≡
1 inline StateNumber NFA::getNumberOfStates() const
2
{ return numberOfStates; }
This code is used in ¶120.
43. 4.2. Definitions
NFA class inline method definitions 99 ≡
1 inline ostream& operator<<(ostream& stream, NFA const& nfa)
2
{ return nfa.put(stream); }
29
¶99
This code is used in ¶120.
NFA class method definitions 100 ≡
1 NFA::NFA(StateNumber states) :
2
numberOfStates(states),
3
state(numberOfStates)
4 {
5
for (StateNumber n = 0; n < numberOfStates; ++n)
6
state[n].setNumber(n);
7 }
¶100
This code is used in ¶121.
NFA class method definitions 101 ≡
1 NFA::NFA(string const& s) :
2
numberOfStates(s.size() + 1),
3
state(numberOfStates)
4 {
5
for (StateNumber n = 0; n < numberOfStates; ++n)
6
state[n].setNumber(n);
7
for (StateNumber n = 0; n < numberOfStates - 1U; ++n)
8
state[n].connect(s[n], n + 1);
9 }
¶101
This code is used in ¶121.
NFA class method definitions 102 ≡
1 NFA::NFA(CharacterSet const& s) :
2
numberOfStates(2),
3
state(2)
4 {
5
for (StateNumber n = 0; n < numberOfStates; ++n)
6
state[n].setNumber(n);
7
state[0].connect(s, 1);
8 }
This code is used in ¶121.
¶102
44. Chapter 4. NFA Class
30
NFA class method definitions 103 ≡
1 NFA::NFA(NFA const& nfa) :
2
numberOfStates(nfa.numberOfStates),
3
state(numberOfStates)
4 {
5
for (StateNumber n = 0; n < numberOfStates; ++n)
6
{
7
state[n].setNumber(n);
8
state[n] = nfa.state[n];
9
}
10 }
This code is used in ¶121.
¶104
NFA class method definitions
1 NFA::~NFA()
2
{}
104
≡
This code is used in ¶121.
¶105
NFA class method definitions 105 ≡
1 NFAState const& NFA::operator[](StateNumber n) const
2 {
3
assert(n < numberOfStates);
4
return state[n];
5 }
This code is used in ¶121.
¶106
NFA class method definitions 106 ≡
1 NFAState& NFA::operator[](StateNumber n)
2 {
3
assert(n < numberOfStates);
4
return state[n];
5 }
This code is used in ¶121.
¶107
NFA class method definitions 107 ≡
1 void NFA::setAnchor(Anchor a)
2 {
3
assert(numberOfStates > 0);
4
state[numberOfStates - 1].setAnchor(a);
5 }
This code is used in ¶121.
¶103
45. 4.2. Definitions
¶108
31
NFA class method definitions 108 ≡
1 void NFA::setAction(Action a)
2 {
3
assert(numberOfStates > 0);
4
state[numberOfStates - 1].setAction(a);
5 }
This code is used in ¶121.
NFA class method definitions 109 ≡
1 void NFA::setToken(string const& s)
2 {
3
assert(numberOfStates > 0);
4
state[numberOfStates - 1].setToken(s);
5 }
¶109
This code is used in ¶121.
NFA class method definitions 110 ≡
1 ostream& NFA::put(ostream& s) const
2 {
3
for (StateNumber n = 0; n < numberOfStates; ++n)
4
s << state[n];
5
return s;
6 }
This code is used in ¶121.
¶110
46. Chapter 4. NFA Class
32
¶111
NFA class method definitions 111 ≡
1 Set NFA::getEpsilonClosure(Set const& inputSet) const
2 {
3
Stack<StateNumber> stack;
4
Set result(inputSet);
5
6
for (Set::const_iterator p(inputSet.begin()), null; p != null; ++p)
{
stack.push(*p);
}
7
8
9
10
11
while (!stack.isEmpty())
{
StateNumber const s = stack.pop();
if (state[s].getLabelType() == NFAState::epsilon)
{
for (uint8_t i = 0;
i < state[s].getFanout(); ++i)
{
StateNumber const next =
state[s].getNextState(i);
if (!result.contains(next))
{
result += next;
stack.push(next);
}
}
}
}
return result;
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
}
This code is used in ¶121.
47. 4.2. Definitions
NFA class method definitions 112 ≡
1 Set NFA::getMoveSet(Set const& inputSet, char c) const
2 {
3
Set result;
33
¶112
4
for (Set::const_iterator p(inputSet.begin()), null; p != null; ++p)
{
StateNumber const s = *p;
if (state[s].isTransitionLabelled(c))
{
assert(state[s].getFanout() == 1);
result += state[s].getNextState(0);
}
}
return result;
5
6
7
8
9
10
11
12
13
14
15
}
This code is used in ¶121.
NFA class method definitions 113 ≡
1 Action NFA::getDominantAction(Set const& inputSet) const
2 {
3
for (Set::const_iterator p(inputSet.begin()), null; p != null; ++p)
4
{
5
Action const a = state[*p].getAction();
6
if (a != advance)
7
return a;
8
}
9
return advance;
10 }
¶113
This code is used in ¶121.
NFA class method definitions 114 ≡
1 Anchor NFA::getDominantAnchor(Set const& inputSet) const
2 {
3
for (Set::const_iterator p(inputSet.begin()), null; p != null; ++p)
4
{
5
Action const a = state[*p].getAction();
6
if (a != advance)
7
return state[*p].getAnchor();
8
}
9
return none;
10 }
This code is used in ¶121.
¶114
48. Chapter 4. NFA Class
34
¶115
NFA class method definitions 115 ≡
1 string NFA::getDominantToken(Set const& inputSet) const
2 {
3
for (Set::const_iterator p(inputSet.begin()), null; p != null; ++p)
4
{
5
Action const a = state[*p].getAction();
6
if (a != advance)
7
return state[*p].getToken();
8
}
9
return "";
10 }
This code is used in ¶121.
¶116
NFA class method definitions 116 ≡
1 NFA operator*(NFA const& nfa0, NFA const& nfa1)
2 {
3
StateNumber const size0 = nfa0.getNumberOfStates();
4
StateNumber const size1 = nfa1.getNumberOfStates();
5
StateNumber const resultSize = size0 + size1 - 1;
6
NFA result(resultSize);
7
for (StateNumber n = 0; n < size0 - 1U; ++n)
8
result[n] = nfa0[n];
9
for (StateNumber n = 0; n < size1; ++n)
10
result[size0 - 1 + n] = nfa1[n];
11
return result;
12 }
This code is used in ¶121.
¶117
NFA class method definitions 117 ≡
1 NFA operator|(NFA const& nfa0, NFA const& nfa1)
2 {
3
StateNumber const size0 = nfa0.getNumberOfStates();
4
StateNumber const size1 = nfa1.getNumberOfStates();
5
StateNumber const resultSize = size0 + size1 + 2;
6
NFA result(resultSize);
7
result[0].connect(static_cast<StateNumber>(1), 1 + size0);
8
for (StateNumber n = 0; n < size0; ++n)
9
result[1 + n] = nfa0[n];
10
for (StateNumber n = 0; n < size1; ++n)
11
result[1 + size0 + n] = nfa1[n];
12
result[size0].connect(resultSize - 1);
13
result[size0 + size1].connect(resultSize - 1);
14
return result;
15 }
This code is used in ¶121.
51. Chapter 5
DFAState Class
5.1
Declarations
DFAState class declaration 122 ≡
1 class DFAState : public State
2 {
3
DFAState class field declarations 123
4
DFAState class method declarations 124
5 };
¶122
This code is used in ¶132.
DFAState class field declarations 123 ≡
1 private: Set const nfaStateSet;
2 private: vector<StateNumber> transition;
¶123
This code is used in ¶122.
DFAState class method declarations 124 ≡
1 public: DFAState(StateNumber, NFA const&, Set const&);
2 public: ~DFAState();
3 public: DFAState& addTransition(char, StateNumber);
4 public: StateNumber getNextState(char) const;
5 public: Set getNFAStateSet() const;
6 public: ostream& put(ostream&) const;
This code is used in ¶122.
¶124
52. Chapter 5. DFAState Class
38
5.2
Definitions
DFAState class inline method definitions 125 ≡
1 inline Set DFAState::getNFAStateSet() const
2
{ return nfaStateSet; }
This code is used in ¶132.
¶126
DFAState class inline method definitions 126 ≡
1 inline ostream& operator<<(ostream& stream, DFAState const& state)
2
{ return state.put(stream); }
This code is used in ¶132.
¶127
DFAState class method definitions 127 ≡
1 DFAState::DFAState(StateNumber n, NFA const& nfa, Set const& s) :
2
State(n, nfa.getDominantAnchor(s),
3
nfa.getDominantAction(s), nfa.getDominantToken(s)),
4
nfaStateSet(s),
5
transition(maximumCharacter)
6 {
7
for (uint8_t i = 0; i < maximumCharacter; ++i)
8
transition[i] = errorState;
9 }
This code is used in ¶133.
¶128
DFAState class method definitions
1 DFAState::~DFAState()
2
{}
128
≡
This code is used in ¶133.
¶129
DFAState class method definitions 129 ≡
1 DFAState& DFAState::addTransition(char c, StateNumber to)
2 {
3
assert(static_cast<uint8_t>(c) < maximumCharacter);
4
transition[c] = to;
5
return *this;
6 }
This code is used in ¶133.
¶130
DFAState class method definitions 130 ≡
1 StateNumber DFAState::getNextState(char c) const
2 {
3
assert(static_cast<uint8_t>(c) < maximumCharacter);
4
return transition[c];
5 }
This code is used in ¶133.
¶125
53. 5.3. Files
39
DFAState class method definitions 131 ≡
1 ostream& DFAState::put(ostream& s) const
2 {
3
s << "DFAState {";
4
State::put(s);
5
s << "n";
6
for (StateNumber n = 0; n < DFA::maximumNumberOfStates; ++n)
7
{
8
CharacterSet set;
9
for (uint8_t c = 0; c < maximumCharacter; ++c)
10
if (transition[c] == n)
11
set += c;
12
if (set != 0)
13
{
14
cout << " goto " << n;
15
cout << " on " << set << "n";
16
}
17
}
18
return s << "}n";
19 }
¶131
This code is used in ¶133.
5.3
Files
File: Lexis/DFAState.h 132 ≡
1 #if !defined(_Lexis_DFAState_h)
2 #define _Lexis_DFAState_h
¶132
3
4
5
6
7
8
#include
#include
#include
#include
#include
<iostream>
<string>
"Lexis/State.h"
"Lexis/NFA.h"
"Toolbox/Set.h"
9
10
using namespace Toolbox;
11
12
13
14
15
16
namespace Lexis
{
DFAState class declaration 122
DFAState class inline method definitions
}
17
18
#endif
125,126
55. Chapter 6
DFA State
6.1
Declarations
DFA class declaration 134 ≡
1 class DFA
2 {
3
DFA constants 135
4
DFA class field declarations 136
5
DFA class method declarations 137,138
6 };
¶134
This code is used in ¶152.
DFA constants 135 ≡
1 public: enum Constants
2 {
3
maximumNumberOfStates = errorState - 1
4 };
¶135
This code is used in ¶134.
DFA class field declarations 136 ≡
1 private: StateNumber numberOfStates;
2 private: vector<DFAState*> state;
3 private: string language;
¶136
This code is used in ¶134.
DFA class method declarations 137 ≡
1 private: StateNumber getStateNumber(Set const&) const;
2 private: StateNumber initializeGroups(
3
vector<Set>&, vector<StateNumber>&) const;
4 private: DFA& rebuild(
5
StateNumber, vector<Set> const&, vector<StateNumber> const&);
This code is used in ¶134.
¶137
56. Chapter 6. DFA State
42
DFA class method declarations 138 ≡
1 public: explicit DFA(NFA const&);
2 public: ~DFA();
3 public: void minimize();
4 public: StateNumber getNumberOfStates() const;
5 public: DFAState const& operator[](StateNumber) const;
6 public: bool isColumnEquivalent(char, char) const;
7 public: bool isRowEquivalent(StateNumber, StateNumber) const;
8 public: string const& getLanguage() const;
9 public: ostream& put(ostream&) const;
¶138
This code is used in ¶134.
6.2
¶139
Definitions
DFA class inline method definitions 139 ≡
1 inline StateNumber DFA::getNumberOfStates() const
2
{ return numberOfStates; }
This code is used in ¶152.
¶140
DFA class inline method definitions 140 ≡
1 inline string const& DFA::getLanguage() const
2
{ return language; }
This code is used in ¶152.
¶141
DFA class inline method definitions 141 ≡
1 inline ostream& operator<<(ostream& stream, DFA const& dfa)
2
{ return dfa.put(stream); }
This code is used in ¶152.
¶142
DFA class method definitions 142 ≡
1 StateNumber DFA::getStateNumber(Set const& s) const
2 {
3
for (StateNumber i = 0; i < numberOfStates; ++i)
4
if (state[i]->getNFAStateSet() == s)
5
return i;
6
return errorState;
7 }
This code is used in ¶153.
57. 6.2. Definitions
43
DFA class method definitions 143 ≡
1 DFA::DFA(NFA const& nfa) :
2
numberOfStates(0),
3
state(maximumNumberOfStates),
4
language(nfa.getLanguage())
5 {
6
Set nfaStates;
7
nfaStates += 0;
8
nfaStates = nfa.getEpsilonClosure(nfaStates);
9
state[0] = new DFAState(0, nfa, nfaStates);
10
11
numberOfStates = 1;
12
13
for (StateNumber i = 0; i < numberOfStates; ++i)
{
cerr << ".";
DFAState* const current = state[i];
assert(current != 0);
14
15
16
17
18
19
for (uint8_t c = 0; c < maximumCharacter; ++c)
{
Set moveSet;
moveSet = nfa.getMoveSet(current->getNFAStateSet(), c);
moveSet = nfa.getEpsilonClosure(moveSet);
20
21
22
23
24
25
StateNumber nextState = errorState;
if (moveSet != 0)
{
nextState = getStateNumber(moveSet);
if (nextState == errorState)
{
assert(numberOfStates < maximumNumberOfStates);
state[numberOfStates] =
new DFAState(numberOfStates, nfa, moveSet);
nextState = numberOfStates;
numberOfStates += 1;
}
current->addTransition(c, nextState);
}
26
27
28
29
30
31
32
33
34
35
36
37
38
39
}
40
}
41
42
}
This code is used in ¶153.
¶143
58. Chapter 6. DFA State
44
¶144
DFA class method definitions 144 ≡
1 DFA::~DFA()
2 {
3
for (StateNumber i = 0; i < numberOfStates; ++i)
4
delete state[i];
5 }
This code is used in ¶153.
¶145
DFA class method definitions 145 ≡
1 DFAState const& DFA::operator[](StateNumber s) const
2 {
3
assert(s < numberOfStates);
4
return *state[s];
5 }
This code is used in ¶153.
¶146
DFA class method definitions 146 ≡
1 ostream& DFA::put(ostream& s) const
2 {
3
for (StateNumber i = 0; i < numberOfStates; ++i)
4
cout << *state[i];
5
return s;
6 }
This code is used in ¶153.
77. Chapter 8
Lexical Analyzer Classes
8.1
8.1.1
Common Types and Constants
Type Declarations
Lexis type declarations 175 ≡
1 typedef uint16_t StateNumber;
¶175
This code is used in ¶235.
Lexis type declarations 176 ≡
1 typedef uint8_t TableEntry;
¶176
This code is used in ¶235.
8.1.2
Constants
Lexis constants 177 ≡
1 StateNumber const errorState = 255;
¶177
This code is used in ¶235.
Lexis constants 178 ≡
1 TableEntry const maximumCharacter = 128;
This code is used in ¶235.
¶178
78. Chapter 8. Lexical Analyzer Classes
64
8.1.3
Anchor Enumeration
Anchor enumeration declaration
1 enum Anchor
2 {
3
none = 0,
4
head = 1,
5
tail = 2,
6
both = 3
7 };
179
≡
This code is used in ¶235.
¶180
Anchor enumeration inline method definitions 180 ≡
1 inline ostream& operator<<(ostream& s, Anchor a)
2 {
3
switch (a)
4
{
5
case none: s << "none"; break;
6
case head: s << "head"; break;
7
case tail: s << "tail"; break;
8
case both: s << "both"; break;
9
}
10
return s;
11 }
This code is used in ¶235.
8.1.4
¶181
Action Enumeration
Action enumeration declaration 181 ≡
1 enum Action
2 {
3
advance = 0,
4
accept = 1,
5
discard = 3 // We want (discard & accept) != 0.
6 };
This code is used in ¶235.
¶182
Action enumeration inline method definitions 182 ≡
1 inline ostream& operator<<(ostream& s, Action a)
2 {
3
switch (a)
4
{
5
case advance: s << "advance"; break;
6
case accept: s << "accept"; break;
7
case discard: s << "discard"; break;
8
}
9
return s;
10 }
This code is used in ¶235.
¶179
79. 8.2. Input Class Template
65
8.2
8.2.1
Input Class Template
Declarations
Input class template declaration 183 ≡
1 template<typename T>
2 class Input
3 {
4
Input class template type declarations 184
5
Input constants 185
6
Input class template field declarations 186
7
Input class template method declarations 187,188
8 };
¶183
This code is used in ¶235.
Input class template type declarations
1 public: typedef T Token;
184
≡
¶184
This code is used in ¶183.
Input constants 185 ≡
1 private: enum Constants
2 {
3
maximumLexemeLength = 1024,
4
bufferSize = 2 * maximumLexemeLength
5 };
¶185
This code is used in ¶183.
Input class template field declarations 186 ≡
1 private: istream& inputStream;
2 private: char buffer[bufferSize];
3 private: char* nextInput;
4 private: char* endOfInput;
5 private: char* const endOfBuffer;
6 private: char* startOfLexeme;
7 private: char* endOfLexeme;
8 private: bool endOfFileRead;
9 private: uint16_t currentLineNumber;
10 private: uint16_t startMarkLineNumber;
11 private: uint16_t endMarkLineNumber;
¶186
This code is used in ¶183.
Input class template method declarations 187 ≡
1 private: void checkAssertions() const;
2 private: int16_t advance();
This code is used in ¶183.
¶187
82. Chapter 8. Lexical Analyzer Classes
68
¶193
Input class template method definitions 193 ≡
1 template <typename T>
2 Input<T>::operator bool()
3
{ return advance() != EOF; }
This code is used in ¶236.
¶194
Input class template method definitions 194 ≡
1 template <typename T>
2 char Input<T>::operator*()
3 {
4
int16_t const c = advance();
5
if (c != EOF)
6
return c;
7
else
8
return 0;
9 }
This code is used in ¶236.
¶195
Input class template method definitions 195 ≡
1 template <typename T>
2 Input<T>& Input<T>::operator++()
3 {
4
if (nextInput == endOfInput && !endOfFileRead)
5
advance();
6
if (nextInput < endOfInput)
7
{
8
if (*nextInput == ’n’)
9
currentLineNumber += 1;
10
nextInput += 1;
11
if (nextInput - startOfLexeme > maximumLexemeLength)
12
{
13
if (*startOfLexeme == ’n’)
14
startMarkLineNumber += 1;
15
startOfLexeme += 1;
16
if (endOfLexeme < startOfLexeme)
17
{
18
if (*endOfLexeme == ’n’)
19
endMarkLineNumber += 1;
20
endOfLexeme += 1;
21
}
22
}
23
}
24
else
25
assert(endOfFileRead);
26
checkAssertions();
27
return *this;
28 }
This code is used in ¶236.
83. 8.2. Input Class Template
Input class template method definitions 196 ≡
1 template <typename T>
2 Input<T>& Input<T>::operator--()
3 {
4
if (nextInput > startOfLexeme)
5
{
6
nextInput -= 1;
7
if (*nextInput == ’n’)
8
currentLineNumber -= 1;
9
if (endOfLexeme > nextInput)
10
{
11
endOfLexeme = nextInput;
12
endMarkLineNumber = currentLineNumber;
13
}
14
}
15
checkAssertions();
16
return *this;
17 }
69
¶196
This code is used in ¶236.
Input class template method definitions 197 ≡
1 template <typename T>
2 void Input<T>::markStart()
3 {
4
startOfLexeme = nextInput;
5
endOfLexeme = nextInput;
6
startMarkLineNumber = currentLineNumber;
7
endMarkLineNumber = currentLineNumber;
8
checkAssertions();
9 }
¶197
This code is used in ¶236.
Input class template method definitions 198 ≡
1 template <typename T>
2 void Input<T>::markEnd()
3 {
4
endOfLexeme = nextInput;
5
endMarkLineNumber = currentLineNumber;
6
checkAssertions();
7 }
This code is used in ¶236.
¶198
84. Chapter 8. Lexical Analyzer Classes
70
¶199
Input class template method definitions 199 ≡
1 template <typename T>
2 void Input<T>::returnToStartMark()
3 {
4
nextInput = startOfLexeme;
5
endOfLexeme = startOfLexeme;
6
currentLineNumber = startMarkLineNumber;
7
checkAssertions();
8 }
This code is used in ¶236.
¶200
Input class template method definitions 200 ≡
1 template <typename T>
2 void Input<T>::returnToEndMark()
3 {
4
nextInput = endOfLexeme;
5
currentLineNumber = endMarkLineNumber;
6
checkAssertions();
7 }
This code is used in ¶236.
¶201
Input class template method definitions 201 ≡
1 template <typename T>
2 void Input<T>::advanceStartMark()
3 {
4
if (startOfLexeme < endOfLexeme)
5
if (*startOfLexeme == ’n’)
6
startMarkLineNumber += 1;
7
startOfLexeme += 1;
8
checkAssertions();
9 }
This code is used in ¶236.
¶202
Input class template method definitions 202 ≡
1 template <typename T>
2 string Input<T>::getLexeme() const
3 {
4
string result(startOfLexeme, endOfLexeme - startOfLexeme);
5
checkAssertions();
6
return result;
7 }
This code is used in ¶236.
85. 8.3. Token Class Template
71
8.3
8.3.1
Token Class Template
Declarations
Token class template declaration 203 ≡
1 template <typename LT>
2 class Token
3 {
4
Token class template type declarations 204
5
Token class template field declarations 205
6
Token class template method declarations 206
7 };
¶203
This code is used in ¶235.
Token class template type declarations 204 ≡
1 public: typedef LT Tables;
2 public: typedef TokenStream<Token> Stream;
3 public: typedef typename LT::TokenType Type;
¶204
This code is used in ¶203.
Token class template field declarations 205 ≡
1 private: Type type;
2 private: string lexeme;
3 private: uint16_t lineNumber;
¶205
This code is used in ¶203.
Token class template method declarations 206 ≡
1 public: Token();
2 public: Token(Type, string const&, uint16_t = 0);
3 public: Token(Token const&);
4 public: Token& operator=(Token const&);
5 public: Type getType() const;
6 public: string const& getLexeme() const;
7 public: uint16_t getLineNumber() const;
8 public: string getName() const;
9 public: ostream& put(ostream& s) const;
This code is used in ¶203.
¶206
86. Chapter 8. Lexical Analyzer Classes
72
8.3.2
Definitions
Token class template inline method definitions 207 ≡
1 template <typename LT>
2 inline Token<LT>::Type Token<LT>::getType() const
3
{ return type; }
This code is used in ¶235.
¶208
Token class template inline method definitions 208 ≡
1 template <typename LT>
2 inline string const& Token<LT>::getLexeme() const
3
{ return lexeme; }
This code is used in ¶235.
¶209
Token class template inline method definitions 209 ≡
1 template <typename LT>
2 inline uint16_t Token<LT>::getLineNumber() const
3
{ return lineNumber; }
This code is used in ¶235.
¶210
Token class template inline method definitions 210 ≡
1 template <typename LT>
2 inline ostream& operator<<(ostream& stream, Token<LT> const& token)
3
{ return token.put(stream); }
This code is used in ¶235.
¶211
Token class template method definitions
1 template <typename LT>
2 Token<LT>::Token() :
3
type(LT::end_of_file),
4
lexeme(""),
5
lineNumber(0)
6
{}
211
≡
This code is used in ¶236.
¶212
Token class template method definitions 212 ≡
1 template <typename LT>
2 Token<LT>::Token(Token<LT>::Type t, string const& s, uint16_t l) :
3
type(t),
4
lexeme(s),
5
lineNumber(l)
6
{ assert(0 <= type && type < LT::numberOfTokens); }
This code is used in ¶236.
¶207
87. 8.3. Token Class Template
Token class template method definitions 213 ≡
1 template <typename LT>
2 Token<LT>::Token(Token<LT> const& t) :
3
type(t.type),
4
lexeme(t.lexeme),
5
lineNumber(t.lineNumber)
6
{ assert(0 <= type && type < LT::numberOfTokens); }
73
¶213
This code is used in ¶236.
Token class template method definitions 214 ≡
1 template <typename LT>
2 Token<LT>& Token<LT>::operator=(Token<LT> const& t)
3 {
4
type = t.type;
5
lexeme = t.lexeme;
6
lineNumber = t.lineNumber;
7
return *this;
8 }
¶214
This code is used in ¶236.
Token class template method definitions 215 ≡
1 template <typename LT>
2 string Token<LT>::getName() const
3 {
4
uint8_t i;
5
for (i = 0; i < LT::numberOfTokens; ++i)
6
if (LT::nameTable[i].type == type)
7
break;
8
assert(i < LT::numberOfTokens);
9
return LT::nameTable[i].name;
10 }
¶215
This code is used in ¶236.
Token class template method definitions 216 ≡
1 template <typename LT>
2 ostream& Token<LT>::put(ostream& s) const
3 {
4
s << "Token {" << getName();
5
s << ", "" << EscapeSequence::encode(lexeme, ’"’) << """;
6
return s << ", " << lineNumber << "}";
7 }
This code is used in ¶236.
¶216
88. Chapter 8. Lexical Analyzer Classes
74
8.4
8.4.1
¶217
TokenStream Class Template
Declarations
TokenStream class template forward declaration
1 template <typename T>
2 class TokenStream;
217
≡
This code is used in ¶235.
¶218
TokenStream class template declaration 218 ≡
1 template <typename T>
2 class TokenStream
3 {
4
TokenStream class template type declarations 219
5
TokenStream class template field declarations 220
6
TokenStream class template method declarations 221,222
7 };
This code is used in ¶235.
¶219
TokenStream class template type declarations 219 ≡
1 public: typedef T Token;
2 public: typedef typename T::Tables Tables;
This code is used in ¶218.
¶220
TokenStream class template field declarations
1 private: Input<Token> inputStream;
220
≡
This code is used in ¶218.
¶221
TokenStream class template method declarations 221 ≡
1 private: TableEntry getTransition(TableEntry, char) const;
2 private: bool isAcceptState(TableEntry) const;
3 private: bool isAcceptOrDiscardState(TableEntry) const;
4 private: bool isTailAnchored(TableEntry) const;
5 private: bool isHeadAnchored(TableEntry) const;
6 private: typename Token::Type getType(TableEntry) const;
This code is used in ¶218.
¶222
TokenStream class template method declarations 222 ≡
1 public: TokenStream(istream&);
2 public: TokenStream& operator>>(Token&);
This code is used in ¶218.
89. 8.4. TokenStream Class Template
8.4.2
¶223
75
Definitions
TokenStream class template inline method definitions 223 ≡
1 template <typename T>
2 inline TokenStream<T>::TokenStream(istream& s) :
3
inputStream(s)
4
{}
This code is used in ¶235.
TokenStream class template method definitions 224 ≡
¶224
1 template <typename T>
2 TableEntry TokenStream<T>::getTransition(TableEntry state, char c) const
3 {
4
assert(state < Tables::numberOfStates);
5
if (static_cast<uint8_t>(c) < maximumCharacter)
6
return Tables::transitionTable[
7
Tables::rowMap[state] * Tables::numberOfColumns
8
+ Tables::columnMap[c]];
9
else
10
return errorState;
11 }
This code is used in ¶236.
TokenStream class template method definitions 225 ≡
1 template <typename T>
2 bool TokenStream<T>::isAcceptState(TableEntry state) const
3 {
4
assert(state < Tables::numberOfStates);
5
return Tables::actionTable[state] == accept;
6 }
¶225
This code is used in ¶236.
TokenStream class template method definitions 226 ≡
1 template <typename T>
2 bool TokenStream<T>::isAcceptOrDiscardState(TableEntry state) const
3 {
4
assert(state < Tables::numberOfStates);
5
return (Tables::actionTable[state] & accept) != 0;
6 }
¶226
This code is used in ¶236.
TokenStream class template method definitions 227 ≡
1 template <typename T>
2 bool TokenStream<T>::isTailAnchored(TableEntry state) const
3 {
4
assert(state < Tables::numberOfStates);
5
return (Tables::anchorTable[state] & tail) != 0;
6 }
This code is used in ¶236.
¶227
90. Chapter 8. Lexical Analyzer Classes
76
¶228
TokenStream class template method definitions 228 ≡
1 template <typename T>
2 bool TokenStream<T>::isHeadAnchored(TableEntry state) const
3 {
4
assert(state < Tables::numberOfStates);
5
return (Tables::anchorTable[state] & head) != 0;
6 }
This code is used in ¶236.
¶229
TokenStream class template method definitions 229 ≡
1 template <typename T>
2 typename T::Type TokenStream<T>::getType(TableEntry state) const
3 {
4
assert(state < Tables::numberOfStates);
5
return Tables::tokenTable[state];
6 }
This code is used in ¶236.
91. 8.4. TokenStream Class Template
TokenStream class template method definitions 230 ≡
1 template <typename T>
2 TokenStream<T>& TokenStream<T>::operator>>(T& result)
3 {
4
TableEntry currentState = 0;
5
TableEntry nextState = 0;
6
TableEntry acceptState = 0;
7
inputStream.markStart();
8
for (;;)
9
{
10
get the next character and compute nextState 231
11
currentState = nextState;
12
if (currentState != errorState)
13
{
14
normal transition processing 232
15
}
16
else
17
{
18
if (acceptState != 0)
19
{
20
return the last acceptable token 233
21
}
22
else
23
{
24
lexical error recovery 234
25
}
26
currentState = 0;
27
acceptState = 0;
28
inputStream.markStart();
29
}
30
}
31 }
77
¶230
This code is used in ¶236.
get the next character and compute nextState 231 ≡
1 if (inputStream)
2 {
3
char const nextChar = *inputStream;
4
++inputStream;
5
nextState = getTransition(currentState, nextChar);
6 }
7 else if (acceptState != 0)
8
nextState = errorState;
9 else
10 {
11
result = T(Tables::end_of_file, "",
12
inputStream.getLineNumber());
13
return *this;
14 }
This code is used in ¶230.
¶231
92. Chapter 8. Lexical Analyzer Classes
78
¶232
normal transition processing 232 ≡
1 if(isAcceptOrDiscardState(currentState))
2 {
3
acceptState = currentState;
4
inputStream.markEnd();
5 }
This code is used in ¶230.
¶233
return the last acceptable token 233 ≡
1 inputStream.returnToEndMark();
2 if (isTailAnchored(acceptState))
3
--inputStream;
4 if (isHeadAnchored(acceptState))
5
inputStream.advanceStartMark();
6 if (isAcceptState(acceptState))
7 {
8
result = T(getType(acceptState),
9
inputStream.getLexeme(),
10
inputStream.getLineNumber());
11
return *this;
12 }
This code is used in ¶230.
¶234
lexical error recovery 234 ≡
1 inputStream.returnToStartMark();
2 char const nextChar = *inputStream;
3 cerr << "Lexis: Input error on line " <<
4
inputStream.getLineNumber() << ".";
5 cerr << " Input character ’" <<
6
EscapeSequence::encode(nextChar, ’’’)
7
<< "’ ignored.n";
8 ++inputStream;
This code is used in ¶230.