'\"
'\" Generated from file 'pt_peg_introduction\&.man' by tcllib/doctools with format 'nroff'
'\" Copyright (c) 2009 Andreas Kupries <andreas_kupries@users\&.sourceforge\&.net>
'\"
.TH "pt::pegrammar" 3tcl 1 tcllib "Parser Tools"
.\" The -*- nroff -*- definitions below are for supplemental macros used
.\" in Tcl/Tk manual entries.
.\"
.\" .AP type name in/out ?indent?
.\"	Start paragraph describing an argument to a library procedure.
.\"	type is type of argument (int, etc.), in/out is either "in", "out",
.\"	or "in/out" to describe whether procedure reads or modifies arg,
.\"	and indent is equivalent to second arg of .IP (shouldn't ever be
.\"	needed;  use .AS below instead)
.\"
.\" .AS ?type? ?name?
.\"	Give maximum sizes of arguments for setting tab stops.  Type and
.\"	name are examples of largest possible arguments that will be passed
.\"	to .AP later.  If args are omitted, default tab stops are used.
.\"
.\" .BS
.\"	Start box enclosure.  From here until next .BE, everything will be
.\"	enclosed in one large box.
.\"
.\" .BE
.\"	End of box enclosure.
.\"
.\" .CS
.\"	Begin code excerpt.
.\"
.\" .CE
.\"	End code excerpt.
.\"
.\" .VS ?version? ?br?
.\"	Begin vertical sidebar, for use in marking newly-changed parts
.\"	of man pages.  The first argument is ignored and used for recording
.\"	the version when the .VS was added, so that the sidebars can be
.\"	found and removed when they reach a certain age.  If another argument
.\"	is present, then a line break is forced before starting the sidebar.
.\"
.\" .VE
.\"	End of vertical sidebar.
.\"
.\" .DS
.\"	Begin an indented unfilled display.
.\"
.\" .DE
.\"	End of indented unfilled display.
.\"
.\" .SO ?manpage?
.\"	Start of list of standard options for a Tk widget. The manpage
.\"	argument defines where to look up the standard options; if
.\"	omitted, defaults to "options". The options follow on successive
.\"	lines, in three columns separated by tabs.
.\"
.\" .SE
.\"	End of list of standard options for a Tk widget.
.\"
.\" .OP cmdName dbName dbClass
.\"	Start of description of a specific option.  cmdName gives the
.\"	option's name as specified in the class command, dbName gives
.\"	the option's name in the option database, and dbClass gives
.\"	the option's class in the option database.
.\"
.\" .UL arg1 arg2
.\"	Print arg1 underlined, then print arg2 normally.
.\"
.\" .QW arg1 ?arg2?
.\"	Print arg1 in quotes, then arg2 normally (for trailing punctuation).
.\"
.\" .PQ arg1 ?arg2?
.\"	Print an open parenthesis, arg1 in quotes, then arg2 normally
.\"	(for trailing punctuation) and then a closing parenthesis.
.\"
.\"	# Set up traps and other miscellaneous stuff for Tcl/Tk man pages.
.if t .wh -1.3i ^B
.nr ^l \n(.l
.ad b
.\"	# Start an argument description
.de AP
.ie !"\\$4"" .TP \\$4
.el \{\
.   ie !"\\$2"" .TP \\n()Cu
.   el          .TP 15
.\}
.ta \\n()Au \\n()Bu
.ie !"\\$3"" \{\
\&\\$1 \\fI\\$2\\fP (\\$3)
.\".b
.\}
.el \{\
.br
.ie !"\\$2"" \{\
\&\\$1	\\fI\\$2\\fP
.\}
.el \{\
\&\\fI\\$1\\fP
.\}
.\}
..
.\"	# define tabbing values for .AP
.de AS
.nr )A 10n
.if !"\\$1"" .nr )A \\w'\\$1'u+3n
.nr )B \\n()Au+15n
.\"
.if !"\\$2"" .nr )B \\w'\\$2'u+\\n()Au+3n
.nr )C \\n()Bu+\\w'(in/out)'u+2n
..
.AS Tcl_Interp Tcl_CreateInterp in/out
.\"	# BS - start boxed text
.\"	# ^y = starting y location
.\"	# ^b = 1
.de BS
.br
.mk ^y
.nr ^b 1u
.if n .nf
.if n .ti 0
.if n \l'\\n(.lu\(ul'
.if n .fi
..
.\"	# BE - end boxed text (draw box now)
.de BE
.nf
.ti 0
.mk ^t
.ie n \l'\\n(^lu\(ul'
.el \{\
.\"	Draw four-sided box normally, but don't draw top of
.\"	box if the box started on an earlier page.
.ie !\\n(^b-1 \{\
\h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul'
.\}
.el \}\
\h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul'
.\}
.\}
.fi
.br
.nr ^b 0
..
.\"	# VS - start vertical sidebar
.\"	# ^Y = starting y location
.\"	# ^v = 1 (for troff;  for nroff this doesn't matter)
.de VS
.if !"\\$2"" .br
.mk ^Y
.ie n 'mc \s12\(br\s0
.el .nr ^v 1u
..
.\"	# VE - end of vertical sidebar
.de VE
.ie n 'mc
.el \{\
.ev 2
.nf
.ti 0
.mk ^t
\h'|\\n(^lu+3n'\L'|\\n(^Yu-1v\(bv'\v'\\n(^tu+1v-\\n(^Yu'\h'-|\\n(^lu+3n'
.sp
-1
.fi
.ev
.\}
.nr ^v 0
..
.\"	# Special macro to handle page bottom:  finish off current
.\"	# box/sidebar if in box/sidebar mode, then invoked standard
.\"	# page bottom macro.
.de ^B
.ev 2
'ti 0
'nf
.mk ^t
.if \\n(^b \{\
.\"	Draw three-sided box if this is the box's first page,
.\"	draw two sides but no top otherwise.
.ie !\\n(^b-1 \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c
.el \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c
.\}
.if \\n(^v \{\
.nr ^x \\n(^tu+1v-\\n(^Yu
\kx\h'-\\nxu'\h'|\\n(^lu+3n'\ky\L'-\\n(^xu'\v'\\n(^xu'\h'|0u'\c
.\}
.bp
'fi
.ev
.if \\n(^b \{\
.mk ^y
.nr ^b 2
.\}
.if \\n(^v \{\
.mk ^Y
.\}
..
.\"	# DS - begin display
.de DS
.RS
.nf
.sp
..
.\"	# DE - end display
.de DE
.fi
.RE
.sp
..
.\"	# SO - start of list of standard options
.de SO
'ie '\\$1'' .ds So \\fBoptions\\fR
'el .ds So \\fB\\$1\\fR
.SH "STANDARD OPTIONS"
.LP
.nf
.ta 5.5c 11c
.ft B
..
.\"	# SE - end of list of standard options
.de SE
.fi
.ft R
.LP
See the \\*(So manual entry for details on the standard options.
..
.\"	# OP - start of full description for a single option
.de OP
.LP
.nf
.ta 4c
Command-Line Name:	\\fB\\$1\\fR
Database Name:	\\fB\\$2\\fR
Database Class:	\\fB\\$3\\fR
.fi
.IP
..
.\"	# CS - begin code excerpt
.de CS
.RS
.nf
.ta .25i .5i .75i 1i
..
.\"	# CE - end code excerpt
.de CE
.fi
.RE
..
.\"	# UL - underline word
.de UL
\\$1\l'|0\(ul'\\$2
..
.\"	# QW - apply quotation marks to word
.de QW
.ie '\\*(lq'"' ``\\$1''\\$2
.\"" fix emacs highlighting
.el \\*(lq\\$1\\*(rq\\$2
..
.\"	# PQ - apply parens and quotation marks to word
.de PQ
.ie '\\*(lq'"' (``\\$1''\\$2)\\$3
.\"" fix emacs highlighting
.el (\\*(lq\\$1\\*(rq\\$2)\\$3
..
.\"	# QR - quoted range
.de QR
.ie '\\*(lq'"' ``\\$1''\\-``\\$2''\\$3
.\"" fix emacs highlighting
.el \\*(lq\\$1\\*(rq\\-\\*(lq\\$2\\*(rq\\$3
..
.\"	# MT - "empty" string
.de MT
.QW ""
..
.BS
.SH NAME
pt::pegrammar \- Introduction to Parsing Expression Grammars
.SH SYNOPSIS
package require \fBTcl  8\&.5\fR
.sp
.BE
.SH DESCRIPTION
.PP
Are you lost ?
Do you have trouble understanding this document ?
In that case please read the overview provided by the
\fIIntroduction to Parser Tools\fR\&. This document is the
entrypoint to the whole system the current package is a part of\&.
.PP
Welcome to the introduction to \fIParsing Expression Grammar\fRs
(short: \fIPEG\fR), the formalism used by the Parser Tools\&.
It is assumed that the reader has a basic knowledge of parsing theory,
i\&.e\&. \fIContext-Free Grammars\fR (short: \fICFG\fR),
\fIlanguages\fR, and associated terms like \fILL(k)\fR,
\fILR(k)\fR, \fIterminal\fR and \fInonterminal\fR \fIsymbols\fR,
etc\&.
We do not intend to recapitulate such basic definitions or terms like
\fIuseful\fR, \fIreachable\fR, (left/right) \fIrecursive\fR,
\fInullable\fR, first/last/follow sets, etc\&.
Please see the \fBReferences\fR at the end instead if you are in
need of places and books which provide such background information\&.
.PP
PEGs are formally very similar to CFGs, with terminal and nonterminal
symbols, start symbol, and rules defining the structure of each
nonterminal symbol\&.
The main difference lies in the choice(sic!) of \fIchoice\fR
operators\&. Where CFGs use an \fIunordered choice\fR to represent
alternatives PEGs use \fIprioritized choice\fR\&. Which is fancy way
of saying that a parser has to try the first alternative first and can
try the other alternatives if only if it fails for the first, and so
on\&.
.PP
On the CFG side this gives rise to LL(k) and LR(k) for making the
choice \fIdeterministic\fR with a bounded \fIlookahead\fR of k
terminal symbols, where LL is in essence \fItopdown\fR aka
\fIrecursive descent\fR parsing, and LR \fIbottomup\fR aka
\fIshift reduce\fR parsing\&.
.PP
On the PEG side we can parse input with recursive descent and
\fIbacktracking\fR of failed choices, the latter of which amounts to
unlimited lookahead\&.
By additionally recording the success or failure of nonterminals at
the specific locations they were tried at and reusing this information
after backtracking we can avoid the exponential blowup of running time
usually associated with backtracking and keep the parsing linear\&. The
memory requirements are of course higher due to this cache, as we are
trading space for time\&.
.PP
This is the basic concept behind \fIpackrat parsers\fR\&.
.PP
A limitation pure PEGs share with LL(k) CFGs is that
\fIleft-recursive\fR grammars cannot be parsed, with the associated
recursive descent parser entering an infinite recursion\&.
This limitation is usually overcome by extending pure PEGs with
explicit operators to specify repetition, zero or more, and one or
more, or, formally spoken, for the \fIkleene closure\fR and
\fIpositive kleene closure\fR\&.
This is what the Parser Tools are doing\&.
.PP
Another extension, specific to Parser Tools, is a set of operators
which map more or less directly to various character classes built
into Tcl, i\&.e\&. the classes reachable via \fBstring is\fR\&.
.PP
The remainder of this document consists of the formal definition of
PEGs for the mathematically inclined, and an appendix listing
references to places with more information on PEGs specifically, and
parsing in general\&.
.SH "FORMAL DEFINITION"
.PP
For the mathematically inclined, a Parsing Expression Grammar is a
4-tuple (VN,VT,R,eS) where
.IP \(bu
VN is a set of \fInonterminal symbols\fR,
.IP \(bu
VT is a set of \fIterminal symbols\fR,
.IP \(bu
R is a finite set of rules, where each rule is a pair (A,e), A in VN,
and \fIe\fR a \fIparsing expression\fR\&.
.IP \(bu
eS is a parsing expression, the \fIstart expression\fR\&.
.PP
.PP
Further constraints are
.IP \(bu
The intersection of VN and VT is empty\&.
.IP \(bu
For all A in VT exists exactly one pair (A,e) in R\&. In other words, R
is a function from nonterminal symbols to parsing expressions\&.
.PP
.PP
Parsing expressions are inductively defined via
.IP \(bu
The empty string (epsilon) is a parsing expression\&.
.IP \(bu
A terminal symbol \fIa\fR is a parsing expression\&.
.IP \(bu
A nonterminal symbol \fIA\fR is a parsing expression\&.
.IP \(bu
\fIe1\fR\fIe2\fR is a parsing expression for parsing expressions
\fIe1\fR and \fI2\fR\&. This is called \fIsequence\fR\&.
.IP \(bu
\fIe1\fR/\fIe2\fR is a parsing expression for parsing expressions
\fIe1\fR and \fI2\fR\&. This is called \fIordered choice\fR\&.
.IP \(bu
\fIe\fR* is a parsing expression for parsing expression
\fIe\fR\&. This is called \fIzero-or-more repetitions\fR, also known
as \fIkleene closure\fR\&.
.IP \(bu
\fIe\fR+ is a parsing expression for parsing expression
\fIe\fR\&. This is called \fIone-or-more repetitions\fR, also known
as \fIpositive kleene closure\fR\&.
.IP \(bu
!\fIe\fR is a parsing expression for parsing expression
\fIe1\fR\&. This is called a \fInot lookahead predicate\fR\&.
.IP \(bu
&\fIe\fR is a parsing expression for parsing expression
\fIe1\fR\&. This is called an \fIand lookahead predicate\fR\&.
.PP
.PP
.PP
PEGs are used to define a grammatical structure for streams of symbols
over VT\&. They are a modern phrasing of older formalisms invented by
Alexander Birham\&. These formalisms were called TS (TMG recognition
scheme), and gTS (generalized TS)\&. Later they were renamed to TPDL
(Top-Down Parsing Languages) and gTPDL (generalized TPDL)\&.
.PP
They can be easily implemented by recursive descent parsers with
backtracking\&. This makes them relatives of LL(k) Context-Free
Grammars\&.
.SH REFERENCES
.IP [1]
\fIThe Packrat Parsing and Parsing Expression Grammars Page\fR [http://www\&.pdos\&.lcs\&.mit\&.edu/~baford/packrat/],
by Bryan Ford, Massachusetts Institute of Technology\&. This is the main
entry page to PEGs, and their realization through Packrat Parsers\&.
.IP [2]
\fIhttp://en\&.wikipedia\&.org/wiki/Parsing_expression_grammar\fR
Wikipedia's entry about Parsing Expression Grammars\&.
.IP [3]
\fIParsing Techniques - A Practical Guide \fR [http://www\&.cs\&.vu\&.nl/~dick/PTAPG\&.html], an online book
offering a clear, accessible, and thorough discussion of many
different parsing techniques with their interrelations and
applicabilities, including error recovery techniques\&.
.IP [4]
\fICompilers and Compiler Generators\fR [http://scifac\&.ru\&.ac\&.za/compilers/], an online book using
CoCo/R, a generator for recursive descent parsers\&.
.PP
.SH "BUGS, IDEAS, FEEDBACK"
This document, and the package it describes, will undoubtedly contain
bugs and other problems\&.
Please report such in the category \fIpt\fR of the
\fITcllib Trackers\fR [http://core\&.tcl\&.tk/tcllib/reportlist]\&.
Please also report any ideas for enhancements you may have for either
package and/or documentation\&.
.PP
When proposing code changes, please provide \fIunified diffs\fR,
i\&.e the output of \fBdiff -u\fR\&.
.PP
Note further that \fIattachments\fR are strongly preferred over
inlined patches\&. Attachments can be made by going to the \fBEdit\fR
form of the ticket immediately after its creation, and then using the
left-most button in the secondary navigation bar\&.
.SH KEYWORDS
EBNF, LL(k), PEG, TDPL, context-free languages, expression, grammar, matching, parser, parsing expression, parsing expression grammar, push down automaton, recursive descent, state, top-down parsing languages, transducer
.SH CATEGORY
Parsing and Grammars
.SH COPYRIGHT
.nf
Copyright (c) 2009 Andreas Kupries <andreas_kupries@users\&.sourceforge\&.net>

.fi