.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings.  \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote.  \*(C+ will
.\" give a nicer C++.  Capital omega is used to do unbreakable dashes and
.\" therefore won't be available.  \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
.    ds -- \(*W-
.    ds PI pi
.    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
.    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
.    ds L" ""
.    ds R" ""
.    ds C` ""
.    ds C' ""
'br\}
.el\{\
.    ds -- \|\(em\|
.    ds PI \(*p
.    ds L" ``
.    ds R" ''
.    ds C`
.    ds C'
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el       .ds Aq '
.\"
.\" If the F register is >0, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD.  Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.\"
.\" Avoid warning from groff about undefined register 'F'.
.de IX
..
.nr rF 0
.if \n(.g .if rF .nr rF 1
.if (\n(rF:(\n(.g==0)) \{\
.    if \nF \{\
.        de IX
.        tm Index:\\$1\t\\n%\t"\\$2"
..
.        if !\nF==2 \{\
.            nr % 0
.            nr F 2
.        \}
.    \}
.\}
.rr rF
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear.  Run.  Save yourself.  No user-serviceable parts.
.    \" fudge factors for nroff and troff
.if n \{\
.    ds #H 0
.    ds #V .8m
.    ds #F .3m
.    ds #[ \f1
.    ds #] \fP
.\}
.if t \{\
.    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
.    ds #V .6m
.    ds #F 0
.    ds #[ \&
.    ds #] \&
.\}
.    \" simple accents for nroff and troff
.if n \{\
.    ds ' \&
.    ds ` \&
.    ds ^ \&
.    ds , \&
.    ds ~ ~
.    ds /
.\}
.if t \{\
.    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
.    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
.    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
.    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
.    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
.    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
.    \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
.    \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
.    \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
.    ds : e
.    ds 8 ss
.    ds o a
.    ds d- d\h'-1'\(ga
.    ds D- D\h'-1'\(hy
.    ds th \o'bp'
.    ds Th \o'LP'
.    ds ae ae
.    ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "SLMBUILD 1"
.TH SLMBUILD 1 "2019-12-06" "perl v5.30.0" "User Contributed Perl Documentation"
.\" For nroff, turn off justification.  Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
slmbuild \- generate language model from idngram file
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
slmbuild [\fIoption\fR]... \fIidngram_file\fR...
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
\&\fBslmbuild\fR generates a back-off smoothing language model from a given idngram file. Generally, the \fIidngram_file\fR is created by \fBids2ngram\fR.
.SH "OPTIONS All the following options are mandatory."
.IX Header "OPTIONS All the following options are mandatory."
.IP "\fB\-n\fR,\fB\-\-NMax\fR \fIN\fR" 4
.IX Item "-n,--NMax N"
1 for unigram, 2 for bigram, 3 for trigram. Any number not in the range of 1..3 is not valid.
.IP "\fB\-o\fR, \fB\-\-out\fR \fIoutput-file\fR" 4
.IX Item "-o, --out output-file"
Specify the output xfilei name.
.IP "\fB\-l\fR, \fB\-\-log\fR" 4
.IX Item "-l, --log"
using \fI\-log(pr)\fR, use \fIpr\fR directly by default.
.IP "\fB\-w\fR, \fB\-\-wordcount\fR \fIN\fR" 4
.IX Item "-w, --wordcount N"
Lexican size, number of different words.
.IP "\fB\-b\fR, \fB\-\-brk\fR \fIid\fR..." 4
.IX Item "-b, --brk id..."
Set the ids which should be treated as breaker.
.IP "\fB\-e\fR, \fB\-\-e\fR \fIid\fR..." 4
.IX Item "-e, --e id..."
Set the ids which should not be put into \s-1LM.\s0
.IP "\fB\-c\fR, \fB\-\-cut\fR \fIc\fR..." 4
.IX Item "-c, --cut c..."
k\-grams whose freq <= c[k] are dropped.
.IP "\fB\-d\fR, \fB\-\-discount\fR \fImethod\fR, \fIparam\fR..." 4
.IX Item "-d, --discount method, param..."
The k\-th \fB\-d\fR parm specifies the discount method
.Sp
For k\-gram, possibble values for method/param are:
.Sp
.Vb 7
\&      B<GT>,I<R>,I<dis>  : B<GT> discount for r E<lt>= I<R>, r is the freq of a ngram.
\&                  Linear discount for those r E<gt> I<R>, i.e. r\*(Aq=r*dis
\&                  0 E<lt>E<lt> dis E<lt> 1.0, for example 0.999 
\&      B<ABS>,[I<dis>] : Absolute discount r\*(Aq=r\-I<dis>. And I<dis> is optional
\&                  0 E<lt>E<lt> I<dis> E<lt> cut[k]+1.0, normally I<dis> E<lt> 1.0.
\&      LIN,[I<dis>] : Linear discount r\*(Aq=r*dis. And dis is optional
\&                  0 E<lt> dis E<lt> 1.0
.Ve
.SH "NOTE"
.IX Header "NOTE"
\&\fB\-n\fR must be given before \fB\-c\fR \fB\-b\fR. And \fB\-c\fR must give right number of cut-off,
also \fB\-d\fRs must appear exactly N times specifying the discounts for 1\-gram, 2\-gram..., 
respectively.
.PP
BREAKER-IDs could be SentenceTokens or ParagraphTokens. Conceptually,
these ids have no meaning when they appeared in the middle of n\-gram.
.PP
EXCLUDE-IDs could be ambiguious-ids. Conceptually, n\-grams which
contain those ids are meaningless.
.PP
We can not erase ngrams according to BREAKER-IDS and EXCLUDE-IDs directly
from \s-1IDNGRAM\s0 file, because some low-level information is still useful in it.
.SH "EXAMPLE"
.IX Header "EXAMPLE"
Following example read 'all.id3gram' and write trigram model 'all.slm'.
.PP
At 1\-gram level, use Good-Turing discount with cut-off 0, i<R>=8, \fIdis\fR=0.9995. At
2\-gram level, use Absolute discount with cut-off 3, dis auto-calc. At 3\-gram
level, use Absolute discount with cut-off 2, dis auto-calc. Word id 10,11,12
are breakers (sentence/para/paper breaker, etc). Exclude-ID is 9. Lexicon 
contains 200000 words. The result languagme model uses \-log(pr).
.PP
\&\fBslmbuild \-l \-n 3 \-o all.slm \-w 200000 \-c 0,3,2 \-d \s-1GT,8,0.9995\s0 \-d \s-1ABS\s0 \-d \s-1ABS\s0 \-b 10,11,12 \-e 9 all.id3gram\fR
.SH "AUTHOR"
.IX Header "AUTHOR"
Originally written by Phill.Zhang <phill.zhang@sun.com>.
Currently maintained by Kov.Chai <tchaikov@gmail.com>.
.SH "SEE ALSO"
.IX Header "SEE ALSO"
\&\fBids2ngram\fR(1), \fBslmprune\fR(1).