.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings. \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote. \*(C+ will
.\" give a nicer C++. Capital omega is used to do unbreakable dashes and
.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
. ds -- \(*W-
. ds PI pi
. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch
. ds L" ""
. ds R" ""
. ds C` ""
. ds C' ""
'br\}
.el\{\
. ds -- \|\(em\|
. ds PI \(*p
. ds L" ``
. ds R" ''
. ds C`
. ds C'
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el .ds Aq '
.\"
.\" If the F register is >0, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD. Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.\"
.\" Avoid warning from groff about undefined register 'F'.
.de IX
..
.nr rF 0
.if \n(.g .if rF .nr rF 1
.if (\n(rF:(\n(.g==0)) \{\
. if \nF \{\
. de IX
. tm Index:\\$1\t\\n%\t"\\$2"
..
. if !\nF==2 \{\
. nr % 0
. nr F 2
. \}
. \}
.\}
.rr rF
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear. Run. Save yourself. No user-serviceable parts.
. \" fudge factors for nroff and troff
.if n \{\
. ds #H 0
. ds #V .8m
. ds #F .3m
. ds #[ \f1
. ds #] \fP
.\}
.if t \{\
. ds #H ((1u-(\\\\n(.fu%2u))*.13m)
. ds #V .6m
. ds #F 0
. ds #[ \&
. ds #] \&
.\}
. \" simple accents for nroff and troff
.if n \{\
. ds ' \&
. ds ` \&
. ds ^ \&
. ds , \&
. ds ~ ~
. ds /
.\}
.if t \{\
. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
. \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
. \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
. \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
. ds : e
. ds 8 ss
. ds o a
. ds d- d\h'-1'\(ga
. ds D- D\h'-1'\(hy
. ds th \o'bp'
. ds Th \o'LP'
. ds ae ae
. ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "PULLNEWS 1"
.TH PULLNEWS 1 "2018-05-14" "INN 2.6.4" "InterNetNews Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
pullnews \- Pull news from multiple news servers and feed it to another
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
\&\fBpullnews\fR [\fB\-BhnOqRx\fR] [\fB\-a\fR \fIhashfeed\fR] [\fB\-b\fR \fIfraction\fR]
[\fB\-c\fR \fIconfig\fR] [\fB\-C\fR \fIwidth\fR]
[\fB\-d\fR \fIlevel\fR] [\fB\-f\fR \fIfraction\fR] [\fB\-F\fR \fIfakehop\fR] [\fB\-g\fR \fIgroups\fR]
[\fB\-G\fR \fInewsgroups\fR] [\fB\-H\fR \fIheaders\fR] [\fB\-k\fR \fIcheckpt\fR] [\fB\-l\fR \fIlogfile\fR]
[\fB\-m\fR \fIheader_pats\fR] [\fB\-M\fR \fInum\fR] [\fB\-N\fR \fItimeout\fR] [\fB\-p\fR \fIport\fR]
[\fB\-P\fR \fIhop_limit\fR] [\fB\-Q\fR \fIlevel\fR] [\fB\-r\fR \fIfile\fR] [\fB\-s\fR \fIto-server\fR[:\fIport\fR]]
[\fB\-S\fR \fImax-run\fR] [\fB\-t\fR \fIretries\fR] [\fB\-T\fR \fIconnect-pause\fR] [\fB\-w\fR \fInum\fR]
[\fB\-z\fR \fIarticle-pause\fR] [\fB\-Z\fR \fIgroup-pause\fR] [\fIfrom-server\fR ...]
.SH "REQUIREMENTS"
.IX Header "REQUIREMENTS"
The \f(CW\*(C`Net::NNTP\*(C'\fR module must be installed. This module is available as part
of the libnet distribution and comes with recent versions of Perl. For
older versions of Perl, you can download it from .
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
\&\fBpullnews\fR reads a config file named \fIpullnews.marks\fR, and connects
to the upstream servers given there as a reader client. This file is
looked for in \fIpathdb\fR when \fBpullnews\fR is run as the user set in
\&\fIrunasuser\fR in \fIinn.conf\fR (which is by default the \f(CW\*(C`news\*(C'\fR user);
otherwise, this file is looked for in the running user's home directory.
.PP
By default, \fBpullnews\fR connects to all servers listed in the configuration
file, but you can limit \fBpullnews\fR to specific servers by listing them
on the command line: a whitespace-separated list of server names can be
specified, like \fIfrom-server\fR for one of them. For each server it connects
to, it pulls over articles and feeds them to the destination server via
the \s-1IHAVE\s0 or \s-1POST\s0 commands. This means that the system \fBpullnews\fR is
run on must have feeding access to the destination news server.
.PP
\&\fBpullnews\fR is designed for very small sites that do not want to bother
setting up traditional peering and is not meant for handling large feeds.
.SH "OPTIONS"
.IX Header "OPTIONS"
.IP "\fB\-a\fR \fIhashfeed\fR" 4
.IX Item "-a hashfeed"
This option is a deterministic way to control the flow of articles and to
split a feed. The \fIhashfeed\fR parameter must be in the form \f(CW\*(C`value/mod\*(C'\fR
or \f(CW\*(C`start\-end/mod\*(C'\fR. The Message-ID of each article is hashed using \s-1MD5,\s0
which results in a 128\-bit hash. The lowest 32\ bits are then taken
by default as the hashfeed value (which is an integer). If the hashfeed
value modulus \f(CW\*(C`mod\*(C'\fR plus one equals \f(CW\*(C`value\*(C'\fR or is between \f(CW\*(C`start\*(C'\fR
and \f(CW\*(C`end\*(C'\fR, \fBpullnews\fR will feed the article. All these numbers must
be integers.
.Sp
For instance:
.Sp
.Vb 2
\& pullnews \-a 1/2 Feeds about 50% of all articles.
\& pullnews \-a 2/2 Feeds the other 50% of all articles.
.Ve
.Sp
Another example:
.Sp
.Vb 3
\& pullnews \-a 1\-3/10 Feeds about 30% of all articles.
\& pullnews \-a 4\-5/10 Feeds about 20% of all articles.
\& pullnews \-a 6\-10/10 Feeds about 50% of all articles.
.Ve
.Sp
You can use an extended syntax of the form \f(CW\*(C`value/mod:offset\*(C'\fR or
\&\f(CW\*(C`start\-end/mod:offset\*(C'\fR (using an underscore \f(CW\*(C`_\*(C'\fR instead of a colon
\&\f(CW\*(C`:\*(C'\fR is also recognized). As \s-1MD5\s0 generates a 128\-bit return value,
it is possible to specify from which byte-offset the 32\-bit integer
used by hashfeed starts. The default value for \f(CW\*(C`offset\*(C'\fR is \f(CW\*(C`:0\*(C'\fR and
thirteen overlapping values from \f(CW\*(C`:0\*(C'\fR to \f(CW\*(C`:12\*(C'\fR can be used. Only up to
four totally independent values exist: \f(CW\*(C`:0\*(C'\fR, \f(CW\*(C`:4\*(C'\fR, \f(CW\*(C`:8\*(C'\fR and \f(CW\*(C`:12\*(C'\fR.
.Sp
Therefore, it allows generating a second level of deterministic
distribution. Indeed, if \fBpullnews\fR feeds \f(CW\*(C`1/2\*(C'\fR, it can go on
splitting thanks to \f(CW\*(C`1\-3/9:4\*(C'\fR for instance. Up to four levels of
deterministic distribution can be used.
.Sp
The algorithm is compatible with the one used by Diablo\ 5.1 and up.
.IP "\fB\-b\fR \fIfraction\fR" 4
.IX Item "-b fraction"
Backtrack on server numbering reset. Specify the proportion (\f(CW0.0\fR to \f(CW1.0\fR)
of a group's articles to pull when the server's article number is less than
our high for that group. When \fIfraction\fR is \f(CW1.0\fR, pull all the articles on
a renumbered server. The default is to do nothing.
.IP "\fB\-B\fR" 4
.IX Item "-B"
Feed is header-only, that is to say \fBpullnews\fR only feeds the headers
of the articles, plus one blank line. It adds the Bytes: header field
if the article does not already have one, and keeps the body only if
the article is a control article.
.IP "\fB\-c\fR \fIconfig\fR" 4
.IX Item "-c config"
Normally, the config file is stored in \fIpullnews.marks\fR in \fIpathdb\fR
when \fBpullnews\fR is run as the news user, or otherwise in the running
user's home directory. If \fB\-c\fR is given, \fIconfig\fR will be used as
the config file instead. This is useful if you're running \fBpullnews\fR
as a system user on an automated basis out of cron or as an individual
user, rather than the news user.
.Sp
See \*(L"\s-1CONFIG FILE\*(R"\s0 below for the format of this file.
.IP "\fB\-C\fR \fIwidth\fR" 4
.IX Item "-C width"
Use \fIwidth\fR characters per line for the progress table. The default value
is \f(CW50\fR.
.IP "\fB\-d\fR \fIlevel\fR" 4
.IX Item "-d level"
Set the debugging level to the integer \fIlevel\fR; more debugging output
will be logged as this increases. The default value is \f(CW0\fR.
.IP "\fB\-f\fR \fIfraction\fR" 4
.IX Item "-f fraction"
This changes the proportion of articles to get from each group to
\&\fIfraction\fR and should be in the range \f(CW0.0\fR to \f(CW1.0\fR (\f(CW1.0\fR being
the default).
.IP "\fB\-F\fR \fIfakehop\fR" 4
.IX Item "-F fakehop"
Prepend \fIfakehop\fR as a host to the Path: header of articles fed.
.IP "\fB\-g\fR \fIgroups\fR" 4
.IX Item "-g groups"
Specify a collection of groups to get. \fIgroups\fR is a list of
newsgroups separated by commas (only commas, no spaces). Each group must
be defined in the config file, and only the remote hosts that carry those
groups will be contacted. Note that this is a simple list of groups, not
a wildmat expression, and wildcards are not supported.
.IP "\fB\-G\fR \fInewsgroups\fR" 4
.IX Item "-G newsgroups"
Add the comma-separated list of groups \fInewsgroups\fR to each server in the
configuration file (see also \fB\-g\fR and \fB\-w\fR).
.IP "\fB\-h\fR" 4
.IX Item "-h"
Print a usage message and exit.
.IP "\fB\-H\fR \fIheaders\fR" 4
.IX Item "-H headers"
Remove these named headers (colon-separated list) from fed articles.
.IP "\fB\-k\fR \fIcheckpt\fR" 4
.IX Item "-k checkpt"
Checkpoint (save) the config file every \fIcheckpt\fR articles
(default is \f(CW0\fR, that is to say at the end of the session).
.IP "\fB\-l\fR \fIlogfile\fR" 4
.IX Item "-l logfile"
Log progress/stats to \fIlogfile\fR (default is \f(CW\*(C`stdout\*(C'\fR).
.IP "\fB\-m\fR \fIheader_pats\fR" 4
.IX Item "-m header_pats"
Feed an article based on header matching. The argument is a number of
whitespace-separated tuples (each tuple being a colon-separated header and
regular expression). For instance:
.Sp
.Vb 1
\& \-m "Hdr1:regexp1 !Hdr2:regexp2 #Hdr3:regexp3 !#Hdr4:regexp4"
.Ve
.Sp
specifies that the article will be passed only if the \f(CW\*(C`Hdr1:\*(C'\fR header
matches \f(CW\*(C`regexp1\*(C'\fR and the \f(CW\*(C`Hdr2:\*(C'\fR header does not match \f(CW\*(C`regexp2\*(C'\fR.
Besides, if the \f(CW\*(C`Hdr3:\*(C'\fR header matches \f(CW\*(C`regexp3\*(C'\fR, that header is
removed; and if the \f(CW\*(C`Hdr4:\*(C'\fR header does not match \f(CW\*(C`regexp4\*(C'\fR, that
header is removed.
.IP "\fB\-M\fR \fInum\fR" 4
.IX Item "-M num"
Specify the maximum number of articles (per group) to process.
The default is to process all new articles. See also \fB\-f\fR.
.IP "\fB\-n\fR" 4
.IX Item "-n"
Do nothing but read articles \-\-\ does not feed articles downstream,
writes no \fBrnews\fR file, does not update the config file.
.IP "\fB\-N\fR \fItimeout\fR" 4
.IX Item "-N timeout"
Specify the timeout length, as \fItimeout\fR seconds,
when establishing an \s-1NNTP\s0 connection.
.IP "\fB\-O\fR" 4
.IX Item "-O"
Use an optimized mode: \fBpullnews\fR checks whether the article already
exists on the downstream server, before downloading it. It may help
for huge articles or a slow link to upstream hosts.
.IP "\fB\-p\fR \fIport\fR" 4
.IX Item "-p port"
Connect to the destination news server on a port other than the default of
\&\f(CW119\fR. This option does not change the port used to connect to the source
news servers.
.IP "\fB\-P\fR \fIhop_limit\fR" 4
.IX Item "-P hop_limit"
Restrict feeding an article based on the number of hops it has already made.
Count the hops in the Path: header (\fIhop_count\fR), feeding the article only
when \fIhop_limit\fR is \f(CW\*(C`+num\*(C'\fR and \fIhop_count\fR is more than \fInum\fR;
or \fIhop_limit\fR is \f(CW\*(C`\-num\*(C'\fR and \fIhop_count\fR is less than \fInum\fR.
.IP "\fB\-q\fR" 4
.IX Item "-q"
Print out less status information while running.
.IP "\fB\-Q\fR \fIlevel\fR" 4
.IX Item "-Q level"
Set the quietness level (\f(CW\*(C`\-Q 2\*(C'\fR is equivalent to \f(CW\*(C`\-q\*(C'\fR). The higher this
value, the less gets logged. The default is \f(CW0\fR.
.IP "\fB\-r\fR \fIfile\fR" 4
.IX Item "-r file"
Rather than feeding the downloaded articles to a destination server, instead
create a batch file that can later be fed to a server using \fBrnews\fR. See
\&\fBrnews\fR\|(1) for more information about the batch file format.
.IP "\fB\-R\fR" 4
.IX Item "-R"
Be a reader (use \s-1MODE READER\s0 and \s-1POST\s0 commands) to the downstream
server. The default is to use the \s-1IHAVE\s0 command.
.IP "\fB\-s\fR \fIto-server\fR[:\fIport\fR]" 4
.IX Item "-s to-server[:port]"
Normally, \fBpullnews\fR will feed the articles it retrieves to the news
server running on localhost. To connect to a different host, specify a
server with the \fB\-s\fR flag. You can also specify the port with this same
flag or use \fB\-p\fR.
.IP "\fB\-S\fR \fImax-run\fR" 4
.IX Item "-S max-run"
Specify the maximum time \fImax-run\fR in seconds for \fBpullnews\fR to run.
.IP "\fB\-t\fR \fIretries\fR" 4
.IX Item "-t retries"
The maximum number (\fIretries\fR) of attempts to connect to a server
(see also \fB\-T\fR). The default is \f(CW0\fR.
.IP "\fB\-T\fR \fIconnect-pause\fR" 4
.IX Item "-T connect-pause"
Pause \fIconnect-pause\fR seconds between connection retries (see also \fB\-t\fR).
The default is \f(CW1\fR.
.IP "\fB\-w\fR \fInum\fR" 4
.IX Item "-w num"
Set each group's high water mark (last received article number) to \fInum\fR.
If \fInum\fR is negative, calculate \fICurrent\fR+\fInum\fR instead (i.e. get the last
\&\fInum\fR articles). Therefore, a \fInum\fR of \f(CW0\fR will re-get all articles on the
server; whereas a \fInum\fR of \f(CW\*(C`\-0\*(C'\fR will get no old articles, setting the
water mark to \fICurrent\fR (the most recent article on the server).
.IP "\fB\-x\fR" 4
.IX Item "-x"
If the \fB\-x\fR flag is used, an Xref: header is added to any article
that lacks one. It can be useful for instance if articles are fed
to a news server which has \fIxrefslave\fR set in \fIinn.conf\fR.
.IP "\fB\-z\fR \fIarticle-pause\fR" 4
.IX Item "-z article-pause"
Sleep \fIarticle-pause\fR seconds between articles. The default is \f(CW0\fR.
.IP "\fB\-Z\fR \fIgroup-pause\fR" 4
.IX Item "-Z group-pause"
Sleep \fIgroup-pause\fR seconds between groups. The default is \f(CW0\fR.
.SH "CONFIG FILE"
.IX Header "CONFIG FILE"
The config file for \fBpullnews\fR is divided into blocks, one block for
each remote server to connect to. A block begins with the host line
(which must have no leading whitespace) and contains just the hostname of
the remote server, optionally followed by authentication details (username
and password for that server). Note that authentication details can also
be provided for the downstream server (a host line could be added for it
in the configuration file, with no newsgroup to fetch).
.PP
Following the host line should be one or more newsgroup lines which start
with whitespace followed by the name of a newsgroup to retrieve. Only one
newsgroup should be listed on each line.
.PP
\&\fBpullnews\fR will update the config file to include the time the group was
last checked and the highest numbered article successfully retrieved and
transferred to the destination server. It uses this data to avoid doing
duplicate work the next time it runs.
.PP
The full syntax is:
.PP
.Vb 3
\& []
\& [