.\" Automatically generated by Pod::Man 2.28 (Pod::Simple 3.28)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings.  \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote.  \*(C+ will
.\" give a nicer C++.  Capital omega is used to do unbreakable dashes and
.\" therefore won't be available.  \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
.    ds -- \(*W-
.    ds PI pi
.    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
.    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
.    ds L" ""
.    ds R" ""
.    ds C` ""
.    ds C' ""
'br\}
.el\{\
.    ds -- \|\(em\|
.    ds PI \(*p
.    ds L" ``
.    ds R" ''
.    ds C`
.    ds C'
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el       .ds Aq '
.\"
.\" If the F register is turned on, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD.  Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.\"
.\" Avoid warning from groff about undefined register 'F'.
.de IX
..
.nr rF 0
.if \n(.g .if rF .nr rF 1
.if (\n(rF:(\n(.g==0)) \{
.    if \nF \{
.        de IX
.        tm Index:\\$1\t\\n%\t"\\$2"
..
.        if !\nF==2 \{
.            nr % 0
.            nr F 2
.        \}
.    \}
.\}
.rr rF
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear.  Run.  Save yourself.  No user-serviceable parts.
.    \" fudge factors for nroff and troff
.if n \{\
.    ds #H 0
.    ds #V .8m
.    ds #F .3m
.    ds #[ \f1
.    ds #] \fP
.\}
.if t \{\
.    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
.    ds #V .6m
.    ds #F 0
.    ds #[ \&
.    ds #] \&
.\}
.    \" simple accents for nroff and troff
.if n \{\
.    ds ' \&
.    ds ` \&
.    ds ^ \&
.    ds , \&
.    ds ~ ~
.    ds /
.\}
.if t \{\
.    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
.    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
.    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
.    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
.    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
.    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
.    \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
.    \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
.    \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
.    ds : e
.    ds 8 ss
.    ds o a
.    ds d- d\h'-1'\(ga
.    ds D- D\h'-1'\(hy
.    ds th \o'bp'
.    ds Th \o'LP'
.    ds ae ae
.    ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "LWP::RobotUA 3pm"
.TH LWP::RobotUA 3pm "2015-12-05" "perl v5.20.2" "User Contributed Perl Documentation"
.\" For nroff, turn off justification.  Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
LWP::RobotUA \- a class for well\-behaved Web robots
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
.Vb 4
\&  use LWP::RobotUA;
\&  my $ua = LWP::RobotUA\->new(\*(Aqmy\-robot/0.1\*(Aq, \*(Aqme@foo.com\*(Aq);
\&  $ua\->delay(10);  # be very nice \-\- max one hit every ten minutes!
\&  ...
\&
\&  # Then just use it just like a normal LWP::UserAgent:
\&  my $response = $ua\->get(\*(Aqhttp://whatever.int/...\*(Aq);
\&  ...
.Ve
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
This class implements a user agent that is suitable for robot
applications.  Robots should be nice to the servers they visit.  They
should consult the \fI/robots.txt\fR file to ensure that they are welcomed
and they should not make requests too frequently.
.PP
But before you consider writing a robot, take a look at
<URL:http://www.robotstxt.org/>.
.PP
When you use an \fILWP::RobotUA\fR object as your user agent, then you do not
really have to think about these things yourself; \f(CW\*(C`robots.txt\*(C'\fR files
are automatically consulted and obeyed, the server isn't queried
too rapidly, and so on.  Just send requests
as you do when you are using a normal \fILWP::UserAgent\fR
object (using \f(CW\*(C`$ua\->get(...)\*(C'\fR, \f(CW\*(C`$ua\->head(...)\*(C'\fR,
\&\f(CW\*(C`$ua\->request(...)\*(C'\fR, etc.), and this
special agent will make sure you are nice.
.SH "METHODS"
.IX Header "METHODS"
The LWP::RobotUA is a sub-class of LWP::UserAgent and implements the
same methods. In addition the following methods are provided:
.ie n .IP "$ua = LWP::RobotUA\->new( %options )" 4
.el .IP "\f(CW$ua\fR = LWP::RobotUA\->new( \f(CW%options\fR )" 4
.IX Item "$ua = LWP::RobotUA->new( %options )"
.PD 0
.ie n .IP "$ua = LWP::RobotUA\->new( $agent, $from )" 4
.el .IP "\f(CW$ua\fR = LWP::RobotUA\->new( \f(CW$agent\fR, \f(CW$from\fR )" 4
.IX Item "$ua = LWP::RobotUA->new( $agent, $from )"
.ie n .IP "$ua = LWP::RobotUA\->new( $agent, $from, $rules )" 4
.el .IP "\f(CW$ua\fR = LWP::RobotUA\->new( \f(CW$agent\fR, \f(CW$from\fR, \f(CW$rules\fR )" 4
.IX Item "$ua = LWP::RobotUA->new( $agent, $from, $rules )"
.PD
The LWP::UserAgent options \f(CW\*(C`agent\*(C'\fR and \f(CW\*(C`from\*(C'\fR are mandatory.  The
options \f(CW\*(C`delay\*(C'\fR, \f(CW\*(C`use_sleep\*(C'\fR and \f(CW\*(C`rules\*(C'\fR initialize attributes
private to the RobotUA.  If \f(CW\*(C`rules\*(C'\fR are not provided, then
\&\f(CW\*(C`WWW::RobotRules\*(C'\fR is instantiated providing an internal database of
\&\fIrobots.txt\fR.
.Sp
It is also possible to just pass the value of \f(CW\*(C`agent\*(C'\fR, \f(CW\*(C`from\*(C'\fR and
optionally \f(CW\*(C`rules\*(C'\fR as plain positional arguments.
.ie n .IP "$ua\->delay" 4
.el .IP "\f(CW$ua\fR\->delay" 4
.IX Item "$ua->delay"
.PD 0
.ie n .IP "$ua\->delay( $minutes )" 4
.el .IP "\f(CW$ua\fR\->delay( \f(CW$minutes\fR )" 4
.IX Item "$ua->delay( $minutes )"
.PD
Get/set the minimum delay between requests to the same server, in
\&\fIminutes\fR.  The default is 1 minute.  Note that this number doesn't
have to be an integer; for example, this sets the delay to 10 seconds:
.Sp
.Vb 1
\&    $ua\->delay(10/60);
.Ve
.ie n .IP "$ua\->use_sleep" 4
.el .IP "\f(CW$ua\fR\->use_sleep" 4
.IX Item "$ua->use_sleep"
.PD 0
.ie n .IP "$ua\->use_sleep( $boolean )" 4
.el .IP "\f(CW$ua\fR\->use_sleep( \f(CW$boolean\fR )" 4
.IX Item "$ua->use_sleep( $boolean )"
.PD
Get/set a value indicating whether the \s-1UA\s0 should \fIsleep()\fR if requests
arrive too fast, defined as \f(CW$ua\fR\->delay minutes not passed since
last request to the given server.  The default is \s-1TRUE. \s0 If this value is
\&\s-1FALSE\s0 then an internal \s-1SERVICE_UNAVAILABLE\s0 response will be generated.
It will have a Retry-After header that indicates when it is \s-1OK\s0 to
send another request to this server.
.ie n .IP "$ua\->rules" 4
.el .IP "\f(CW$ua\fR\->rules" 4
.IX Item "$ua->rules"
.PD 0
.ie n .IP "$ua\->rules( $rules )" 4
.el .IP "\f(CW$ua\fR\->rules( \f(CW$rules\fR )" 4
.IX Item "$ua->rules( $rules )"
.PD
Set/get which \fIWWW::RobotRules\fR object to use.
.ie n .IP "$ua\->no_visits( $netloc )" 4
.el .IP "\f(CW$ua\fR\->no_visits( \f(CW$netloc\fR )" 4
.IX Item "$ua->no_visits( $netloc )"
Returns the number of documents fetched from this server host. Yeah I
know, this method should probably have been named \fInum_visits()\fR or
something like that. :\-(
.ie n .IP "$ua\->host_wait( $netloc )" 4
.el .IP "\f(CW$ua\fR\->host_wait( \f(CW$netloc\fR )" 4
.IX Item "$ua->host_wait( $netloc )"
Returns the number of \fIseconds\fR (from now) you must wait before you can
make a new request to this host.
.ie n .IP "$ua\->as_string" 4
.el .IP "\f(CW$ua\fR\->as_string" 4
.IX Item "$ua->as_string"
Returns a string that describes the state of the \s-1UA.\s0
Mainly useful for debugging.
.SH "SEE ALSO"
.IX Header "SEE ALSO"
LWP::UserAgent, WWW::RobotRules
.SH "COPYRIGHT"
.IX Header "COPYRIGHT"
Copyright 1996\-2004 Gisle Aas.
.PP
This library is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.