.TH pavuk 1 "2011-05-19T08:14" "0.9.35" "Internet utils"
.SH NAME
pavuk \- HTTP, HTTP over SSL, FTP, FTP over SSL and Gopher recursive document retrieval program
.SH SYNOPSIS
.PP
.B pavuk [-mode {normal | resumeregets | singlepage | singlereget | sync | dontstore | ftpdir | mirror}]
.I [-X] [-runX] [-bg/-nobg] [prefs/-noprefs] [-h] [-v] [-progress/-noprogress] [-stime/-nostime] [-xmaxlog $nr]
.I [-logfile $file] [-slogfile $file] [-auth_file $file] [-msgcat $dir] [-language $str] [-gui_font $font] [-quiet/-verbose
.I [-read_css/-noread_css]
.I [-cdir $dir]
.I [-scndir $dir]
.I [-scenario $str] [-dumpscn $filename]
.I [-lmax $nr] [-dmax $nr]
.I [-leave_level $nr]
.I [-maxsize $nr] [-minsize $nr]
.I [-asite $list] [-dsite $list]
.I [-adomain $list] [-ddomain $list]
.I [-asfx $list] [-dsfx $list]
.I [-aprefix $list] [-dprefix $list]
.I [-amimt $list] [-dmimet $list]
.I [-pattern $pattern] [-url_pattern $pattern]
.I [-rpattern $regexp] [-url_rpattern $regexp]
.I [-skip_pattern $pattern] [-skip_url_pattern $pattern]
.I [-skip_rpattern $regexp] [-skip_url_rpattern $regexp]
.I [-newer_than $time] [-older_than $time]
.I [-schedule $time] [-reschedule $nr]
.I [-dont_leave_site/-leave_site] [-dont_leave_dir/-leave_dir]
.I [-http_proxy $site[:$port]] [-ftp_proxy $site[:$port]]
.I [-ssl_proxy $site[:$port]] [-gopher_proxy $site[:$port]]
.I [-ftp_httpgw/-noftp_httpgw] [-ftp_dirtyproxy/-noftp_dirtyproxy]
.I [-gopher_httpgw/-nogopher_httpgw]
.I [-noFTP/-FTP] [-noHTTP/-HTTP] [-noSSL/-SSL] [-noGopher/-Gopher]
.I [-FTPdir/-noFTPdir]
.I [-noCGI/-CGI]
.I [-FTPlist/-noFTPlist] [-FTPhtml/-noFTPhtml]
.I [-noRelocate/-Relocate] [-force_reget/-noforce_reget]
.I [-nocache/-cache] [-check_size/-nocheck_size]
.I [-noRobots/-Robots] [-noEnc/-Enc]
.I [-auth_name $user] [-auth_passwd $pass] [-auth_scheme 1/2/3/4/user/Basic/Digest/NTLM]
.I [-auth_reuse_nonce/-no_auth_reuse_nonce]
.I [-http_proxy_user $user] [-http_proxy_pass $pass] [-http_proxy_auth 1/2/3/4/user/Basic/Digest/NTLM]
.I [-auth_reuse_proxy_nonce/-no_auth_reuse_proxy_nonce]
.I [-ssl_key_file $file] [-ssl_cert_file $file] [-ssl_cert_passwd $pass]
.I [-from $email] [-send_from/-nosend_from] [-identity $str] [-auto_referer/-noauto_referer]
.I [-referer/-noreferer]
.I [-alang $list] [-acharset $list]
.I [-retry $nr] [-nregets $nr] [-nredirs $nr] [-rollback $nr] [-sleep $nr] [-timeout $nr]
.I [-preserve_time/-nopreserve_time] [-preserve_perm/-nopreserve_perm] [-preserve_slinks/-nopreserve_slinks]
.I [-bufsize $nr] [-maxrate $nr] [-minrate $nr]
.I [-user_condition $str]
.I [-cookie_file $file] [-cookie_send/-nocookie_send] [-cookie_recv/-nocookie_recv] [-cookie_update/-nocookie_update] [-cookies_max $nr] [-disabled_cookie_domains $list]
.I [-disable_html_tag $TAG,[$ATTRIB][;...]] [-enable_html_tag $TAG,[$ATTRIB][;...]]
.I [-tr_del_chr $str] [-tr_str_str $str1 $str2] [-tr_chr_chr $chrset1 $chrset2]
.I [-index_name $str] [-store_index/-nostore_index]
.I [-store_name $str] [-debug/-nodebug] [-debug_level $level]
.I [-browser $str]
.I [-urls_file $file]
.I [-file_quota $nr] [-trans_quota $nr] [-fs_quota $nr] [-enable_js/-disable_js]
.I [-fnrules $t $m $r] [-store_info/-nostore_info]
.I [-all_to_local/-noall_to_local] [-sel_to_local/-nosel_to_local]
.I [-all_to_remote/-noall_to_remote]
.I [-url_strategie $strategie]
.I [-remove_adv/-noremove_adv] [-adv_re $RE]
.I [-check_bg/-nocheck_bg]
.I [-send_if_range/-nosend_if_range]
.I [-sched_cmd $str]
.I [-unique_log/-nounique_log]
.I [-post_cmd $str] [-ssl_version $v] [-unique_sslid/-nounique_sslid]
.I [-aip_pattern $re] [-dip_pattern $re]
.I [-use_http11/-nouse_http11]
.I [-local_ip $addr]
.I [-request $req]
.I [-formdata $req] [-httpad $str]
.I [-nthreads $nr] [-immesg/-noimmesg]
.I [-dumpfd $nr] [-dump_urlfd $nr]
.I [-unique_name/-nounique_name]
.I [-leave_site_enter_dir/-dont_leave_site_enter_dir]
.I [-max_time $nr]
.I [-del_after/-nodel_after] [-singlepage/-nosinglepage]
.I [-dump_after/-nodump_after] [-dump_response/-nodump_response]
.I [-auth_ntlm_domain $str] [-auth_proxy_ntlm_domain $str]
.I [-js_pattern $re] [-follow_cmd $str]
.I [-retrieve_symlink/-noretrieve_symlink]
.I [-js_transform $p $t $h $a] [-js_transform2 $p $t $h $a]
.I [-ftp_proxy_user $str] [-ftp_proxy_pass $str]
.I [-limit_inlines/-dont_limit_inlines]
.I [-ftp_list_options $str] [-fix_wuftpd_list/-nofix_wuftpd_list]
.I [-post_update/-nopost_update] [-info_dir $dir]
.I [-mozcache_dir $dir] [-aport $list] [-dport $list]
.I [-hack_add_index/-nohack_add_index]
.I [-default_prefix $str] [-rsleep/-norsleep]
.I [-ftp_login_handshake $host $handshake]
.I [-js_script_file $file]
.I [-dont_touch_url_pattern $pat]
.I [-dont_touch_url_rpattern $pat]
.I [-dont_touch_tag_rpattern $pat]
.I [-tag_pattern $tag $attrib $url]
.I [-tag_rpattern $tag $attrib $url]
.I [-nss_cert_dir $dir]
.I [-nss_accept_unknown_cert/-nonss_accept_unknown_cert]
.I [-nss_domestic_policy/-nss_export_policy]
.I [-[no]verify]
.I [-tlogfile $file]
.I [-trelative {object | program}]
.I [-transparent_proxy FQDN[:port]]
.I [-transparent_ssl_proxy FQDN[:port]]
.I [-sdemo]
.I [-noencode]
.I [URLs]
.br
.PP
.B pavuk -mode {normal | singlepage | singlereget}
.I [-base_level $nr]
.br
.PP
.B pavuk -mode sync
.I [-ddays $nr] [-subdir $dir] [-remove_old/-noremove_old]
.br
.PP
.B pavuk -mode resumeregets
.I [-subdir $dir]
.br
.PP
.B pavuk -mode linkupdate
.I [-X] [-h] [-v] [-cdir $dir] [-subdir $dir] [-scndir $dir] [-scenario $str]
.PP
.B pavuk -mode reminder
.I [-remind_cmd $str]
.br
.PP
.B pavuk -mode mirror
.I [-subdir $dir] [-remove_old/-noremove_old] [-remove_before_store/-noremove_before_store] [-always_mdtm/-noalways_mdtm]
.br
.PP
.SH DESCRIPTION
.sp
This manual page describes how to use pavuk. Pavuk can be used to
mirror contents of internet/intranet servers and to maintain copies in
a local tree of documents. Pavuk stores retrieved documents in
locally mapped disk space. The structure of the local tree is the same
as the one on the remote server. Each supported service (protocol) has
its own subdirectory in the local tree. Each referenced server has
its own subdirectory in these protocols subdirectories; followed by
the port number on which the service resides, delimited by character
'_'. Then the directory structure follows. As of version 0.9pl19 this
can be be changed. With the option
.B -fnrules
you can change the default layout of the local document tree,
without losing link consistency.
.br
With
.B pavuk
it is possible to have up-to-date copies of remote documents in the
local disk space.
.br
As of version 0.3pl2, pavuk can automatically restart broken connections, and reget
partial content from an FTP server (which must support the
.B REST
command), from a properly configured HTTP/1.1 server, or from a HTTP/1.0
server which supports
.B Ranges.
.br
As of version 0.6 it is possible to handle configurations via so
called scenarios. The best way to create such a configuration file is
to use the X Window interface and simply save the created
configuration. The other way is to use the -dumpscn switch.
.br
As of version 0.7pl1 it is possible to store authentification information
into an authinfo file, which pavuk can then parse and use.
.br
As of version 0.8pl4 pavuk can fetch documents for use in a local
proxy/cache server without storing them to local documents tree.
.br
As of version 0.9pl4 pavuk supports
.B SOCKS (4/5)
proxies if you have the required libraries.
.br
As of version 0.9pl12 pavuk can preserve permissions of remote files and
symbolic links, so it can be used for powerful FTP mirroring.
.br
Pavuk supports SSL connections to FTP servers, if you specify ftps://
URL instead of ftp://.
.br
Pavuk can automatically handle file names with unsafe characters for filesystem.
This is yet implemented only for Win32 platform and it is hard coded.
.br
Pavuk can now use \fBHTTP/1.1\fR protocol for communication with HTTP servers.
It can use persistent connections, so one TCP connection should be used to
transfer several documents without closing it. This feature saves network
bandwidth and also speedup network communication.
.br
Pavuk can do configurable \fBPOST\fR requests to HTTP servers and support
also file uploading via HTTP POST request.
.br
Pavuk can automatically fill found HTML forms, if user will supply data
for its fields before with option \fB-formdata\fR.
.br
Pavuk can run configurable number of concurrently running downloading threads when
compiled with multithreading support.
.SH Format of supported URLs
.sp
.sp
\fIHTTP\fR
.br
http://[[user][:password]@]host[:port][/document]
.br
[[user][:password]@]host[:port][/document]
.sp
\fIHTTPS\fR
.br
https://[[user][:password]@]host[:port][/document]
.br
ssl[.domain][:port][/document]
.sp
\fIFTP\fR
.br
ftp://[[user][:password]@]host[:port][/relative_path][;type=x]
.br
ftp://[[user][:password]@]host[:port][//absolute_path][;type=x]
.br
ftp[.domain][:port][/document][;type=x]
.sp
\fIFTPS\fR
.br
ftps://[[user][:password]@]host[:port][/relative_path][;type=x]
.br
ftps://[[user][:password]@]host[:port][//absolute_path][;type=x]
.br
ftps[.domain][:port][/document][;type=x]
.sp
\fIGopher\fR
.br
gopher://host[:port][/type[document]]
.br
gopher[.domain][:port][/type[document]]
.SH Default mapping of URLs to local filenames
.sp
.sp
\fIHTTP\fR
.br
http://[[user][:password]@]host[:port][/document][?query]
.br
to
.br
http/host_port/[document][?query]
.sp
\fIHTTPS\fR
.br
https://[[user][:password]@]host[:port][/document][?query]
.br
to
.br
https/host_port/[document][?query]
.sp
\fIFTP\fR
.br
ftp://[[user][:password]@]host[:port][/path]
.br
to
.br
ftp/host_port/[path]
.sp
\fIFTPS\fR
.br
ftps://[[user][:password]@]host[:port][/path]
.br
to
.br
ftps/host_port/[path]
.sp
\fIGopher\fR
.br
gopher://host[:port][/type[document]]
.br
to
.br
gopher/host_port/[type[document]]
\fINOTE:\fR Pavuk will use the string with which it queries the target
server as the name of the results file. This file name may, in some cases,
contain punctuations such as \fB$,?,=,&\fR etc. Such punctuation can cause
problems when you are trying to browse downloaded files with your browser or
you are trying to process downloaded files with shell scripts or view files
with file management utilities which reference the name of the results file.
If you believe that this maybe causing problems for you, then you can remove
all punctuation from the result file name with the option:
\fB-tr_del_chr [:punct:]\fR or with other options for adjusting filenames.
.SH OPTIONS
.sp
All options are case insensitive.
.SH List of options chapters
.sp
.I Mode
.br
.I Help
.br
.I Indicate/Logging/Interface options
.br
.I Netli options
.br
.I Special start
.br
.I Scenario/Task options
.br
.I Directory options
.br
.I Preserve options
.br
.I Proxy options
.br
.I Proxy Authentification
.br
.I Protocol/Download Option
.br
.I Authentification
.br
.I Site/Domain/Port Limitation Options
.br
.I Limitation Document properties
.br
.I Limitation Document name
.br
.I Limitation Protocol Option
.br
.I Other Limitation Options
.br
.I Javascript support
.br
.I Cookie
.br
.I HTML rewriting engine tuning options
.br
.I Filename/URL Conversion Option
.br
.I Other Options
.br
.SH Mode
.sp
.TP
.I -mode {normal, linkupdate, sync, singlepage, singlereget, resumeregets}
Set operation mode.
.br
.B normal
- retrieves recursive documents
.br
.B linkupdate
- update remote URLs in local HTML documents to local URLs if these URLs
exist in the local tree
.br
.B sync
- synchronize remote documents with local tree (if a local copy of
a document is older than remote, the document is retrieved again,
otherwise nothing happens)
.br
.B singlepage
- URL is retrieved as one page with all inline objects (picture, sound ...)
this mode is now obsoleted by \fB-singlepage\fR option.
.br
.B resumeregets
- pavuk scans the local tree for files that were not retrieved fully
and retrieves them again (uses partial get if possible)
.br
.B singlereget
- get URL until it is retrieved in full
.br
.B dontstore
- transfer page from server, but don't store it to the local tree.
This mode is suitable for fetching pages that are held in a local
proxy/cache server.
.br
.B reminder
- used to inform the user about changed documents
.br
.B ftpdir
- used to list of contents of FTP directories
.br
.sp
default operation mode is
.B normal
mode.
.SH Help
.sp
.TP
.I -h
Print long verbose help message
.TP
.I -v
Show version informations and configuration at compilation time.
.SH Indicate/Logging/Interface options
.sp
.TP
.I -quiet
Don't show any messages on the screen.
.TP
.I -verbose
Force to show output messages on the screen (default)
.TP
.I -progress/-noprogress
Show retrieving progress while running in the terminal (default is progress off)
.TP
.I -stime/-nostime
Show start and end time of transfer. (default isn't this information shown)
.TP
.I -xmaxlog $nr
Maximum number of log lines in the Log widget. 0 means unlimited.
This option is available only when compiled with the GTK+ GUI. (default value is 0)
.TP
.I -logfile $file
File where all produced messages are stored.
.TP
.I -unique_log/-nounique_log
When logfile as specified with the option
.B -logfile
is already used by another process, try to generate new unique name
for the log file. (default is this option turned off)
.TP
.I -slogfile $file
File to store short logs in. This file contains one line of
informations per processed document. This is meant to be used in
connection with any sort of script to produce some statistics, for
validating links on your website, or for generating simple sitemaps.
Multiple pavuk processes can use this file concurrently, without
overwriting each others entries.
Record structure:
.br
.sp
.RS
.nf
- \fBPID\fR of pavuk process
- \fBTIME\fR current time
- \fBCOUNTER\fR in the format current/total number of URLs
- \fBSTATUS\fR contains the type of the error: FATAL, ERR,
WARN or OK
- \fBERRCODE\fR is the number code of the error
(see errcode.h in pavuk sources)
- \fBURL\fR of the document
- \fBPARENTURL\fR first parent document of this URL
(when it doesn't have parent - [none])
- \fBFILENAME\fR is the name of the local file the
document is saved under
- \fBSIZE\fR size of requested document if known
- \fBDOWNLOAD_TIME\fR time which takes downloading of this
document in format seconds.mili_seconds
- \fBHTTPRESP\fR contains the first line of the HTTP server
response
.fi
.RE
.TP
.I -language $str
Native language that pavuk should use for communication with its
user (works only when there is a message catalog for that language)
\fBGNU gettext\fR support (for message internationalization) must also be
compiled in. Default language is taken from your NLS environment variables.
.TP
.I -gui_font $font
Font used in the GUI interface. To list available X fonts use the
.B xlsfonts
command.
This option is available only when compiled with GTK+ GUI support.
.SH Netli options
.SP
.TP
.I -[no]read_css
Enable or disable fetching objects mentioned in style sheets.
.SP
.TP
.I -[no]verify
Enable or disable verifying server CERTS in SSL mode.
.SP
.TP
.I -tlogfile $file
Turn on Netli logging with output to specified file.
.SP
.TP
.I -trelative {object | program}
Make Netli timings relative to the start of the first object or the program.
.SP
.TP
.I -transparent_proxy FQDN[:port]
When processing URL, send the original, but send it to the IP address
at FQDN
.SP
.TP
.I -transparent_ssl_proxy FQDN[:port]
When processing HTTPS URL, send the original, but send it to the IP address
at FQDN
.SP
.TP
.I -sdemo
Output in sdemo compatible format. This is only used by sdemo. (For
now it simply means output '-1' rather than '*' when measurements are
invalid.)
.SP
.TP
.I -noencode
Do not escape characters that are "unsafe" in URLS.
.SH Special start
.sp
.TP
.I -X
Start program with X Window interface (if compiled with support for GTK+).
Pavuk as default starts without GUI, and behaves as regular commandline tool.
.TP
.I -runX
When used together with the \fB-X\fR option, pavuk starts processing of
URLs immediately after the GUI window is launched. Without the \fB-X\fR
given, this option doesn't have any effect.
Only available when compiled with GTK+ support .
.TP
.I -bg/-nobg
This option allows pavuk to detach from its terminal and run in
background mode. Pavuk will not output any messages to the terminal
than. If you want to see messages, you have to use the
.B -log_file
option to specify a file where messages will be written.
Default pavuk executes at foreground.
.TP
.I -check_bg/-nocheck_bg
Normally, programs sent into the background after being run in
foreground continue to output messages to the terminal. If this
option is activated, pavuk checks if it is running as background job
and will not write any messages to the terminal in this case. After
it becomes a foreground job again, it will start writing messages to
terminal in the normal way. This option is available only when your
system supports retrieving of terminal info via \fBtc*()\fR functions.
.TP
.I -prefs/-noprefs
When you turn this option on, pavuk will preserve all settings when exiting, and
when you run pavuk with GUI interface again, all settings will be restored.
The settings will be stored in the
.B ~./pavuk_prefs
file. Default pavuk want restore its option when started.
This option is available only when compiled with GTK+.
.TP
.I -schedule $time
Execute pavuk at the time specified as parameter. The Format of the
$time parameter is YYYY.MM.DD.hh.mm. You need a properly configured
scheduling with the \fBat\fR command on your system for using this option.
If default configuration (at -f %f %t %d.%m.%Y) of scheduling command won't work on your system, try to adjust it with \fB-sched_cmd\fR option.
.TP
.I -reschedule $nr
Execute pavuk periodically with $nr hours period. You need properly
configured scheduling with the \fBat\fR command on your system for using this option.
.TP
.I -sched_cmd $str
Command to use for scheduling. Pavuk explicitly supports scheduling with
\fBat\fR $str should contain regular characters and macros, escaped by \fB%\fR character.
Supported macros are:
.br
.in +3
.B %f
- for script filename
.br
.B %t
- for time (in format HH:MM)
.br
- all macros as supported by the \fBstrftime()\fR function
.in -3
.TP
.I -urls_file $file
If you use this option, pavuk will read URLs from $file before it
starts processing. In this file, each URL needs to be on a separate
line. After the last URL, a single dot \fB.\fR followed by a LF
(line-feed) character denotes the end. Pavuk will start processing
right after all URLs have been read. If \fB$file\fR is given as the \fB-\fR
character, standard input will be read.
.TP
.I -store_info/-nostore_info
This option causes pavuk to store information about each document
into a separate file in the \fB.pavuk_info\fR
directory. This file is used to store the original URL from which
the document was downloaded. For files that are downloaded via
HTTP or HTTPS protocols, the whole HTTP response header is stored
there. I recommend to use this option when you are using options
that change the default layout of the local document tree, because
this info file helps pavuk to map the local filename to the
URL. This option is also very useful when different URLs have the
same filename in the local tree. When this occurs, pavuk detects
this using info files, and it will prefix the local name with
numbers. At default is disabled storing of this extra informations.
.TP
.I -info_dir $dir
You can set with this option location of separate directory for storing
info files created when \fB-store_info\fR option is used. This is useful
when you don't want to mix in destination directory the info files with
regular document files. The structure of the info files is preserved, just
are stored in different directory.
.TP
.I -request $req
With this option you can specify extended informations for starting URLs.
With this option you can specify query data for \fBPOST\fR or \fBGET\fR .
Current syntax of this option is :
\fBURL:["]$url["] [METHOD:["]{GET|POST}["]] [ENCODING:["]{u|m}["]] [FIELD:["]variable=value["]] [FILE:["]variable=filename["] [LNAME:["]local_filename["]]\fR
.sp
.RS
.nf
- \fBURL:\fR specifies request URL
- \fBMETHOD:\fR specifies request method for URL and is
one of \fIGET\fR or \fIPOST\fR.
- \fBENCODING:\fR specifies encoding for request body data.
\fBm\fR is for \fImultipart/form-data\fR encoding
\fBu\fR is for \fIapplication/x-www-form-urlencoded\fR
encoding
- \fBFIELD:\fR specifies field of request data in format
\fBvariable=value\fR. For encoding of special characters
in \fIvariable\fR and \fIvalue\fR you can use same encoding
as is used in \fBapplication/x-www-form-urlencoded\fR
encoding.
- \fBFILE:\fR specifies special field of query, which is
used to specify file for \fBPOST\fR based file upload.
- \fBLNAME:\fR specifies localname for this request
.fi
.RE
When you need to use inside the \fBFIELD:\fR and \fBFILE:\fR fields of
request specification special characters, you should use the
\fBapplication/x-www-form-urlencoded\fR encoding of characters. It means
all nonASCII characters, quote character ("), space character ( ),
ampersand character (&), percent character (%) and equal character (=)
should be encoded in form \fB%xx\fR where \fBxx\fR is hexadecimal
representation of ASCII value of character. So for example \fI%\fR character
should be encoded like \fI%25\fR.
.TP
.I -formdata $req
This option gives you chance to specify contents for HTML forms found during
traversing document tree.
Current syntax of this option is same as for \fB-request\fR option, but
\fBENCODING:\fR and \fBMETHOD:\fR are meaningless in this option semantics.
In \fBURL:\fR you have to specify HTML form action URL, which will be matched
against action URLs found in processed HTML documents. If pavuk finds action URL
which matches that supplied in \fB-formdata\fR option, pavuk will construct
\fBGET\fR or \fBPOST\fR request from data supplied in this option and from default
form field values supplied in HTML document. Values supplied on commandline have
precedence before that supplied in HTML file.
.TP
.I -nthreads $nr
By means of this option you can specify how many concurrent threads will
download documents. Default pavuk executes 3 concurrent downloading threads.
This option is available only when pavuk is compiled to support multithreading.
.TP
.I -immesg/-noimmesg
Default pavuks behavior when running multiple downloading threads is to buffer
all output messages in memory buffer and flush that buffered data just when thread
finishes processing of one document. With this option you can change this
behavior to see the messages immediately when it is produced. It is only usable
when you want to debug some specials in multithreading environment.
This option is available only when pavuk is compiled to support multithreading.
.TP
.I -dumpfd $nr
For scripting is sometimes usable to be able to download document directly to
pipe or variable instead of storing it to regular file. In such case you can use
this option to dump data for example to stdout ($nr = 1).
.TP
.I -dump_after/-nodump_after
While using \fB-dumpfd\fR option in multithreaded pavuk, it is required to dump
document in one moment because documents downloaded in multiple threads can
overlap. This option is also useful when you want to dump document after pavuk
adjusts links inside HTML documents.
.TP
.I -dump_response/-nodump_response
This option have effect only when used with \fB-dumpfd\fR option. It is used to dump HTTP response headers.
.TP
.I -dump_urlfd $nr
When you will use this option, pavuk will output all URLs found in HTML
documents to file descriptor $nr. You can use this option to extract and
convert all URLs to absolute.
.SH Scenario/Task options
.sp
.TP
.I -scenario $str
Name of scenario to load and/or run. Scenarios are files with a structure
similar to the \fB.pavukrc\fR file.
Scenarios contain saved configurations. You can use it for periodical
mirroring. Parameters from scenarios specified at the command line
can be overwritten by command line parameters.
To be able to use this option, you need to specify scenario base
directory with option \fB-scndir\fR.
.TP
.I -dumpscn $filename
Store actual configuration into scenario file with name \fB$filename\fR.
This is useful to quickly create pre-configured scenarios for manual
editing.
.SH Directory options
.sp
.TP
.I -msgcat $dir
Directory which contains the message catalog for pavuk.
If you do not have permission
to store a pavuk message catalog in the system directory, you should simply
create similar structure of directories in your home directory as it is on
your system.
.sp
.B For example:
.sp
Your native language is German, and your home directory is
/home/jano.
.sp
You should at first create the directory
/home/jano/locales/de/LC_MESSAGES/, then put the German pavuk.mo
there and set -msgcat to /home/jano/locales/. If you have properly
set locale environment values, you will see pavuk speaking German.
This option is available only when you compiled in support for GNU
gettext messages internationalization.
.TP
.I -cdir $dir
Directory where are all retrieved documents are stored. If not
specified, the current directory is used. If the specified directory doesn't
exist, it will be created.
.TP
.I -scndir $dir
Directory in which your scenarios are stored.
You must use this option when you are loading or storing scenario files.
.SH Preserve options
.sp
.TP
.I -preserve_time/-nopreserve_time
Store downloaded document with same modification time as on the remote site. Modification
time will be set only when such information is available (some FTP servers do not
support the \fBMDTM\fR command, and some documents on HTTP servers are created online so pavuk
can't retrieve the modification time of this document).
At default modification time of documents isn't preserved.
.TP
.I -preserve_perm/-nopreserve_perm
Store downloaded document with the same permissions as on the remote site.
This option has effect only when downloading a file through FTP protocol and
assumes that the \fB-ftplist\fR option is used. At default permissions are not
preserved.
.TP
.I -preserve_slinks/-nopreserve_slinks
Set symbolic links to point exactly to same location as on the remote server;
don't do any relocations.
This option has effect only when downloading file through FTP protocol and
assumes that the \fB-ftplist\fR option is used.
Default symbolic links are not preserved, and are retrieved as regular
documents with full contents of linked file.
.sp
For example, assume that on the FTP server ftp.xx.org there is a symbolic link
/pub/pavuk/pavuk-current.tgz, which points to /tmp/pub/pavuk-0.9pl11.tgz.
Pavuk will create symbolic link ftp/ftp.xx.org_21/pub/pavuk/pavuk-current.tgz
.br
if option -preserve_slinks will be used this symbolic link will point to
/tmp/pub/pavuk-0.9pl11.tgz
.br
if option -preserve_slinks want be used, this symbolic link will point to
../../tmp/pub/pavuk-0.9pl11.tgz
.TP
.I -retrieve_symlink/-noretrieve_symlink
Retrieve files behind symbolic links instead of replicating symlinks in local tree.
.SH Proxy options
.sp
.TP
.I -http_proxy $site[:$port]
If this parameter is used, then all HTTP requests are going through this proxy
server. This is useful if your site resides behind a firewall, or if you want
to use a HTTP proxy cache server. The default port number is 8080.
Pavuk allows you to specify multiple HTTP proxies
(using multiple -http_proxy options) and it will rotate proxies with roundrobin priority disabling proxies with errors.
.TP
.I -nocache/-cache
Use this option whenever you want to get the document directly from
the site and not from your HTTP proxy cache server. Default pavuk allows
transfer of document copies from cache.
.TP
.I -ftp_proxy $site[:$port]
If this parameter is used, then all FTP requests are going through this proxy
server.
This is useful when your site resides behind a firewall, or if you want to use
FTP proxy cache server. The default port number is 22.
Pavuk supports three different types of proxies for FTP, see the options
\fB-ftp_httpgw, -ftp_dirtyproxy.\fR
If none of the mentioned options is used, then pavuk assumes a regular
FTP proxy with \fBUSER user@host\fR connecting to remote FTP server.
.TP
.I -ftp_httpgw/-noftp_httpgw
The specified FTP proxy is a HTTP gateway for the FTP protocol. Default FTP proxy is regular FTP proxy.
.TP
.I -ftp_dirtyproxy/-noftp_dirtyproxy
The specified FTP proxy is a HTTP proxy which supports a \fBCONNECT\fR request
(pavuk should use full FTP protocol, except of active data connections).
Default FTP proxy is regular FTP proxy.
If both -ftp_dirtyproxy and -ftp_httpgw are specified, -ftp_dirtyproxy is preferred.
.
.TP
.I -gopher_proxy $site[:$port]
Gopher gateway or proxy/cache server.
.TP
.I -gopher_httpgw/-nogopher_httpgw
The specified Gopher proxy server is a HTTP gateway for Gopher protocol.
When \fB-gopher_proxy\fR is set and this \fB-gopher_httpgw\fR option isn't used,
pavuk is using proxy as HTTP tunnel with \fBCONNECT\fR request to open connections to Gopher servers.
.TP
.I -ssl_proxy $site[:$port]
SSL proxy (tunneling) server [as that in CERN httpd + patch or in
Squid] with enabled \fBCONNECT\fR request (at least on port 443). This
option is available only when compiled with SSL support (you need
the SSleay or OpenSSL libraries with development headers)
.SH Proxy Authentification
.sp
.TP
.I -http_proxy_user $user
Username for HTTP proxy authentification.
.TP
.I -http_proxy_pass $pass
Password for HTTP proxy authentification.
.TP
.I -http_proxy_auth {1/2/3/4/user/Basic/Digest/NTLM}
Authentification scheme for proxy access. Similar meaning as the
\fB-auth_scheme\fR option (see help for this option for more details).
Default is 2 (Basic scheme).
.TP
.I -auth_proxy_ntlm_domain $str
NT or LM domain used for authorization again HTTP proxy server when NTLM authentification scheme is required. This option is available only when compiled with OpenSSL or libdes libraries.
.TP
.I -auth_reuse_proxy_nonce/-noauth_reuse_proxy_nonce
When using HTTP Proxy Digest access authentification scheme use
first received nonce value in multiple following requests.
.TP
.I -ftp_proxy_user $user
Username for FTP proxy authentification.
.TP
.I -ftp_proxy_pass $pass
Password for FTP proxy authentification.
.SH Protocol/Download Options
.sp
.TP
.I -ftp_passive
Uses passive ftp when downloading via ftp.
.TP
.I -ftp_active
Uses active ftp when downloading via ftp.
.TP
.I -active_ftp_port_range $min:$max
This option permits to specify the ports used for
active ftp. This permits easier firewall configuration since the range
of ports can be restricted.
.sp
Pavuk will randomly choose a number from within the specified
range until an open port is found. Should no open ports be found
within the given range, pavuk will default to a normal
kernel-assigned port, and a message (debug level net) is output.
.sp
The port range selected must be in the non-privileged range
(eg. greater than or equal to 1024); it is STRONGLY RECOMMENDED that
the chosen range be large enough to handle many simultaneous
active connections (for example, 49152-65534, the IANA-registered
ephemeral port range).
.TP
.I -always_mdtm/-noalways_mdtm
Force pavuk to always use "MDTM" to determine the file modification
time and never uses cached times determined when listing the remote
files.
.TP
.I -remove_before_store/-noremove_before_store
Force unlink'ing of files before new content is stored to a file. This
is helpful if the local files are hardlinked to some other directory
and after mirroring the hardlinks are checked. All "broken" hardlinks
indicate a file update.
.TP
.I -retry $nr
Set the number of attempts to transfer processed document.
Default set to 1, this mean pavuk will retry once to get documents
which failed on first attempt.
.TP
.I -nregets $nr
Set the number of allowed regets on a single document, after a broken transfer.
Default value for this option is 2.
.TP
.I -nredirs $nr
Set number of allowed HTTP redirects. (use this for prevention of loops)
Default value for this option is 5, and conform to HTTP specification.
.TP
.I -force_reget/-noforce_reget
Force reget'ing of the whole document after a broken transfer when
the server doesn't support retrieving of partial content.
Pavuk default behavior is to stop getting documents which don't allow
restarting of transfer from specified position.
.TP
.I -timeout $nr
Timeout for stalled connections in minutes. This value is also used for
connection timeouts. For sub-minute timeouts you can use floating point numbers.
Default timeout is 0, an that means timeout checking is disabled.
.TP
.I -noRobots/-Robots
This switch suppresses the use of the \fBrobots.txt\fR
standard, which is used to restrict access of Web robots to some
locations on the web server. Default is allowed checking of robots.txt
files on HTTP servers. Enable this option always when you are downloading
huge sets of pages with unpredictable layout.
This prevents you from upsetting server administrators :-).
.TP
.I -noEnc/-Enc
This switch suppresses using of \fBgzip\fR or \fBcompress\fR or \fBdeflate\fR
encoding in transfer. I don't know if some servers are broken or what, but they
are propagating that MIME type application/gzip or application/compress as
encoded. Turn this option off, when you doesn't have libz support compiled in
and also \fBgzip\fR program which is used to decode document encoded this way.
At default is decoding of downloaded document disabled.
.TP
.I -check_size/-nocheck_size
The option -nocheck_size should be used if you are trying to
download pages from a HTTP server which sends a wrong
\fBContent-Length:\fR field in the MIME header of response.
Default pavuk behavior is to check this field and complain
when something is wrong.
.TP
.I -maxrate $nr
If you don't want to give all your transfer bandwidth to pavuk, use
this option to set pavuk's maximum transfer rate. This option
accepts a floating point number to specify the transfer rate in kB/s. If
you want get optimal settings, you also have to play with the size of the read
buffer (option \fB-bufsize\fR) because pavuk is doing flow control only at
application level.
At default pavuk use full bandwidth.
.TP
.I -minrate $nr
If you hate slow transfer rates, this option allows you to break
transfers with slow speed. You can set the minimum transfer rate,
and if the connection gets slower than the given rate, the transfer
will be stopped. The minimum transfer rate is given in kB/s.
At default pavuk doesn't check this limit.
.TP
.I -bufsize $nr
This option is used to specify the size of the read buffer (default
size: 32kB). If you have a very fast connection, you may increase
the size of the buffer to get a better read performance. If you need
to decrease the transfer rate, you may need to decrease the size of
the buffer and set the maximum transfer rate with the \fB-maxrate\fR
option. This option accepts the size of the buffer in kB.
.TP
.I -fs_quota $nr
If you are running pavuk on a multiuser system, you may need to
avoid filling up your file system. This option lets you specify how
many space must remain free. If pavuk detects an underrun of the
free space, it will stop downloading files. Specify this quota in
kB. Default value is 0, and that mean no checking of this quota.
.TP
.I -file_quota $nr
This option is useful when you want to limit downloading of big files,
but want to download at least $nr kilobytes from big files.
A big file will be transferred, and when it reaches the specified size,
transfer will break. Such document will be processed as properly downloaded,
so be careful when using this option.
At default pavuk is transferring full size of documents.
.TP
.I -trans_quota $nr
If you are aware that your selection should address a big amount of
data, you can use this option to limit the amount of transferred data.
Default is by size unlimited transfer.
.TP
.I -max_time $nr
Set maximum amount of time for program run. After time is exceeded, pavuk
will stop downloading. Time is specified in minutes. Default value is 0, and it means downloading time is not limited.
.TP
.I -url_strategy $strategy
This option allows you to specify a downloading order for URLs in document tree.
This option accepts the following strings as parameters :
.br
.sp
.RS
.B level
- will order URLs as it loads it from HTML files (default)
.br
.B leveli
- as previous, but inline objects URLs come first
.br
.B pre
- will insert URLs from actual HTML document at start, before other
.br
.B prei
- as previous, but inline objects URLs come first
.br
.RE
.TP
.I -send_if_range/-nosend_if_range
Send \fBIf-Range:\fR header in HTTP request. I found out, that some HTTP
servers (greetings, MS :-)) are sending different \fBETag:\fR
fields in different responses for the same, unchanged
document. This causes problems when pavuk attempts to reget a
document from such a server: pavuk will remember the old ETag value and
uses it it following requests for this document.
If the server checks it with the new ETag value and it differs,
it will refuse to send only part of the document, and start the download
from scratch.
.TP
.I -ssl_version $v
Set required SSL protocol version for SSL communication.
\fB$v\fR is one of ssl2, ssl23, ssl3 or tls1.
This option is available only when compiled with SSL support.
Default is ssl23.
.TP
.I -unique_sslid/-nounique_sslid
This option can be used if you want to use a unique \fBSSL ID\fR for all
SSL sessions. Default pavuk behavior is to negotiate each time new session
ID for each connection.
This option is available only when compiled with SSL support.
.TP
.I -use_http11/-nouse_http11
This option is used to switch between HTTP/1.0 and HTTP/1.1 protocol
used with HTTP servers. Now is using of HTTP/1.1 protocol not default
because its implementation is very fresh and not 100% tested. Even though
using of HTTP/1.1 is very recommended, because it is faster than HTTP/1.0
and uses less network bandwidth for initiating connections. In any further
version I will activate using of HTTP/1.1 as default.
.TP
.I -local_ip $addr
You can use this option when you want to use specified network interface
for communication with other hosts. This option is suitable for multihomed
hosts with several network interfaces. Address should be entered as regular
IP address or as host name.
.TP
.I -identity $str
This option allows you to specify content of \fBUser-Agent:\fR field of HTTP request.
This is usable, when scripts on remote server returns different document on same
URL for different browsers, or if some HTTP server refuse to serve document
for Web robots like pavuk. Default pavuk sends in \fBUser-Agent:\fR field
\fBpavuk/$VERSION\fR string.
.TP
.I -auto_referer/-noauto_referer
This option forces pavuk to send HTTP \fBReferer:\fR header field with starting URLs.
Content of this field will be self URL. Using this option is required,
when remote server checks the Referer: field.
At default pavuk wont send Referer: field with starting URLs.
.TP
.I -referer/-noreferer
This option allows to enable and disable the transmission of HTTP \fBReferer:\fR
header field. At default pavuk sends Referer: field.
.TP
.I -httpad $str
In some cases you may want to add user defined fields to HTTP/HTTPS requests.
This option is exactly for this purpose. In \fB$str\fR you can directly specify
content of additional header. If you specify only raw header, it will be used
only for starting requests. When you want to use this header with each request
while crawling, prefix the header with \fB+\fR character.
.TP
.I -del_after/-nodel_after
This option allows you to delete FILES from REMOTE server, when download is
properly finished. At default is this option off.
.TP
.I -FTPlist/-noFTPlist
When option -FTPlist will be used, pavuk will retrieve content of FTP
directories with FTP command \fBLIST\fR instead of \fBNLST\fR. So the same listing will be
retrieved as with "ls -l" UNIX command.
This option is required if you need to preserve permissions of remote files or
you need to preserve symbolic links.
Pavuk supports wide listing on FTP servers with regular \fBBSD\fR or \fBSYSV\fR style "ls -l"
directory listing, on FTP servers with \fBEPFL\fR listing format, \fBVMS\fR style listing,
\fBDOS/Windows\fR style listing and \fBNovel\fR listing format.
Default pavuk behavior is to use NLST fro FTP directory listings.
.TP
.I -ftp_list_options $str
Some FTP servers require to supply extra options to LIST or NLST FTP commands
to show all files and directories properly. But be sure not to use any extra
options which can reformat output of the listing. Useful is especially \fB-a\fR
option which force FTP server to show also dot files and directories and
with broken WuFTP servers it also helps to produce full directory listings
not just files.
.TP
.I -fix_wuftpd/-nofix_wuftpd
This option is result of several attempts to to get working properly the
\fB-remove_old\fR option with WuFTPd server when \fB-ftplist\fR option is
used. The problem is that FTP command LIST on WuFTPd don't mind when trying
to list nonexisting directory, and indicates success in FTP response code.
When you activate this option, pavuk uses extra FTP command (STAT -d dir)
to check whether the directory really exists. Don't use this option until
you are sure that you really need it!
.SH Authentification
.sp
.TP
.I -auth_file $file
File where you have stored authentification information for access
to some service. For file structure see below in \fBFILES\fR section.
.TP
.I -auth_name $user
If you are using this parameter, program is doing authentification with each HTTP
access to document. Use this only if you know that only one HTTP server could be
accessed or use \fB-asite\fR option to specify site to which you use
authentification. Else your auth parameters will be sent to each accessed
HTTP server.
.TP
.I -auth_passwd $passwd
Value of this parameter is used as password for authentification
.TP
.I -auth_scheme {1/2/3/4/user/Basic/Digest/NTLM}
This parameter specifies used authentification scheme.
.br
.B 1 or user
means
.B user
authentification scheme is used as defined in HTTP/1.0 or HTTP/1.1.
Password and user name are sent unencoded.
.br
.B 2 or Basic
means
.B Basic
authentification scheme is used as defined in HTTP/1.0.
Password and user name are sent BASE64 encoded.
.br
.B 3 or Digest
means
.B Digest
access authentification scheme based on MD5 checksums as defined in RFC2069.
.br
.B 4 or NTLM
means
.B NTLM
proprietary access authentification scheme used by Microsoft IIS or Proxy servers.
When you use this scheme, you must also specify NT or LM domain with option \fB-auth_ntlm_domain\fR. This scheme is supported only when compiled with OpenSSL or libdes libraries.
.TP
.I -auth_ntlm_domain $str
NT or LM domain used for authorization again HTTP server when NTLM authentification scheme is required. This option is available only when compiled with OpenSSL or libdes libraries.
.TP
.I -auth_reuse_nonce/-noauth_reuse_nonce
While using HTTP Digest access authentification scheme use first received nonce
value in more following requests.
Default pavuk negotiates nonce for each request.
.TP
.I -ssl_key_file $file
File with public key for SSL certificate (learn more from SSLeay or OpenSSL documentation)
This option is available only when compiled with SSL support (you need SSleay or OpenSSL libraries and development headers)
.TP
.I -ssl_cert_file $file
Certificate file in PEM format (learn more from SSLeay or OpenSSL documentation)
This option is available only when compiled with SSL support (you need SSleay or OpenSSL libraries and development headers)
.TP
.I -ssl_cer_passwd $str
Password used to generate certificate (learn more from SSLeay or OpenSSL documentation)
This option is available only when compiled with SSL support (you need SSLeay or OpenSSL libraries and development headers)
.TP
.I -nss_cert_dir $dir
Config directory for NSS (Netscape SSL implementation) certificates. Usually
~/.netscape (created by Netscape communicator/navigator) or profile directory
below ~/.mozilla (created by Mozilla browser). The directory should contain
\fBcert7.db\fR and \fBkey3.db\fR files. If you don't use Mozilla nor
Netscape, you must create this files by utilities distributed with NSS
libraries. Pavuk opens certificate database only readonly.
This option is available only when pavuk is compiled with SSL support
provided by Netscape NSS SSL implementation.
.TP
.I [-nss_accept_unknown_cert/-nonss_accept_unknown_cert]
By default will pavuk reject connection to SSL server which certificate is not
stored in local certificate database (set by \fB-nss_cert_dir\fR option).
You must explicitly force pavuk to allow connection to servers with unknown
certificates.
This option is available only when pavuk is compiled with SSL support
provided by Netscape NSS SSL implementation.
.TP
.I [-nss_domestic_policy/-nss_export_policy]
Selects sets of ciphers allowed/disabled by USA export rules.
This option is available only when pavuk is compiled with SSL support
provided by Netscape NSS SSL implementation.
.TP
.I -from $email
This parameter is used when accessing anonymous FTP server as password or is
optionally inserted into \fBFrom\fR field in HTTP request. If not specified
pavuk discovers this from \fBUSER\fR environment variable and from site
hostname.
.TP
.I -send_from/-nosend_from
This option is used for enabling or disabling sending of user identification,
entered in \fB-from\fR
option, as FTP anonymous user password and \fBFrom:\fR field of HTTP request.
As default is this option off.
.TP
.I -ftp_login_handshake $host $handshake
When you need to use nonstandard login procedure for some of FTP servers,
you can use this option to change default pavuk login procedure. To allow
more flexibility, you can assign the login procedure to some server or to
all. When \fI$host\fR is specified as empty string (\fI""\fR), than attached
login procedure is assigned to all FTP servers besides those having assigned
own login procedures. In the \fI$handshake\fR parameter you can specify exact
login procedure specified by FTP commands followed by expected FTP response
codes delimited with backslash (\fI\\\fR) characters.
.br
For example this is
default login procedure when logging in regular ftp server without going
through proxy server : \fIUSER %u\\331\\PASS %p\\230\fR. There are two
commands followed by two response codes. After USER command pavuk expects
FTP response code 331 and after PASS command pavuk expects from server FTP
response code 230. In ftp commands you can use following macros which will be
replaced by respective values:
.sp
.br
\fI%u\fR - user name used to access FTP server
.br
\fI%p\fR - password used to access FTP server
.br
\fI%U\fR - user name used to access FTP proxy server
.br
\fI%P\fR - password used to access FTP proxy server
.br
\fI%h\fR - hostname of FTP server
.br
\fI%s\fR - port number on which FTP server listens
.SH Site/Domain/Port Limitation Options
.sp
.TP
.I -asite $list
Specify comma separated list of allowed sites on which referenced documents
are stored.
.TP
.I -dsite $list
Specify comma separated list of disallowed sites.
Previous parameter is
opposite to this one. If both are used the last occurrence of them is used to
be valid.
.TP
.I -adomain $list
Specify comma separated list of allowed domains on which referenced documents
are stored.
.TP
.I -ddomain $list
Specify comma separated list of disallowed domains. Previous parameter is
opposite to this one. If both are used the last occurrence of them is used to
be valid.
.TP
.I -aport $list
In \fB$list\fR, you can write comma separated list of ports from which you
allow to download documents.
.TP
.I -dport $list
This option is opposite option to previous option. It is used to specify
denied ports. If both \fB-aport\fR and \fB-dport\fR options are used the
last occurrence of them is used to be valid and all other occurrences will
be omitted.
.SH Limitation Document properties
.sp
.TP
.I -amimet $list
List of comma separated allowed MIME types. You can use with this option also wildcard patterns.
.TP
.I -dmimet $list
List of comma separated disallowed MIME types. You can use with this option also wildcard patterns.
Previous parameter is opposite to this one. If both are used the last occurrence of them is used to be valid.
.TP
.I -maxsize $nr
Maximum allowed size of document.
This option is applied only when pavuk is able to detect the document before starting the transfer.
Default value is 0, and it means this limit isn't applied.
.TP
.I -minsize $nr
minimal allowed size of document.
This option is applied only when pavuk is able to detect the document before starting the transfer.
Default value is 0, and it means this limit isn't applied.
.TP
.I -newer_than $time
Allow only transfer of documents with modification time newer than specified in parameter $time. Format of $time is: YYYY.MM.DD.hh:mm.
To apply this option pavuk must be able to detect modification time of document.
.TP
.I -older_than $time
Allow only transfer of documents with modification time older than specified in parameter $time. Format of $time is: YYYY.MM.DD.hh:mm.
To apply this option pavuk must be able to detect modification time of document.
.TP
.I -noCGI/-CGI
this switch prevents to transfer dynamically generated parametric documents
through CGI interface. This is detected with occurrence of \fB?\fR character inside URL.
Default pavuk behavior is to allow transfer of URLs with query strings.
.TP
.I -alang $list
this allows you to specify ordered comma separated list of preferred natural
languages. This option work only with HTTP and HTTPS protocol using
\fBAccept-Language:\fR MIME field.
.TP
.I -acharset $list
This options allows you to enter comma separated list of preferred encoding of
transfered documents. This works only with HTTP and HTTPS urls and only if such
document encodings are located on destination server.
.br
.B example: -acharset iso-8859-2,windows-1250,utf8
.SH Limitation Document name
.sp
.TP
.I -asfx $list
This parameter allows you to specify set of suffixes used to restrict selection
of documents which will be processed.
.TP
.I -dsfx $list
Set of suffixes that are used to specify restriction on selection of documents.
This one is inverse to previous option. They are segregating each other.
.TP
.I -aprefix $list, -dprefix $list
This two options allow you to specify set of allowed or disallowed prefixes
of documents. They are segregating each other.
.TP
.I -pattern $pattern
This option allows you to specify wildcard pattern for documents. All documents
are tested if they match this pattern.
.TP
.I -rpattern $reg_exp
This is equal option as previous, but this uses regular expressions.
Available only on platforms which have any supported RE implementation.
.TP
.I -skip_pattern $pattern
This option allows you to specify wildcard pattern for documents that should be skipped.
All documents are tested if they match this pattern.
.TP
.I -skip_rpattern $reg_exp
This is equal option as previous, but this uses regular expressions.
Available only on platforms which have any supported RE implementation.
.TP
.I -url_pattern $pattern
This option allows you to specify wildcard pattern for URLs. All URLs
are tested if they match this pattern.
.br
.B Example:
.br
-url_pattern http://\\*.idata.sk:\\*/~ondrej/\\* . this option enables all HTTP URLs from domain .idata.sk on all ports which are located under /~ondrej/.
.TP
.I -url_rpattern $reg_exp
This is equal option as previous, but this uses regular expressions.
Available only on platforms which have any supported RE implementation.
.TP
.I -skip_url_pattern $pattern
This option allows you to specify wildcard pattern for URLs that should be skipped.
All URLs are tested if they match this pattern.
.TP
.I -skip_url_rpattern $reg_exp
This is equal option as previous, but this uses regular expressions.
Available only on platforms which have any supported RE implementation.
.TP
.I -aip_pattern $re
This option allows you to limit set of transferred documents by server IP address.
IP address can be specified as regular expressions, so it is possible to specify
set of IP addresses by one expression.
Available only on platforms which have any supported RE implementation.
.TP
.I -dip_pattern $re
This option similar to previous option, but is used to specify set of disallowed
IP addresses.
Available only on platforms which have any supported RE implementation.
.TP
.I -tag_pattern $tag $attrib $url
More powerful version of \fI-url_pattern\fR option for more precise matching
of allowed URLs based on HTML tag name pattern, HTML tag attribute name pattern
and on URL pattern. You can use in all three parameters of this option wildcard
patterns, thus something like \fB-tag_pattern '*' '*' url_pattern\fR is equal
to \fB-url_pattern url_pattern\fR. The \fI$tag\fR and \fI$attrib\fR parameters
are always matched again uppercase strings. For example if you want just let
pavuk follow only regular links ignoring any stylesheets, images, etc.,
use option \fB-tag_pattern A HREF '*'\fR.
.TP
.I -tag_rpattern $tag $attrib $url
This is variation on the \fI-tag_pattern\fR. It uses regular expression
patterns in parameters instead of wildcard patterns used in the previous option.
.SH Limitation Protocol Option
.sp
.TP
.I -noHTTP/-HTTP
This switch suppresses all transfers through HTTP protocol.
Default is transfer trough HTTP enabled.
.TP
.I -noSSL/-SSL
This switch suppresses all transfers through HTTPS protocol (HTTP protocol over SSL) .
Default is transfer trough HTTPS enabled.
This option is available only when compiled with SSL support (you need SSleay or OpenSSL libraries and development headers)
.TP
.I -noGopher/-Gopher
Suppress all transfers through Gopher Internet protocol.
Default is transfer trough Gopher enabled.
.TP
.I -noFTP/-FTP
This switch prevents processing documents allocated on all FTP servers.
Default is transfer trough FTP enabled.
.TP
.I -noFTPS/-FTPS
This switch prevents processing documents allocated on all FTP servers accessed through SSL.
Default is transfer trough FTPS enabled.
This option is available only when compiled with SSL support (you need SSleay or OpenSSL libraries and development headers)
.TP
.I -FTPhtml/-noFTPhtml
By using of option -FTPhtml you can force pavuk to process HTML files downloaded with FTP protocol.
At default pavuk won't parse HTML files from FTP servers.
.TP
.I -FTPdir/-noFTPdir
Force recursive processing of FTP directories too.
At default is recursive downloading from FTP servers denied.
.TP
.I -disable_html_tag $TAG,[$ATTRIB][;...]
.I -enable_html_tag $TAG,[$ATTRIB][;...]
Enable or disable processing of particular HTML tags or attributes.
At default all supported HTML tags are enabled.
.sp
For example if you don't want to process all images you should use option
.B -disable_html_tag 'IMG,SRC;INPUT,SRC;BODY,BACKGROUND' .
.SH Other Limitation Options
.sp
.TP
.I -subdir $dir
Subdirectory of local tree directory, to limit some of the modes
{sync, resumeregets, linkupdate} in its tree scan.
.TP
.I -dont_leave_site/-leave_site
(Don't) leave starting site. At default pavuk can span host when recursing through WWW tree.
.TP
.I -dont_leave_dir/-leave_dir
(Don't) leave starting directory. If -dont_leave_dir option is used
pavuk will stay only in starting directory (including its own subdirectories).
At default pavuk can leave starting directories.
.TP
.I -leave_site_enter_dir/-dont_leave_site_enter_dir
If you are downloading WWW tree which spans multiple hosts with huge trees, you may want to allow downloading of document which are in directory hierarchy below directory which we visited as first on each site. To obtain this, use option -dont_leave_site_enter_dir. As default pavuk will go also to higher directory levels on that site.
.TP
.I -lmax $nr
Set maximum allowed level of tree traverse. Default is set to 0,
what means that pavuk can traverse at infinitum.
As of version 0.8pl1 inline objects of HTML pages are placed at same
level as parent HTML page.
.TP
.I -leave_level $nr
Maximum level of documents outside from site of starting URL.
Default is set to 0, and 0 means that checking is not applied.
.TP
.I -site_level $nr
Maximum level of sites outside from site of starting URL.
Default is set to 0, and 0 means that checking is not applied.
.TP
.I -dmax $nr
Set maximum allowed number of documents that are processed.
Default value is 0.
That means no restrictions are used in number of processed documents.
.TP
.I -singlepage/-nosinglepage
Using option \fB-singlepage\fR allows you to transfer just HTML pages with all its
inlined objects (pictures, sounds, frame documents, ...).
As default is disabled single page transfer. This option makes \fB-mode singlepage\fR
option obsolete.
.TP
.I -limit_inlines/-dont_limit_inlines
With this option you can control whether limiting options apply also to inline
objects (pictures, sounds, ...). This is useful when you want to download
specified set of HTML pages with all inline options without any restrictions.
.TP
.I -user_condition $str
Script or program name for users own conditions.
You can write any script which should with exit value decide if download URL or not.
Script gets from pavuk any number of options, with this meaning :
.in +3
.sp
.B -url $url
- processed URL
.br
.B -parent $url
- any number of parent URLs
.br
.B -level $nr
- level of this URL from starting URL
.br
.B -size $nr
- size of requested URL
.br
.B -date $datenr
- modification time of requested URL in format
.I YYYYMMDDhhmmss
.br
.in -3
.sp
The exit status 0 of script or program means that current URL should be rejected and nonzero exit status means that URL should be accepted.
.br
.B Warning :
use user conditions only if required because of big slowdowns
caused by forking scripts for each checked URL.
.TP
.I -follow_cmd $str
This option allows you to specify script or program which can by its exit status decide whether to follow URLs from current HTML document. This script will be called after download of each HTML document.
The script will get following options as it's parameters:
.in +3
.sp
.B -url $url
- URL of current HTML document
.br
.B -infile $file
- local file where is stored HTML document
.in -3
.sp
The exit status 0 of script or program means that URLs from current document will be disallowed, other exit status means, that pavuk can follow links from current HTML document.
.SH Javascript support
Support for scripting languages like JavaScript or VBScript in pavuk is done bit hacky way. There is no interpreter for this languages, so not all things will work. Whole support which pavuk have for this scripting languages is based on regular expression patterns specified by user. Pavuk search for this patterns in DOM event attributes of HTML tags, in javascript:... URLs, in inline scripts in HTML documents enclosed between tags and in separate javascript files.
Support for scripting languages is only available when pavuk is compiled with proper regular expression library (POSIX/GNU/PCRE).
.TP
.I -enable_js/-disable_js
This options are used to enable or disable processing of Javascript parts of HTML documents. You must enable this option to be able to use processing of javascript patterns.
.TP
.I -js_pattern $re
With this option you are specifying what patterns match interested parts of Javascript for extracting URLs. The parameter must be RE pattern with exactly one subpattern which match exactly the URL part. For example to match URL in following type of javascript expressions :
.br
document.b1.src='pics/button1_pre.jpg'
.br
you can use this pattern
.br
"^document\.[a-zA-Z0-9_]*\.src[ \t]*=[ \t]*'(.*)'$"
.TP
.I -js_transform $p $t $h $a
This option is similar to previous, but you can use custom transform rules for the URL parts of patterns and also specify the exact HTML tag and attribute where to look for this pattern. The \fB$p\fR is the pattern to match the interested part of script. The \fB$t\fR is transform rule for the URL, in this parameter the \fB$x\fR parts will be replaced by x-th subpattern of the \fB$p\fR pattern. The \fB$h\fR parameter is exact HTML tag or "*" when this apply to javascript: URLs or DOM event attribs or "" (empty string) when this apply to javascript body of HTML document or separate JS file. The \fB$a\fR parameter is exact HTML attrib of tag or "" (empty string) when this rule apply to javascript body.
.TP
.I -js_transform2 $p $t $h $a
This option is very similar to previous. The meaning of all parameters is same,
just the pattern \fB$p\fR can have only one substring which will be used in
the transform rule \fB$t\fR. This is required to allow rewriting of URL parts
of the tags and scripts. This option can also be used to force pavuk to
recognize HTML targ/attribute pairs which pavuk does not support.
.SH Cookie
.sp
.TP
.I -cookie_file $file
File where are stored cookie infos. This file must be in Netscape cookie file format (generated with Netscape Navigator or Communicator ...).
.TP
.I -cookie_send/-nocookie_send
Use collected cookies in HTTP/HTTPS requests.
Pavuk will not send at default cookies.
.TP
.I -cookie_recv/-nocookie_recv
Store received cookies from HTTP/HTTPS responses into memory cookie cache.
At default pavuk will not remember received cookies.
.TP
.I -cookie_update/-nocookie_update
Update cookie file on disk and synchronize it with changes made by any concurrent processes.
At default pavuk will not update cookie file on disk.
.TP
.I -cookies_max $nr
Maximum number of cookies in memory cookie cache.
Default value is 0, and that means no restrictions for cookies number.
.TP
.I -disabled_cookie_domains $list
Comma-separated list of cookie domains which are permitted to send cookies stored into cookie cache
.TP
.I -cookie_check/-nocookie_check
Check when receiving cookie, if cookie domain is equal to domain of server which sends this cookie. At default pavuk check is server is setting cookies for its domain, and if it tries to set cookie for foreign domain pavuk will complain about that and will reject such cookie.
.SH HTML rewriting engine tuning options
.sp
.TP
.I -noRelocate/-Relocate
This switch prevents the program to rewrite relative URLs to absolute, after
HTML document is transfered. Default pavuk behavior is to maintain link
consistence of HTML documents. So always when HTML document is downloaded pavuk
will rewrite all URLs to point to local document if it is available and if it
is not available it will point to remote document. After document is properly
downloaded, pavuk will update links in HTML documents, which point to this one.
.TP
.I -all_to_local/-noall_to_local
This option forces pavuk to change all URLs inside HTML document to local URLs
immediately after download of document. Default is this option disabled.
.TP
.I -sel_to_local/-nosel_to_local
This option forces pavuk to change all URLs, which accomplish conditions for download,
to local inside HTML document immediately after download of document.
I recommend to use this option, when you are sure, that transfer will be
without any problems. This option can save a lot of processor time.
Default is this option disabled.
.TP
.I -all_to_remote/-noall_to_remote
This option forces pavuk to change all URLs inside HTML document to remote URLs
immediately after download of document.
Default is this option disabled.
.TP
.I -post_update/-nopost_update
This option is especially designed to allow in \fB-fnrules\fR option
doing rules based on MIME type of document. This option forces pavuk
to generate local names for documents just after pavuk knows what is
the MIME type of document. This have big impact on the rewriting engine
of links inside HTML documents. This option causes disfunction of other
options for controlling the link rewriting engine. Use this option only
when you know what you are doing :-)
.TP
.I -dont_touch_url_pattern $pat
This options serves to deny rewriting and processing of particular URLs
in HTML documents by pavuk HTML rewriting engine. This option accepts
wildcard patterns to specify such URLs. Matching is done against untouched
URLs so when he URL is relative, you must use pattern which matches the
relative URL, when it is absolute, you must use absolute URL.
.TP
.I -dont_touch_url_rpattern $pat
This option is variation on previous option. This one uses regular patterns
for matching of URLs instead of wildcard patterns used by
\fB-dont_touch_url_pattern\fR option. This option is available only when pavuk
is compiled with support for regular expression patterns.
.TP
.I -dont_touch_tag_rpattern $pat
This option is variation on previous option, just matching is made on full HTML
tag with included <>. This option accepts regular expression patterns. It is
available only when pavuk is compiled with support for regular expression
patterns.
.SH Filename/URL Conversion Option
.sp
.TP
.I -tr_del_chr $str
All characters found in \fB$str\fR will be deleted from local name of document.
\fB$str\fR should contain escape sequences similar like in tr command:
.br
\fB\\n\fR
- newline
.br
\fB\\r\fR
- carriage return
.br
\fB\\t\fR
- horizontal tab space
.br
\fB\\0xXX\fR
- hexadecimal ASCII value
.br
.B [:upper:]
- all uppercase letters
.br
.B [:lower:]
- all lowercase letters
.br
.B [:alpha:]
- all letters
.br
.B [:alnum:]
- all letters and digits
.br
.B [:digit:]
- all digits
.br
.B [:xdigit:]
- all hexadecimal digits
.br
.B [:space:]
- all horizontal and vertical whitespace
.br
.B [:blank:]
- all horizontal whitespace
.br
.B [:cntrl:]
- all control characters
.br
.B [:print:]
- all printable characters including space
.br
.B [:nprint:]
- all non printable characters
.br
.B [:punct:]
- all punctation characters
.br
.B [:graph:]
- all printable characters excluding space
.TP
.I -tr_str_str $str1 $str2
String \fB$str1\fR from local name of document will be replaced with \fB$str2\fR.
.TP
.I -tr_chr_chr $chrset1 $chrset2
Characters from \fB$chrset1\fR from local name of document will be replaced with
corresponding character from \fB$chrset2\fR. \fB$charset1\fR and \fB$charset2\fR should have
same syntax as \fB$str\fR in \fB-tr_del_chr\fR option.
.TP
.I -store_name $str
When you want to change local filename of first file downloaded with
singlepage mode, you should use this option.
.TP
.I -index_name $str
With this option you can change directory index name. As default is used \fB_._.html\fR .
.TP
.I -store_index/-nostore_index
With option -nostore_index you should deny storing of directory indexes into HTML files.
.TP
.I -fnrules $t $m $r
This is a very powerful option! This option is used to flexible change layout
of local document tree. It accepts three parameters. First parameter \fB$t\fR is
used to say what type is following pattern.
.B F
is used for wildcard pattern (uses \fIfnmatch()\fR) and
.B R
is used for regular expression pattern (using any supported RE implementation).
Second parameter is matching pattern used to select URLs for this rule.
If URL match this pattern, then local name for this URL is computed following
rules of third parameter.
And third parameter is local name building rule. Pavuk now supports two kinds
of local name building rules. One is simple based only on \fIsimple\fR macros and
other more complicated \fIextended\fR rule, which also enables to perform several
functions.
Recognition between those two kinds of rules is done by looking at first
character of rule.
In case when first character is \fB'('\fR, rule is extended
and in all other cases it is the simple kind of rule.
.sp
\fBSimple rule\fR should contain literals or escaped macros.
Macros are escaped by % character or by $ character.
.sp
.B Here is list of recognized macros:
.sp
.sp
.br
.B $x
- where x is any positive number. This macro is replaced with x-th substring
matched by RE pattern. (If you use this you need to understand RE !)
.br
.B %i
- is replaced with protocol id (http, https, ftp, gopher)
.br
.B %p
- is replaced with password. (use this only when usable)
.br
.B %u
- is replaced with username.
.br
.B %h
- is replaced with host name.
.br
.B %m
- is replaced with domain name.
.br
.B %r
- is replaced with port number.
.br
.B %d
- is replaced with path to document.
.br
.B %n
- is replaced with document name.
.br
.B %b
- is replaced with basename of document (without extension).
.br
.B %e
- is replaced with extension.
.br
.B %s
- is replaced with searchstring.
.br
.B %M
- is replaced with MIME type of document. When you are using this macro, you *must* use also \fB-post_update\fR option else it won't work.
.br
.B %E
- is replaced with default extension assigned to MIME type of document. When you are using this macro, you *must* use also \fB-post_update\fR option else it won't work.
.br
.B %x
- where x is positive number. This macro is replaced with x-th directory from path to document from beginning.
.br
.B %-x
- where x is positive number. This macro is replaced with x-th directory from path to document from end.
.br
.sp
Here is example. If you want place document into single directories by extension,
you should use following fnrules option:
.br
.B -fnrules F '*' '/%e/%n'
.sp
\fBExtended rule\fR ever begins with character \fB\'('\fR. It uses some kind
of \fILISP\fR like syntax.
.sp
.B Here are base rules for writing extended rules :
- the local filename of of this kind is return value function
.br
- each function is enclosed inside round braces \fB()\fR
.br
- first token right after opening brace is function name
.br
- each function have nonzero fixed number of parameters
.br
- each function returns numeric or string value
.br
- function parameters are separated by any number of space characters
.br
- parameter of function should be string, number, macro or other function
.br
- string is ever quoted with \fB"\fR
.br
- each numeric parameter can be in any encoding supported by
\fIstrtod()\fR function (octal, decimal, hexadecimal, ...)
.br
- there is no implicit conversion from number to string
.br
- each macro is prefixed by \fB%\fR character and is one character long
.br
- each macro is replaced by its string representation from current URL
.br
- function parameters are typed strictly
.br
- toplevel function must return string value
.br
.sp
Extended rule supports full set of \fI%\fR escaped macros supported with
simple rules, plus two following addition macros :
.br
\fB%U\fR - URL string
.br
\fB%o\fR - default localname for URL
.sp
.B Here is description of all supported functions
.sp
\fBsc\fR - concat two string parameters
- accepts two string parameters
- returns string value
.br
\fBss\fR - substring form string
- accepts three parameters.
- first is string from which we want to cut subpart
- second is number which represents starting position in string
- third is number which represents ending position in string
- returns string value
.br
\fBhsh\fR - compute modulo hash value from string with specified base
- accepts two parameters
- first is string for which we are computing the hash value
- second is numeric value for base of modulo hash
- returns numeric value
.br
\fBmd5\fR - compute MD5 checksum for string
- accepts one string value
- returns string which represents MD5 checksum
.br
\fBlo\fR - convert all characters inside string to lower case
- accepts ane string value
- returns string value
.br
\fBup\fR - convert all characters inside string to upper case
- accepts one string value
- returns string value
.br
\fBue\fR - encode unsafe characters in string with same encoding
which is used for encoding unsafe characters inside URL (\fI%xx\fR)
As default are encoded all nonascii values when this function is
used.
- accepts two string values
- first is string which we want to encode
- second is string which contains unsafe characters
- return string value
.br
\fBdc\fR - delete unwanted characters from string
(have similar functionality as \fB-tr_del_chr\fR option)
- accepts two string values
- first is string from which we want delete
- second is string which contains characters we want to delete.
- returns string value
.br
\fBtc\fR - replace character with other character in string
(have similar functionality as \fB-tr_chr_chr\fR option)
- accepts three string values
- first is string inside which we want to replace characters
- second is set of characters which we want to replace
- third is set of characters with which we are replacing
- returns string value
.br
\fBts\fR - replace some string inside string with any other string
(have similar functionality as \fB-tr_str_str\fR option)
- accepts three string values
- first is string inside which we want to replace string
- second is the from string
- third is to string
- returns string value
.br
\fBspn\fR - calculate initial length of string which contains only specified set of characters.
(have same functionality as \fIstrspn()\fR libc function)
- accepts two string values
- first is input string
- second is set of acceptable characters
- returns numeric value
.br
\fBcspn\fR - calculate initial length of string which doesn't contain specified set of characters.
(have same functionality as \fIstrcspn()\fR libc function)
- accepts two string values
- first is input string
- second is set of unacceptable characters
- returns numeric value
.br
\fBsl\fR - calculate length of string
- accepts one string value
- returns numeric value
.br
\fBns\fR - convert number to string by format
- accepts two parameters
- first parameter is format string same as for \fIprintf()\fR function
- second is number which we want to convert
- returns string value
.br
\fBlc\fR - return position of last occurrence of specified character inside string
- accepts two string parameters
- first string which we are searching in
- second string contains character for which we are looking for
- returns numeric value
.br
\fB+\fR - add two numeric values
- accepts two numeric values
- returns numeric value
.br
\fB-\fR - subtract two numeric values
- accepts two numeric values
- returns numeric value
.br
\fB%\fR - modulo addition
- accepts two numeric values
- returns numeric value
.br
\fB*\fR - multiple two numeric values
- accepts two numeric values
- returns numeric value
.br
\fB/\fR - divide two numeric values
- accepts two numeric values
- returns numeric value
.br
\fBrmpar\fR - remove parameter from query string
- accepts two string
- first string is string which we are adjusting
- second parameter is name of parameter which should be removed
- returns adjusted string
.br
\fBgetval\fR - get query string parameter value
- accepts two string
- first string is query string from which to get the parameter
value (usually %s)
- second string is name of parameter for which we want to get
the value
- returns value of the parameter or empty string when the parameter
doesn't exists
.br
\fBsif\fR - logical decision
- accepts three parameters
- first is numeric and when is zero than result of this decision
is result of second parameter, else result is result of third
parameter
- second parameter is string
- third parameter is string
- returns string result of decision
.br
\fB!\fR - logical not
- accepts one numeric parameter
- returns negation of parameter
.br
\fB&\fR - logical and
- accept two numeric parameters
- returns logical and of parameters
.br
\fB|\fR - logical or
- accept two numeric parameters
- returns logical or of parameters
.br
\fBgetext\fR - get file extension
- accept one sting (filename or path)
- return string containing extension of parameter
.br
\fBseq\fR - compare two strings
- accepts two strings for comparison
- returns numeric value 0 - if different 1 - if equal
.br
.br
\fBjsf\fR - execute JavaScript function
- accepts one string parameter which holds name of
JavaScript function specified in script loaded with
\fI-js_script_file\fR option.
- returns string value equal to return value of
JavaScript function
- this function is available only when pavuk is compiled
with support for JavaScript bindings
.sp
For example, if you are mirroring very huge number of internet sites
into same local directory, too much entries in one directory, should
cause performance problems. You may use for example \fIhsh\fR or \fImd5\fR
functions to generate one additional level of hash directories based on hostname
whit one of following options :
.sp
\fI-fnrules F '*' '(sc (nc "%02d/" (hsh %h 100)) %o)'\fR
.br
\fI-fnrules F '*' '(sc (ss (md5 %h) 0 2) %o)'\fR
.br
.TP
.I -base_level $nr
Number of directory levels to omit in local tree.
.sp
For example when downloading
URL ftp://ftp.idata.sk/pub/unix/www/pavuk-0.7pl1.tgz you enter at command line
.B -base_level 4
in local tree will be created www/pavuk-0.7pl1.tgz not
ftp/ftp.idata.sk_21/pub/unix/www/pavuk-0.7pl1.tgz as normally.
.TP
.I -default_prefix $str
Default prefix of mirrored directory. This option is used only when you are
trying to synchronize content of remote directory which was downloaded using
\fB-base_level\fR option. Also you must use directory based synchronization
method, not URL based synchronization method. This is especially useful, when
used in conjunction with \fB-remove_old\fR option.
.TP
.I -remove_adv/-noremove_adv
This option is used for turn on/off of removing HTML tags which contains advertisement banners.
The banners are not removed from HTML file, but are commented out.
Such URLs also will not be downloaded.
This option have effect only when used with option \fB-adv_re\fR.
Default is turned off.
This option is available only when your system have support for one of supported regular expressions implementation.
.TP
.I -adv_re $RE
This option is used to specify regular expressions for matching URLs of advertisement banners.
For example :
-adv_re http://ad.doubleclick.net/.*
is used to match all files from server ad.doubleclick.net.
This option is available only when your system have any supported regular expressions implementation.
.TP
.I -unique_name/-nounique_name
Pavuk as default always attempts to assign to unique URL unique local filename.
If this behavior is not wanted, you can use option
.B -nounique_name
to disable this.
.SH Other Options
.sp
.TP
.I -sleep $nr
This option allows you to specify number of seconds during that the program will
be suspended between two transfers. Useful to deny server overload.
Default value for this option is 0.
.TP
.I -rsleep/-norsleep
When this option is active, pavuk randomizes the the sleep time between transfers in interval between zero and value specified with \fB-sleep\fR option. Default is this option inactive.
.TP
.I -ddays $nr
If document has modification time later as $nr days, then in sync mode pavuk attempts to
retrieve newer copy of document from remote server. Default value is 0.
.TP
.I -remove_old/-noremove_old
Remove improper documents (that, which doesn't exist on remote site).
This option have effect only when used in directory based \fBsync\fR mode.
When used with URL based sync mode, pavuk will not remove any old files
which were excluded from document tree and are not referenced in any HTML
document.
You must also use option \fB-subdir\fR, to let pavuk find files which belongs to current mirror.
As default pavuk won't remove any old files.
.TP
.I -browser $str
is used to set your browser command (in URL tree dialog you can
use right click to raise menu, from which you can start browser on actually
selected URL).
This option is available only when compiled with GTK GUI and with support for URL tree preview.
.TP
.I -debug/-nodebug
turns on displaying of debug messages. This option is available only when compiled with -DDEBUG.
If -debug option is used pavuk will output verbose information about documents,
whole protocol level information, locking informations and more
(depends on \fB-debug_level\fR setup). This options is used just like trigger
to enable output of debug messages selected by \fB-debug_level\fR option.
Default is debug mode turned off.
.TP
.I -debug_level $level
Set level of required debug informations. \fB$level\fR can be numeric value
which represent binary mask for requested debug levels, or comma separated list
of supported debug levels.
Currently pavuk supports following debug levels :
.br
\fBhtml\fR - for HTML parser debugging
.br
\fBprotos\fR - to see server side protocol messages
.br
\fBprotoc\fR - to see client side protocol messages
.br
\fBprocs\fR - to see some special procedure calls
.br
\fBlocks\fR - for debugging of documents locking
.br
\fBnet\fR - for debugging some low level network stuff
.br
\fBmisc\fR - for miscellaneous unsorted debug messages
.br
\fBuser\fR - for verbose user level messages
.br
\fBall\fR - request all currently supported debug levels
.br
\fBmtlock\fR - locking of resources in multithreading environment
.br
\fBmtthr\fR - launching/weaking/sleeping/stoping of threads in multithreaded environment
.br
\fBprotod\fR - for DEBUGGING of POST requests
.br
\fBlimits\fR - for debugging limiting options, you will see the reason why particular URLs are rejected by pavuk and which option caused this.
.br
\fBssl\fR - to enable verbose reporting about SSL related things.
.br
.TP
.I -remind_cmd $str
This option have effect only when running pavuk in
.B reminder
mode. To command specified with this option pavuk sends result of running reminder mode.
There are listed URLs which are changed and URLs which have any errors.
Default remind command is "mailx user@server -s \\"pavuk reminder result\\"" .
.TP
.I -nscache_dir $dir
Path to Netscape browser cache directory. If you specify this path, pavuk
attempts to find out if you have URL in this cache. If URL is there it will
be fetched else pavuk will download it from net. The cache directory index
file must be named \fIindex.db\fR and must be located in the cache directory.
To support this feature, pavuk have to be linked with BerkeleyDB 1.8x .
.TP
.I -mozcache_dir $dir
Path to Mozilla browser cache directory. Same functionality as with previous
option, just for different browser with different cache formats.
Pavuk supports both formats of Mozilla browser disk cache (old for versions
<0.9 and new used in 0.9=<).
The old format cache directory must contain cache directory index database
with name \fIcache.db\fR. Then new format cache directory must contain map
file \fI_CACHE_MAP_\fR, and three block files
\fI_CACHE_001_\fR, \fI_CACHE_002_\fR, \fI_CACHE_003_\fR.
To support old Mozilla cache format, pavuk have to be linked with
BerkeleyDB 1.8x. New Mozilla cache format doesn't require any external library.
.TP
.I -post_cmd $str
Post-processing command, which will be executed after successful download of document.
This command may somehow handle with document. During time of running this command, pavuk
leaves actual document locked, so there isn't chance that some other pavuk process
will modify document.
This postprocessing command will get three additional parameters from pavuk.
.in +3
.br
- local name of document
.br
-
.B 1/0
1 if document is HTML document, 0 if not
.br
- original URL of this document
.in -3
.TP
.I -hack_add_index/-nohack_add_index
This is bit hacky option. It forces pavuk to add to URL queue also directory
indexes of all queued documents. This allow pavuk to download more documents
from site, than it is able achieve in normal traversing of HTML documents.
Bit dirty but useful in some cases.
.TP
.I -js_script_file $file
Pavuk have optionally builtin JavaScript interpreter to allow high level
customization of some internal procedures. Currently you are allowed to
customize with your own JavaScript functions two things. You can use it
to set precise limiting options, or you can write own functions which can
be used inside rules of \fI-fnrules\fR option.
With this option you can load JavaScript script with functions into pavuks
internal JavaScript interpreter. To learn more about this capabilities read
separate document jsbind.txt which comes with pavuk sources in toplevel
directory.
This option is available only when you have compiled pavuk with support for
JavaScript bindings.
.SH EXIT STATUS
As of version 0.9pl29 pavuk have changed indication of status by exit codes.
In earlier versions exit status 0 was for no error and nonzero exit status
was something like count of failed documents.
In all version after 0.0pl29 there are defined following exit codes:
.br
.sp
0 - no error, everything is OK
1 - error in configuration of pavuk options or
error in config files
2 - some error occurred while downloading documents
.SH ENVIRONMENTAL VARIABLES
.sp
.TP
.I USER
variable is used to construct email address from user and hostname
.TP
.I LC_* or LANG
used to set internationalized environment
.TP
.I PAVUKRC_FILE
with this variable you can specify alternative location for your pavukrc
configuration file.
.SH REQUIRED EXTERNAL PROGRAMS
.sp
.TP
.IB at
is used for scheduling.
.TP
.IB gunzip
is used to decode gzip or compress encoded documents.
.SH Bugs
.sp
If you find any, please let me know.
.SH FILES
.sp
.TP
.IB @SYSCONFDIR@/pavukrc
.TP
.IB ~/.pavukrc
.TP
.IB ~/.pavuk_prefs
.br
.sp
These files are used as default configuration files.
You may specify there some constant values like your proxy server or
your preferred WWW browser. Configuration options reflect command line options.
Not all parameters are suitable for use in default configuration file.
You should select only some of them, which you really need.
.br
.sp
File
.B ~/.pavuk_prefs
is special file which contains automatically stored configuration.
This file is used only when running GUI interface of pavuk and option
.B -prefs
is active.
.br
.sp
First (if present) parsed file is
.B @SYSCONFDIR@/pavukrc
then
.B ~/.pavukrc
(if present), then
.B ~/.pavuk_prefs
(if present).
Last the command line is parsed. The precedence is as follows :
.br
.sp
- highest -
.br
Entered in user interface
.br
Entered in command line
.br
~/.pavuk_prefs
.br
~/.pavukrc
.br
@SYSCONFDIR@/pavukrc
.br
- lowest -
.sp
.br
Here is table of config file - command line options pairs.
.br
.sp
.RS
.nf
MaxLevel: ---> -lmax
MaxDocs: ---> -dmax
MaxSize: ---> -maxsize
MinSize: ---> -minsize
SleepBetween: ---> -sleep
MaxRetry: ---> -retry
MaxRegets: ---> -nregets
MaxRedirections: ---> -nredirs
CommTimeout: ---> -timeout
RegetRollbackAmount: ---> -rollback
DocExpiration: ---> -ddays
UseCache: ---> -nocache
UseRobots: ---> -noRobots
AllowFTP: ---> -noFTP
AllowHTTP: ---> -noHTTP
AllowSSL: ---> -noSSL
AllowGopher: ---> -noGopher
AllowCGI: ---> -noCGI
AllowGZEncoding: ---> -noEnc
AllowFTPRecursion: ---> -FTPdir
ForceReget: ---> -force_reget
Debug: ---> -debug
AllowedSites: ---> -asite
DisallowedSites: ---> -dsite
AllowedDomains: ---> -adomain
DisallowedDomains: ---> -ddomain
AllowedPrefixes: ---> -aprefix
DisallowedPrefixes: ---> -dprefix
AllowedSuffixes: ---> -asfx
DisallowedSuffixes: ---> -dsfx
AllowedMIMETypes: ---> -amimet
DisallowedMIMETypes: ---> -dmimet
PreferredLanguages: ---> -alang
PreferredCharset: ---> -acharset
WorkingDir: ---> -cdir
WorkingSubDir: ---> -subdir
HTTPAuthorizationScheme: ---> -auth_scheme
HTTPAuthorizationName: ---> -auth_name
HTTPAuthorizationPassword: ---> -auth_passwd
AuthReuseDigestNonce: ---> -auth_reuse_nonce
SSLCertPassword: ---> -ssl_cert_passwd
SSLCertFile: ---> -ssl_cert_file
SSLKeyFile: ---> -ssl_key_file
EmailAddress: ---> -from
MatchPattern: ---> -pattern
REMatchPattern: ---> -rpattern
SkipMatchPattern: ---> -skip_pattern
SkipREMatchPattern: ---> -skip_rpattern
URLMatchPattern: ---> -url_pattern
URLREMatchPattern: ---> -url_rpattern
SkipURLMatchPattern: ---> -skip_url_pattern
SkipURLREMatchPattern: ---> -skip_url_rpattern
DefaultMode: ---> -mode
FTPProxy: ---> -ftp_proxy
HTTPProxy: ---> -http_proxy
SSLProxy: ---> -ssl_proxy
GopherProxy: ---> -gopher_proxy
FTPViaHTTPProxy: ---> -ftp_httpgw
GopherViaHTTPProxy: ---> -gopher_httpgw
HTTPProxyUser: ---> -http_proxy_user
HTTPProxyPass: ---> -http_proxy_pass
HTTPProxyAuth: ---> -http_proxy_auth
AuthReuseProxyDigestNonce: ---> -auth_reuse_proxy_nonce
Browser: ---> -browser
ScenarioDir: ---> -scndir
ShowProgress: ---> -progress
XMaxLogSize: ---> -xmaxlog
LogFile: ---> -logfile
RemoveOldDocuments: ---> -remove_old
AuthFile: ---> -auth_file
BaseLevel: ---> -base_level
FTPDirtyProxy: ---> -ftp_dirtyproxy
ActiveFTPData: ---> -ftp_active/-ftp_passive
ActiveFTPPortRange: ---> -active_ftp_port_range
AlwaysMDTM: ---> -always_mdtm/-noalways_mdtm
RemoveBeforeStore: ---> -(no)remove_before_store
ShowDownloadTime: ---> -stime
NLSMessageCatalogDir: ---> -msgcat
Quiet: ---> -quiet/-verbose
NewerThan: ---> -newer_than
OlderThan: ---> -older_than
Reschedule: ---> -reschedule
DontLeaveSite: ---> -dont_leave_site/-leave_site
DontLeaveDir: ---> -dont_leave_dir/-leave_dir
PreserveTime: ---> -preserve_time/-nopreserve_time
LeaveLevel: ---> -leave_level
GUIFont: ---> -gui_font
UserCondition: ---> -user_condition
CookieFile: ---> -cookie_file
CookieSend: ---> -cookie_send/-nocookie_send
CookieRecv: ---> -cookie_recv/-nocookie_recv
CookieUpdate: ---> -cookie_update/-nocookie_update
CookiesMax: ---> -cookies_max
CookieCheckDomain: ---> -cookie_check/-nocookie_check
DisabledCookieDomains: ---> -disabled_cookie_domains
DisableHTMLTag: ---> -disable_html_tag
EnableHTMLTag: ---> -enable_html_tag
TrDeleteChar: ---> -tr_del_chr
TrStrToStr: ---> -tr_str_str
TrChrToChr: ---> -tr_chr_chr
IndexName: ---> -index_name
StoreName: ---> -store_name
PreservePermisions: ---> -preserve_perm/-nopreserve_perm
PreserveAbsoluteSymlinks: ---> -preserve_slinks/-nopreserve_slinks
FTPListCMD: ---> -FTPlist/-noFTPlist
MaxRate: ---> -maxrate
MinRate: ---> -minrate
ReadBufferSize: ---> -bufsize
BgMode: ---> -bg/-nobg
CheckSize: ---> -check_size/-nocheck_size
SLogFile: ---> -slogfile
Identity: ---> -identity
SendFromHeader: ---> -send_from/-nosend_from
RunX: ---> -runX
FnameRules: ---> -fnrules
StoreDocInfoFiles: ---> -store_info/-nostore_info
AllLinksToLocal: ---> -all_to_local/-noall_to_local
AllLinksToRemote: ---> -all_to_remote/-noall_to_remote
SelectedLinksToLocal: ---> -sel_to_local/-nosel_to_local
ReminderCMD: ---> -remind_cmd
AutoReferer: ---> -auto_referer/-noauto_referer
URLsFile: ---> -urls_file
UsePreferences: ---> -prefs/-noprefs
FTPhtml: ---> -FTPhtml/-noFTPhtml
StoreDirIndexFile: ---> -store_index/-nostore_index
Language: ---> -language
FileSizeQuota: ---> -file_quota
TransferQuota: ---> -trans_quota
FSQuota: ---> -fs_quota
EnableJS: ---> -enable_js/-disable_js
UrlSchedulingStrategy: ---> -url_strategy
NetscapeCacheDir: ---> -nscache_dir
RemoveAdvertisement: ---> -remove_adv/-noremove_adv
AdvBannerRE: ---> -adv_re
CheckIfRunnigAtBackground: ---> -check_bg/-nocheck_bg
SendIfRange: ---> -send_if_range/-nosend_if_range
SchedulingCommand: ---> -sched_cmd
UniqueLogName: ---> -unique_log/-nounique_log
PostCommand: ---> -post_cmd
SSLVersion: ---> -ssl_version
UniqueSSLID: ---> -unique_sslid/-nounique_sslid
AddHTTPHeader: ---> -httpad
StatisticsFile: ---> -statfile
WaitOnExit: ---> -ewait
AllowedIPAdrressPattern: ---> -aip_pattern
DisallowedIPAdrressPattern:---> -dip_pattern
SiteLevel: ---> -site_level
UseHTTP11: ---> -use_http11
MaxRunTime: ---> -max_time
LocalIP: ---> -local_ip
RequestInfo: ---> -request
HashSize: ---> -hash_size
NumberOfThreads: ---> -nthreads
ImmediateMessages: ---> -immesg/-noimmsg
HTMLFormData: ---> -formdata
DumpFD: ---> -dumpfd
DumpUrlFD: ---> -dump_urlfd
DeleteAfterTransfer: ---> -del_after/-nodel_after
UniqueDocName: ---> -unique_name/-nounique_name
LeaveSiteEnterDirectory: ---> -leave_site_enter_dir/-dont_leave_site_enter_dir
SinglePage: ---> -singlepage/-nosinglepage
NTLMAuthorizationDomain: ---> -auth_ntlm_domain
NTLMProxyAuthorizationDomain:
---> -auth_proxy_ntlm_domain
JavascriptPattern: ---> -js_pattern
FollowCommand: ---> -follow_cmd
RetrieveSymlinks: ---> -retrieve_symlink/-noretrieve_symlink
JSTransform: ---> -js_transform
JSTransform2: ---> -js_transform2
FTPProxyUser: ---> -ftp_proxy_user
FTPProxyPassword: ---> -ftp_proxy_pass
LimitInlineObjects: ---> -limit_inlines/-dont_limit_inlines
FTPListOptions: ---> -ftp_list_options
FixWuFTPDBrokenLISTcmd: ---> -fix_wuftpd_list/-nofix_wuftpd_list
PostUpdate: ---> -post_update/-nopost_update
SeparateInfoDir: ---> -info_dir
MozillaCacheDir: ---> -mozcache_dir
AllowedPorts: ---> -aport
DisallowedPorts: ---> -dport
HackAddIndex: ---> -hack_add_index/-nohack_add_index
JavaScriptFile: ---> -js_script_file
FtpLoginHandshake: ---> -ftp_login_handshake
NSSCertDir: ---> -nss_cert_dir
NSSAcceptUnknownCert: ---> -nss_accept_unknown_cert/-nonss_accept_unknown_cert
NSSDomesticPolicy: ---> -nss_domestic_policy/-nss_export_policy
DontTouchUrlREPattern: ---> -dont_touch_url_rpattern
DontTouchUrlPattern: ---> -dont_touch_url_pattern
DontTouchTagREPattern: ---> -dont_touch_tag_rpattern
HTMLTagPattern: ---> -tag_pattern
HTMLTagREPattern: ---> -tag_rpattern
URL: ---> one URL (more lines with URL:
... means more URL's)
.fi
.RE
line which begins with '#' means comment.
.br
TrStrToStr: and TrChrToChr: must contain two quoted strings.
All parameter names are case insensitive. If here is missing any option, try to
look inside
.B config.c
source file.
.br
.sp
See pavukrc.sample file for example
.TP
.IB .pavuk_authinfo
.sp
File should contain as many authentification records as you need.
Records are separated by any number of empty lines.
Parameter name is case insensitive.
.sp
Structure of record:
.sp
.RS
.nf
\fBProto:\fR ---> identification of protocol
(ftp/http/https/..)
- required field
\fBHost:\fR ---> host name
- required field
\fBUser:\fR ---> name of user
- optional
\fBPass:\fR ---> password for user
- optional
\fBBase:\fR ---> base prefix of document path
- optional
\fBRealm:\fR ---> realm for HTTP authorization
- optional
\fBNTLMDomain:\fR ---> NT/LM domain for NTLM authorization
- optional
\fBType:\fR ---> HTTP authentification scheme
- 1/user - user auth scheme
- 2/Basic - Basic auth scheme (default)
- 3/Digest - Digest auth scheme
- 4/NTLM - NTLM auth scheme
- optional
.fi
.RE
.TP
see pavuk_authinfo.sample file for example
.TP
.IB ~/.pavuk_keys
this is file where are stored information about configurable menu option shortcuts.
This is available only when compiled with Gtk+1.2 and higher.
.TP
.IB ~/.pavuk_remind_db
this file contains informations about URLs for running in
.B reminder
mode. Structure of this file is very easy. Each line contains information abou one URL.
first entry in line is last known modification time of URL (stored in time_t format - number of secons from 1.1.1970 GMT).
And second entry is URL.
.SH EXAMPLE COMMAND LINE
.sp
pavuk -mode mirror -nobg -store_info -info_dir
.br
/mirror/info -nthreads 1 -cdir /mirror/incoming -subdir
.br
/mirror/incoming -preserve_time -nopreserve_perm
.br
-nopreserve_slinks -noretrieve_symlink -force_reget
.br
-noRobots -trans_quota 16384 -maxsize 16777216
.br
-max_time 28 -nodel_after -remove_before_store -ftpdir
.br
-ftplist -ftp_list_options -a -dont_leave_site
.br
-dont_leave_dir -all_to_local -remove_old -nostore_index
.br
-active_ftp_port_range 57344:65535 -always_mdtm
.br
-ftp_passive -base_level 2 http:///doc/
.SH SEE ALSO
.sp
Look into
.B ChangeLog
file for more informations about new features in particular versions of pavuk.
.SH AUTHOR
.sp
Main development \fBOndrejicka Stefan\fR
.br
Look into
.B CREDITS
file of sources for additional information.
.SH AVAILABILITY
.sp
pavuk is available from
.B http://pavuk.sourceforge.net/
.PP