.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings.  \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote.  \*(C+ will
.\" give a nicer C++.  Capital omega is used to do unbreakable dashes and
.\" therefore won't be available.  \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
.    ds -- \(*W-
.    ds PI pi
.    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
.    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
.    ds L" ""
.    ds R" ""
.    ds C` ""
.    ds C' ""
'br\}
.el\{\
.    ds -- \|\(em\|
.    ds PI \(*p
.    ds L" ``
.    ds R" ''
.    ds C`
.    ds C'
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el       .ds Aq '
.\"
.\" If the F register is >0, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD.  Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.\"
.\" Avoid warning from groff about undefined register 'F'.
.de IX
..
.nr rF 0
.if \n(.g .if rF .nr rF 1
.if (\n(rF:(\n(.g==0)) \{\
.    if \nF \{\
.        de IX
.        tm Index:\\$1\t\\n%\t"\\$2"
..
.        if !\nF==2 \{\
.            nr % 0
.            nr F 2
.        \}
.    \}
.\}
.rr rF
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear.  Run.  Save yourself.  No user-serviceable parts.
.    \" fudge factors for nroff and troff
.if n \{\
.    ds #H 0
.    ds #V .8m
.    ds #F .3m
.    ds #[ \f1
.    ds #] \fP
.\}
.if t \{\
.    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
.    ds #V .6m
.    ds #F 0
.    ds #[ \&
.    ds #] \&
.\}
.    \" simple accents for nroff and troff
.if n \{\
.    ds ' \&
.    ds ` \&
.    ds ^ \&
.    ds , \&
.    ds ~ ~
.    ds /
.\}
.if t \{\
.    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
.    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
.    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
.    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
.    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
.    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
.    \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
.    \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
.    \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
.    ds : e
.    ds 8 ss
.    ds o a
.    ds d- d\h'-1'\(ga
.    ds D- D\h'-1'\(hy
.    ds th \o'bp'
.    ds Th \o'LP'
.    ds ae ae
.    ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "Catmandu::Importer 3pm"
.TH Catmandu::Importer 3pm "2019-01-29" "perl v5.28.1" "User Contributed Perl Documentation"
.\" For nroff, turn off justification.  Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
Catmandu::Importer \- Namespace for packages that can import
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
.Vb 1
\&    # From the command line
\&
\&    # JSON is an importer and YAML an exporter
\&    $ catmandu convert JSON to YAML < data.json
\&
\&    # OAI is an importer and JSON an exporter
\&    $ catmandu convert OAI \-\-url http://biblio.ugent.be/oai to JSON 
\&
\&    # Fetch remote content
\&    $ catmandu convert JSON \-\-file http://example.com/data.json to YAML
\&    
\&    # From Perl
\&    
\&    use Catmandu;
\&    use Data::Dumper;
\&
\&    my $importer = Catmandu\->importer(\*(AqJSON\*(Aq, file => \*(Aqdata.json\*(Aq);
\&
\&    $importer\->each(sub {
\&        my $item = shift;
\&        print Dumper($item);
\&    });
\&
\&    my $num = $importer\->count;
\&
\&    my $first_item = $importer\->first;
\&
\&    # Convert OAI to JSON in Perl
\&    my $importer = Catmandu\->importer(\*(AqOAI\*(Aq, url => \*(Aqhttp://biblio.ugent.be/oai\*(Aq);
\&    my $exporter = Catmandu\->exporter(\*(AqJSON\*(Aq);
\&
\&    $exporter\->add_many($importer);
.Ve
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
A Catmandu::Importer is a Perl package that can generate structured data from
sources such as \s-1JSON, YAML, XML, RDF\s0 or network protocols such as Atom, OAI-PMH,
\&\s-1SRU\s0 and even \s-1DBI\s0 databases. Given an Catmandu::Importer a programmer can read
data from using one of the many Catmandu::Iterable methods:
.PP
.Vb 6
\&    $importer\->to_array;
\&    $importer\->count;
\&    $importer\->each(\e&callback);
\&    $importer\->first;
\&    $importer\->rest;
\&    ...etc...
.Ve
.PP
Every Catmandu::Importer is also Catmandu::Fixable and thus inherits a 'fix'
parameter that can be set in the constructor. When given a 'fix' parameter, then each 
item returned by the generator will be automatically Fixed using one or 
more Catmandu::Fixes.
E.g.
.PP
.Vb 3
\&    my $importer = Catmandu\->importer(\*(AqJSON\*(Aq,fix => [\*(Aqupcase(title)\*(Aq]);
\&    $importer\->each( sub {
\&        my $item = shift ; # Every $item\->{title} is now upcased... 
\&
\&    });
\&
\&    # or via a Fix file
\&    my $importer = Catmandu\->importer(\*(AqJSON\*(Aq,fix => [\*(Aq/my/fixes.txt\*(Aq]);
\&    $importer\->each( sub {
\&        my $item = shift ; # Every $item\->{title} is now upcased... 
\&
\&    });
.Ve
.SH "CONFIGURATION"
.IX Header "CONFIGURATION"
.IP "file" 4
.IX Item "file"
Read input from a local file given by its path. If the path looks like a
url, the content will be fetched first and then passed to the importer.
Alternatively a scalar reference can be passed to read from a string.
.IP "fh" 4
.IX Item "fh"
Read input from an IO::Handle. If not specified, Catmandu::Util::io is used to
create the input stream from the \f(CW\*(C`file\*(C'\fR argument or by using \s-1STDIN.\s0
.IP "encoding" 4
.IX Item "encoding"
Binmode of the input stream \f(CW\*(C`fh\*(C'\fR. Set to \f(CW\*(C`:utf8\*(C'\fR by default.
.IP "fix" 4
.IX Item "fix"
An \s-1ARRAY\s0 of one or more Fix-es or Fix scripts to be applied to imported items.
.IP "data_path" 4
.IX Item "data_path"
The data at \f(CW\*(C`data_path\*(C'\fR is imported instead of the original data.
.Sp
.Vb 8
\&   # given this imported item:
\&   {abc => [{a=>1},{b=>2},{c=>3}]}
\&   # with data_path \*(Aqabc\*(Aq, this item gets imported instead:
\&   [{a=>1},{b=>2},{c=>3}]
\&   # with data_path \*(Aqabc.*\*(Aq, 3 items get imported:
\&   {a=>1}
\&   {b=>2}
\&   {c=>3}
.Ve
.IP "variables" 4
.IX Item "variables"
Variables given here will interpolate the \f(CW\*(C`file\*(C'\fR and \f(CW\*(C`http_body\*(C'\fR options. The
syntax is the same as URI::Template.
.Sp
.Vb 5
\&    # named arguments
\&    my $importer = Catmandu\->importer(\*(AqJSON\*(Aq,
\&        file => \*(Aqhttp://{server}/{path}\*(Aq,
\&        variables => {server => \*(Aqbiblio.ugent.be\*(Aq, path => \*(Aqfile.json\*(Aq},
\&    );
\&
\&    # positional arguments
\&    my $importer = Catmandu\->importer(\*(AqJSON\*(Aq,
\&        file => \*(Aqhttp://{server}/{path}\*(Aq,
\&        variables => \*(Aqbiblio.ugent.be,file.json\*(Aq,
\&    );
\&
\&    # or
\&    my $importer = Catmandu\->importer(\*(AqJSON\*(Aq,
\&        url => \*(Aqhttp://{server}/{path}\*(Aq,
\&        variables => [\*(Aqbiblio.ugent.be\*(Aq,\*(Aqfile.json\*(Aq],
\&    );
\&
\&    # or via the command line
\&    $ catmandu convert JSON \-\-file \*(Aqhttp://{server}/{path}\*(Aq \-\-variables \*(Aqbiblio.ugent.be,file.json\*(Aq
.Ve
.SH "HTTP CONFIGURATION"
.IX Header "HTTP CONFIGURATION"
These options are only relevant if \f(CW\*(C`file\*(C'\fR is a url. See LWP::UserAgent for details about these options.
.IP "http_body" 4
.IX Item "http_body"
Set the \s-1GET/POST\s0 message body.
.IP "http_method" 4
.IX Item "http_method"
Set the type of \s-1HTTP\s0 request '\s-1GET\s0', '\s-1POST\s0' , ...
.IP "http_headers" 4
.IX Item "http_headers"
A reference to a HTTP::Headers objects.
.SS "Set an own \s-1HTTP\s0 client"
.IX Subsection "Set an own HTTP client"
.IP "user_agent(LWP::UserAgent\->new(...))" 4
.IX Item "user_agent(LWP::UserAgent->new(...))"
Set an own \s-1HTTP\s0 client
.SS "Alternative set the parameters of the default client"
.IX Subsection "Alternative set the parameters of the default client"
.IP "http_agent" 4
.IX Item "http_agent"
A string containing the name of the \s-1HTTP\s0 client.
.IP "http_max_redirect" 4
.IX Item "http_max_redirect"
Maximum number of \s-1HTTP\s0 redirects allowed.
.IP "http_timeout" 4
.IX Item "http_timeout"
Maximum execution time.
.IP "http_verify_hostname" 4
.IX Item "http_verify_hostname"
Verify the \s-1SSL\s0 certificate.
.IP "http_retry" 4
.IX Item "http_retry"
Maximum times to retry the \s-1HTTP\s0 request if it temporarily fails. Default is not
to retry.  See LWP::UserAgent::Determined for the \s-1HTTP\s0 status codes
that initiate a retry.
.IP "http_timing" 4
.IX Item "http_timing"
Maximum times and timeouts to retry the \s-1HTTP\s0 request if it temporarily fails. Default is not
to retry.  See LWP::UserAgent::Determined for the \s-1HTTP\s0 status codes
that initiate a retry and the format of the timing value.
.SH "METHODS"
.IX Header "METHODS"
.SS "first, each, rest , ..."
.IX Subsection "first, each, rest , ..."
See Catmandu::Iterable for all inherited methods.
.SH "CODING"
.IX Header "CODING"
Create your own importer by creating a Perl package in the Catmandu::Importer namespace that
implements \f(CW\*(C`Catmandu::Importer\*(C'\fR. Basically, you need to create a method 'generate' which 
returns a callback that creates one Perl hash for each call:
.PP
.Vb 1
\&    my $importer = Catmandu::Importer::Hello\->new;
\&
\&    $importer\->generate(); # record
\&    $importer\->generate(); # next record
\&    $importer\->generate(); # undef = end of stream
.Ve
.PP
Here is an example of a simple \f(CW\*(C`Hello\*(C'\fR importer:
.PP
.Vb 1
\&    package Catmandu::Importer::Hello;
\&
\&    use Catmandu::Sane;
\&    use Moo;
\&
\&    with \*(AqCatmandu::Importer\*(Aq;
\&
\&    sub generator {
\&        my ($self) = @_;
\&        state $fh = $self\->fh;
\&        my $n = 0;
\&        return sub {
\&            $self\->log\->debug("generating record " . ++$n);
\&            my $name = $self\->fh\->readline;
\&            return defined $name ? { "hello" => $name } : undef;
\&        };
\&    }
\&
\&    1;
.Ve
.PP
This importer can be called via the command line as:
.PP
.Vb 3
\&    $ catmandu convert Hello to JSON < /tmp/names.txt
\&    $ catmandu convert Hello to YAML < /tmp/names.txt
\&    $ catmandu import Hello to MongoDB \-\-database_name test < /tmp/names.txt
.Ve
.PP
Or, via Perl
.PP
.Vb 1
\&    use Catmandu;
\&
\&    my $importer = Catmandu\->importer(\*(AqHello\*(Aq, file => \*(Aq/tmp/names.txt\*(Aq);
\&    $importer\->each(sub {
\&        my $items = shift;
\&    });
.Ve
.SH "SEE ALSO"
.IX Header "SEE ALSO"
Catmandu::Iterable , Catmandu::Fix ,
Catmandu::Importer::CSV, Catmandu::Importer::JSON , Catmandu::Importer::YAML