.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.42) .\" .\" Standard preamble: .\" ======================================================================== .de Sp \" Vertical space (when we can't use .PP) .if t .sp .5v .if n .sp .. .de Vb \" Begin verbatim text .ft CW .nf .ne \\$1 .. .de Ve \" End verbatim text .ft R .fi .. .\" Set up some character translations and predefined strings. \*(-- will .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left .\" double quote, and \*(R" will give a right double quote. \*(C+ will .\" give a nicer C++. Capital omega is used to do unbreakable dashes and .\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, .\" nothing in troff, for use with C<>. .tr \(*W- .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' .ie n \{\ . ds -- \(*W- . ds PI pi . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch . ds L" "" . ds R" "" . ds C` "" . ds C' "" 'br\} .el\{\ . ds -- \|\(em\| . ds PI \(*p . ds L" `` . ds R" '' . ds C` . ds C' 'br\} .\" .\" Escape single quotes in literal strings from groff's Unicode transform. .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" .\" If the F register is >0, we'll generate index entries on stderr for .\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index .\" entries marked with X<> in POD. Of course, you'll have to process the .\" output yourself in some meaningful fashion. .\" .\" Avoid warning from groff about undefined register 'F'. .de IX .. .nr rF 0 .if \n(.g .if rF .nr rF 1 .if (\n(rF:(\n(.g==0)) \{\ . if \nF \{\ . de IX . tm Index:\\$1\t\\n%\t"\\$2" .. . if !\nF==2 \{\ . nr % 0 . nr F 2 . \} . \} .\} .rr rF .\" .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). .\" Fear. Run. Save yourself. No user-serviceable parts. . \" fudge factors for nroff and troff .if n \{\ . ds #H 0 . ds #V .8m . ds #F .3m . ds #[ \f1 . ds #] \fP .\} .if t \{\ . ds #H ((1u-(\\\\n(.fu%2u))*.13m) . ds #V .6m . ds #F 0 . ds #[ \& . ds #] \& .\} . \" simple accents for nroff and troff .if n \{\ . ds ' \& . ds ` \& . ds ^ \& . ds , \& . ds ~ ~ . ds / .\} .if t \{\ . ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" . ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' . ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' . ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' . ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' . ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' .\} . \" troff and (daisy-wheel) nroff accents .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' .ds 8 \h'\*(#H'\(*b\h'-\*(#H' .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] .ds ae a\h'-(\w'a'u*4/10)'e .ds Ae A\h'-(\w'A'u*4/10)'E . \" corrections for vroff .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' . \" for low resolution devices (crt and lpr) .if \n(.H>23 .if \n(.V>19 \ \{\ . ds : e . ds 8 ss . ds o a . ds d- d\h'-1'\(ga . ds D- D\h'-1'\(hy . ds th \o'bp' . ds Th \o'LP' . ds ae ae . ds Ae AE .\} .rm #[ #] #H #V #F C .\" ======================================================================== .\" .IX Title "Catmandu::Store::ElasticSearch 3pm" .TH Catmandu::Store::ElasticSearch 3pm "2022-10-14" "perl v5.34.0" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l .nh .SH "NAME" Catmandu::Store::ElasticSearch \- A searchable store backed by Elasticsearch .SH "SYNOPSIS" .IX Header "SYNOPSIS" .Vb 1 \& # From the command line \& \& # Import data into ElasticSearch \& $ catmandu import JSON to ElasticSearch \-\-bag catmandu < data.json \& \& # Export data from ElasticSearch \& $ catmandu export ElasticSearch \-\-bag catmandu to JSON > data.json \& \& # Export only one record \& $ catmandu export ElasticSearch \-\-bag catmandu \-\-id 1234 \& \& # Export using an ElasticSearch query \& $ catmandu export ElasticSearch \-\-bag catmandu \-\-query "name:Recruitment OR name:college" \& \& # Export using a CQL query (needs a CQL mapping) \& $ catmandu export ElasticSearch \-\-bag catmandu \-\-cql\-query "name any college" \& \& # You need to specify the client version if your Elasticsearch server version is \& # not the same as your default Search::Elasticsearch client version \& $ catmandu import JSON to ElasticSearch \-\-bag catmandu \-\-client \*(Aq5_0::Direct\*(Aq < data.json \& \& # From Perl \& \& use Catmandu; \& \& my $store = Catmandu\->store(\*(AqElasticSearch\*(Aq); \& # options will be passed to the underlying Search::Elasticsearch client \& my $store = Catmandu\->store(\*(AqElasticSearch\*(Aq, nodes => [\*(Aqserver.example.com:9200\*(Aq]); \& \& my $obj1 = $store\->bag(\*(Aqcatmandu\*(Aq)\->add({ name => \*(AqPatrick\*(Aq }); \& \& printf "obj1 stored as %s\en" , $obj1\->{_id}; \& \& # Force an id in the store \& my $obj2 = $store\->bag(\*(Aqcatmandu\*(Aq)\->add({ _id => \*(Aqtest123\*(Aq , name => \*(AqNicolas\*(Aq }); \& \& # Commit all changes \& $store\->bag(\*(Aqcatmandu\*(Aq)\->commit; \& \& $store\->bag(\*(Aqcatmandu\*(Aq)\->delete(\*(Aqtest123\*(Aq); \& \& $store\->bag(\*(Aqcatmandu\*(Aq)\->delete_all; \& \& # All bags are iterators \& $store\->bag\->each(sub { ... }); \& $store\->bag\->take(10)\->each(sub { ... }); \& \& # Query the store using a simple ElasticSearch query \& my $hits = $store\->bag\->search(query => \*(Aq(content:this OR name:this) AND (content:that OR name:that)\*(Aq); \& \& # Native queries are also supported by providing a hash of terms \& # See the ElasticSearch manual for more examples \& my $hits = $store\->bag\->search( \& query => { \& # All name.exact fields that start with \*(Aqtest\*(Aq \& prefix => { \& \*(Aqname.exact\*(Aq => \*(Aqtest\*(Aq \& } \& } , \& limit => 1000); \& \& # Catmandu::Store::ElasticSearch supports CQL... \& my $hits = $store\->bag\->search(cql_query => \*(Aqname any "Patrick"\*(Aq); .Ve .SH "METHODS" .IX Header "METHODS" .SS "new(%params)" .IX Subsection "new(%params)" .SS "new(%params, bags => { mybag => { index => 'myindex', mapping => \e%map cql_mapping => \e%map } })" .IX Subsection "new(%params, bags => { mybag => { index => 'myindex', mapping => %map cql_mapping => %map } })" Create a new Catmandu::Store::ElasticSearch store. ElasticSearch connection parameters will be passed on to the underlying client. .PP Optionally provide for each bag a \f(CW\*(C`index\*(C'\fR to indicate which index to use. This defaults to the bag's name. .PP Optionally provide for each bag a \f(CW\*(C`type\*(C'\fR to indicate the name of the mapping. This defaults to the bag's name. .PP Optionally provide for each bag a \f(CW\*(C`mapping\*(C'\fR which contains a ElasticSearch schema for each field in the index (See below). .PP Optionally provide for each bag a \f(CW\*(C`cql_mapping\*(C'\fR to map fields to \s-1CQL\s0 indexes. .PP Optionally provide for each bag an \f(CW\*(C`on_error\*(C'\fR error handler (See below). .SH "INHERITED METHODS" .IX Header "INHERITED METHODS" This Catmandu::Store implements: .IP "Catmandu::Store" 3 .IX Item "Catmandu::Store" .PP Each Catmandu::Bag in this Catmandu::Store implements: .IP "Catmandu::Bag" 3 .IX Item "Catmandu::Bag" .PD 0 .IP "Catmandu::Droppable" 3 .IX Item "Catmandu::Droppable" .IP "Catmandu::Searchable" 3 .IX Item "Catmandu::Searchable" .IP "Catmandu::CQLSearchable" 3 .IX Item "Catmandu::CQLSearchable" .PD .SH "INDEX MAPPING" .IX Header "INDEX MAPPING" The mapping contains a Elasticsearch schema mappings for each bag defined in the index. E.g. .PP .Vb 7 \& { \& properties => { \& title => { \& type => \*(Aqtext\*(Aq \& } \& } \& } .Ve .PP See for more information on mappings. .PP These mappings can be passed inside a Perl program, or be written into a Catmandu 'catmandu.yml' configuration file. E.g. .PP .Vb 11 \& # catmandu.yml \& store: \& search: \& package: ElasticSearch \& options: \& bags: \& mybag: \& mapping: \& properties: \& title: \& type: text .Ve .PP Via the command line these configuration parameters can be read in by using the name of the store, \f(CW\*(C`search\*(C'\fR in this case: .PP .Vb 2 \& $ catmandu import JSON to search \-\-bag mybag < data.json \& $ catmandu export search \-\-bag mybag to JSON > data.json .Ve .SH "CQL MAPPING" .IX Header "CQL MAPPING" Catmandu::Store::ElasticSearch supports \s-1CQL\s0 searches when a cql_mapping is provided for each bag. This hash contains a translation of \s-1CQL\s0 fields into Elasticsearch searchable fields. .PP .Vb 10 \& # Example mapping \& { \& indexes => { \& title => { \& op => { \& \*(Aqany\*(Aq => 1 , \& \*(Aqall\*(Aq => 1 , \& \*(Aq=\*(Aq => 1 , \& \*(Aq<>\*(Aq => 1 , \& \*(Aqexact\*(Aq => {field => [qw(mytitle.exact myalttitle.exact)]} \& } , \& field => \*(Aqmytitle\*(Aq, \& sort => 1, \& cb => [\*(AqBiblio::Search\*(Aq, \*(Aqnormalize_title\*(Aq] \& } \& } \& } .Ve .PP The \s-1CQL\s0 mapping above will support for the 'title' field the \s-1CQL\s0 operators: any, all, =, <> and exact. .PP The 'title' field will be mapping into the Elasticsearch field 'mytitle', except for the 'exact' operator. In case of 'exact' we will search both the \&'mytitle.exact' and 'myalttitle.exact' fields. .PP The \s-1CQL\s0 mapping allows for sorting on the 'title' field. If, for instance, we would like to use a special ElasticSearch field for sorting we could have written \*(L"sort => { field => 'mytitle.sort' }\*(R". .PP The callback field \f(CW\*(C`cb\*(C'\fR contains a reference to subroutines to rewrite or augment a search query. In this case, the Biblio::Search package contains a normalize_title subroutine which returns a string or an \s-1ARRAY\s0 of strings with augmented title(s). E.g. .PP .Vb 1 \& package Biblio::Search; \& \& sub normalize_title { \& my ($self,$title) = @_; \& my $new_title =~ s{[^A\-Z0\-9]+}{}g; \& $new_title; \& } \& \& 1; .Ve .PP Also this configuration can be added to a catmandu.yml configuration file like: .PP .Vb 10 \& # catmandu.yml \& store: \& search: \& package: ElasticSearch \& options: \& client: 6_0::Direct \& bags: \& book: \& mapping: \& properties: \& title: \& type: text \& cql_mapping: \& indexes: \& title: \& op: \& \*(Aqany\*(Aq: true \& \*(Aqall\*(Aq: true \& \*(Aq=\*(Aq: true \& \*(Aq<>\*(Aq: true \& \*(Aqexact\*(Aq: \& field: [ \*(Aqmytitle.exact\*(Aq , \*(Aqmyalttitle.exact\*(Aq ] \& field: mytitle \& sort: true \& cb: [ \*(AqBiblio::Search\*(Aq , \*(Aqnormalize_title\*(Aq ] .Ve .PP Via the command line these configuration parameters can be read in by using the name of the store, \f(CW\*(C`search\*(C'\fR in this case: .PP .Vb 1 \& $ catmandu export search \-\-bag book \-q \*(Aqtitle any blablabla\*(Aq to JSON > data.json .Ve .SH "COMPATIBILITY" .IX Header "COMPATIBILITY" The appropriate client should be installed: .PP .Vb 4 \& # Elasticsearch 6.x \& cpanm Search::Elasticsearch::Client::6_0::Direct \& # Elasticsearch 1.x \& cpanm Search::Elasticsearch::Client::1_0::Direct .Ve .PP And specified in the options: .PP .Vb 1 \& Catmandu::Store::ElasticSearch\->new(client => \*(Aq1_0::Direct\*(Aq) .Ve .PP If you want to use the \f(CW\*(C`delete_by_query\*(C'\fR method with Elasticsearch 2.0 you have to install the delete by query plugin . .SH "ERROR HANDLING" .IX Header "ERROR HANDLING" Error handling can be activated by specifying an error handling callback for index when creating a store. E.g. to create an error handler for the bag 'data' index use: .PP .Vb 4 \& my $error_handler = sub { \& my ($action, $response, $i) = @_; \& do_something_with_error($response); \& }; \& \& my $store = Catmandu::Store::ElasticSearch\->new( \& bags => { data => { on_error => $error_handler } } \& }); .Ve .PP Instead of a callback, the following shortcuts are also accepted for on_error: .PP log: log the response .PP throw: throw the response as an error .PP ignore: do nothing .PP .Vb 3 \& my $store = Catmandu::Store::ElasticSearch\->new( \& bags => { data => { on_error => \*(Aqlog\*(Aq } } \& }); .Ve .SH "UPGRADING FROM A PRE 1.0 VERSION" .IX Header "UPGRADING FROM A PRE 1.0 VERSION" Versions of this store < 1.0 used Elasticsearch types to map bags to a single index. Support for multiple types in one index has since been removed from Elasticsearch and since 1.0 each bag is mapped to an index. .PP You need to export you data before upgrading, update the configuration and then import you data again. .SH "SEE ALSO" .IX Header "SEE ALSO" Catmandu::Store .SH "AUTHOR" .IX Header "AUTHOR" .ie n .IP "Nicolas Steenlant, """"" 4 .el .IP "Nicolas Steenlant, \f(CW\fR" 4 .IX Item "Nicolas Steenlant, " .SH "CONTRIBUTORS" .IX Header "CONTRIBUTORS" .PD 0 .ie n .IP "Dave Sherohman, ""dave.sherohman at ub.lu.se""" 4 .el .IP "Dave Sherohman, \f(CWdave.sherohman at ub.lu.se\fR" 4 .IX Item "Dave Sherohman, dave.sherohman at ub.lu.se" .ie n .IP "Robin Sheat, ""robin at kallisti.net.nz""" 4 .el .IP "Robin Sheat, \f(CWrobin at kallisti.net.nz\fR" 4 .IX Item "Robin Sheat, robin at kallisti.net.nz" .ie n .IP "Patrick Hochstenbach, ""patrick.hochstenbach at ugent.be""" 4 .el .IP "Patrick Hochstenbach, \f(CWpatrick.hochstenbach at ugent.be\fR" 4 .IX Item "Patrick Hochstenbach, patrick.hochstenbach at ugent.be" .PD .SH "LICENSE AND COPYRIGHT" .IX Header "LICENSE AND COPYRIGHT" This program is free software; you can redistribute it and/or modify it under the terms of either: the \s-1GNU\s0 General Public License as published by the Free Software Foundation; or the Artistic License. .PP See http://dev.perl.org/licenses/ for more information.