.\" Automatically generated by Pod::Man 2.28 (Pod::Simple 3.28) .\" .\" Standard preamble: .\" ======================================================================== .de Sp \" Vertical space (when we can't use .PP) .if t .sp .5v .if n .sp .. .de Vb \" Begin verbatim text .ft CW .nf .ne \\$1 .. .de Ve \" End verbatim text .ft R .fi .. .\" Set up some character translations and predefined strings. \*(-- will .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left .\" double quote, and \*(R" will give a right double quote. \*(C+ will .\" give a nicer C++. Capital omega is used to do unbreakable dashes and .\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, .\" nothing in troff, for use with C<>. .tr \(*W- .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' .ie n \{\ . ds -- \(*W- . ds PI pi . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch . ds L" "" . ds R" "" . ds C` "" . ds C' "" 'br\} .el\{\ . ds -- \|\(em\| . ds PI \(*p . ds L" `` . ds R" '' . ds C` . ds C' 'br\} .\" .\" Escape single quotes in literal strings from groff's Unicode transform. .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" .\" If the F register is turned on, we'll generate index entries on stderr for .\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index .\" entries marked with X<> in POD. Of course, you'll have to process the .\" output yourself in some meaningful fashion. .\" .\" Avoid warning from groff about undefined register 'F'. .de IX .. .nr rF 0 .if \n(.g .if rF .nr rF 1 .if (\n(rF:(\n(.g==0)) \{ . if \nF \{ . de IX . tm Index:\\$1\t\\n%\t"\\$2" .. . if !\nF==2 \{ . nr % 0 . nr F 2 . \} . \} .\} .rr rF .\" .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). .\" Fear. Run. Save yourself. No user-serviceable parts. . \" fudge factors for nroff and troff .if n \{\ . ds #H 0 . ds #V .8m . ds #F .3m . ds #[ \f1 . ds #] \fP .\} .if t \{\ . ds #H ((1u-(\\\\n(.fu%2u))*.13m) . ds #V .6m . ds #F 0 . ds #[ \& . ds #] \& .\} . \" simple accents for nroff and troff .if n \{\ . ds ' \& . ds ` \& . ds ^ \& . ds , \& . ds ~ ~ . ds / .\} .if t \{\ . ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" . ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' . ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' . ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' . ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' . ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' .\} . \" troff and (daisy-wheel) nroff accents .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' .ds 8 \h'\*(#H'\(*b\h'-\*(#H' .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] .ds ae a\h'-(\w'a'u*4/10)'e .ds Ae A\h'-(\w'A'u*4/10)'E . \" corrections for vroff .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' . \" for low resolution devices (crt and lpr) .if \n(.H>23 .if \n(.V>19 \ \{\ . ds : e . ds 8 ss . ds o a . ds d- d\h'-1'\(ga . ds D- D\h'-1'\(hy . ds th \o'bp' . ds Th \o'LP' . ds ae ae . ds Ae AE .\} .rm #[ #] #H #V #F C .\" ======================================================================== .\" .IX Title "MCE::Map 3pm" .TH MCE::Map 3pm "2014-10-24" "perl v5.20.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l .nh .SH "NAME" MCE::Map \- Parallel map model similar to the native map function .SH "VERSION" .IX Header "VERSION" This document describes MCE::Map version 1.517 .SH "SYNOPSIS" .IX Header "SYNOPSIS" .Vb 2 \& ## Exports mce_map, mce_map_f, and mce_map_s \& use MCE::Map; \& \& ## Array or array_ref \& my @a = mce_map { $_ * $_ } 1..10000; \& my @b = mce_map { $_ * $_ } [ 1..10000 ]; \& \& ## File_path, glob_ref, or scalar_ref \& my @c = mce_map_f { chomp; $_ } "/path/to/file"; \& my @d = mce_map_f { chomp; $_ } $file_handle; \& my @e = mce_map_f { chomp; $_ } \e$scalar; \& \& ## Sequence of numbers (begin, end [, step, format]) \& my @f = mce_map_s { $_ * $_ } 1, 10000, 5; \& my @g = mce_map_s { $_ * $_ } [ 1, 10000, 5 ]; \& \& my @h = mce_map_s { $_ * $_ } { \& begin => 1, end => 10000, step => 5, format => undef \& }; .Ve .SH "DESCRIPTION" .IX Header "DESCRIPTION" This module provides a parallel map implementation via Many-core Engine. \s-1MCE\s0 incurs a small overhead due to passing of data. Therefore, a fast code block will likely run faster using the native map function in Perl. The overhead quickly diminishes as the complexity of the code block increases. .PP .Vb 2 \& my @m1 = map { $_ * $_ } 1..1000000; ## 0.251 secs \& my @m2 = mce_map { $_ * $_ } 1..1000000; ## 0.525 secs .Ve .PP Chunking, enabled by default, greatly reduces the overhead behind the scene. The time for mce_map below also includes the time for data exchanges between the manager and worker processes. More parallelization will be seen when the code block requires additional \s-1CPU\s0 time code-wise. .PP .Vb 3 \& sub calc { \& sqrt $_ * sqrt $_ / 1.3 * 1.5 / 3.2 * 1.07 \& } \& \& my @m1 = map { calc } 1..1000000; ## 0.756 secs \& my @m2 = mce_map { calc } 1..1000000; ## 0.623 secs .Ve .PP The mce_map_s function will provide better times, useful when the input data is simply a range of numbers. Workers generate sequences mathematically among themselves without any interaction from the manager process. Two arguments are required for mce_map_s (begin, end). Step defaults to 1 if begin is smaller than end, otherwise \-1. .PP .Vb 1 \& my @m3 = mce_map_s { calc } 1, 1000000; ## 0.517 secs .Ve .PP Although this document is about MCE::Map, the MCE::Stream module can write results immediately without waiting for all chunks to complete. This is made possible by passing the reference of the array (in this case \f(CW@m4\fR and \f(CW@m5\fR). .PP .Vb 1 \& use MCE::Stream; \& \& sub calc { \& sqrt $_ * sqrt $_ / 1.3 * 1.5 / 3.2 * 1.07 \& } \& \& my @m4; mce_stream \e@m4, sub { calc }, 1..1000000; \& \& ## Completes in 0.436 secs. That is amazing considering the \& ## overhead for passing data between the manager and worker. \& \& my @m5; mce_stream_s \e@m5, sub { calc }, 1, 1000000; \& \& ## Completed in 0.301 secs. Like with mce_map_s, specifying a \& ## sequence specification turns out to be faster due to lesser \& ## overhead for the manager process. .Ve .SH "OVERRIDING DEFAULTS" .IX Header "OVERRIDING DEFAULTS" The following list 5 options which may be overridden when loading the module. .PP .Vb 1 \& use Sereal qw(encode_sereal decode_sereal); \& \& use MCE::Map \& max_workers => 4, ## Default \*(Aqauto\*(Aq \& chunk_size => 100, ## Default \*(Aqauto\*(Aq \& tmp_dir => "/path/to/app/tmp", ## $MCE::Signal::tmp_dir \& freeze => \e&encode_sereal, ## \e&Storable::freeze \& thaw => \e&decode_sereal ## \e&Storable::thaw \& ; .Ve .PP There is a simpler way to enable Sereal with \s-1MCE 1.5.\s0 The following will attempt to use Sereal if available, otherwise will default back to using Storable for serialization. .PP .Vb 1 \& use MCE::Map Sereal => 1; \& \& ## Serialization is through Sereal if available. \& my @m2 = mce_map { $_ * $_ } 1..10000; .Ve .SH "CUSTOMIZING MCE" .IX Header "CUSTOMIZING MCE" .IP "init" 3 .IX Item "init" The init function accepts a hash of \s-1MCE\s0 options. The gather option, if specified, will be set to undef due to being used internally by the module. .Sp .Vb 1 \& use MCE::Map; \& \& MCE::Map::init { \& chunk_size => 1, max_workers => 4, \& \& user_begin => sub { \& print "## ", MCE\->wid, " started\en"; \& }, \& \& user_end => sub { \& print "## ", MCE\->wid, " completed\en"; \& } \& }; \& \& my @a = mce_map { $_ * $_ } 1..100; \& \& print "\en", "@a", "\en"; \& \& \-\- Output \& \& ## 2 started \& ## 1 started \& ## 3 started \& ## 4 started \& ## 1 completed \& ## 4 completed \& ## 2 completed \& ## 3 completed \& \& 1 4 9 16 25 36 49 64 81 100 121 144 169 196 225 256 289 324 361 \& 400 441 484 529 576 625 676 729 784 841 900 961 1024 1089 1156 \& 1225 1296 1369 1444 1521 1600 1681 1764 1849 1936 2025 2116 2209 \& 2304 2401 2500 2601 2704 2809 2916 3025 3136 3249 3364 3481 3600 \& 3721 3844 3969 4096 4225 4356 4489 4624 4761 4900 5041 5184 5329 \& 5476 5625 5776 5929 6084 6241 6400 6561 6724 6889 7056 7225 7396 \& 7569 7744 7921 8100 8281 8464 8649 8836 9025 9216 9409 9604 9801 \& 10000 .Ve .SH "API DOCUMENTATION" .IX Header "API DOCUMENTATION" .IP "mce_map { code } list" 3 .IX Item "mce_map { code } list" Input data can be defined using a list or passing a reference to an array. .Sp .Vb 2 \& my @a = mce_map { $_ * 2 } 1..1000; \& my @b = mce_map { $_ * 2 } [ 1..1000 ]; .Ve .IP "mce_map_f { code } file" 3 .IX Item "mce_map_f { code } file" The fastest of these is the /path/to/file. Workers communicate the next offset position among themselves without any interaction from the manager process. .Sp .Vb 3 \& my @c = mce_map_f { chomp; $_ . "\er\en" } "/path/to/file"; \& my @d = mce_map_f { chomp; $_ . "\er\en" } $file_handle; \& my @e = mce_map_f { chomp; $_ . "\er\en" } \e$scalar; .Ve .IP "mce_map_s { code } sequence" 3 .IX Item "mce_map_s { code } sequence" Sequence can be defined as a list, an array reference, or a hash reference. The functions require both begin and end values to run. Step and format are optional. The format is passed to sprintf (% may be omitted below). .Sp .Vb 1 \& my ($beg, $end, $step, $fmt) = (10, 20, 0.1, "%4.1f"); \& \& my @f = mce_map_s { $_ } $beg, $end, $step, $fmt; \& my @g = mce_map_s { $_ } [ $beg, $end, $step, $fmt ]; \& \& my @h = mce_map_s { $_ } { \& begin => $beg, end => $end, step => $step, format => $fmt \& }; .Ve .IP "mce_map { code } iterator" 3 .IX Item "mce_map { code } iterator" An iterator reference can by specified for input data. Iterators are described under \*(L"\s-1SYNTAX\s0 for \s-1INPUT_DATA\*(R"\s0 at MCE::Core. .Sp .Vb 1 \& my @a = mce_map { $_ * 2 } make_iterator(10, 30, 2); .Ve .SH "MANUAL SHUTDOWN" .IX Header "MANUAL SHUTDOWN" .IP "finish" 3 .IX Item "finish" \&\s-1MCE\s0 workers remain persistent as much as possible after running. Shutdown occurs when the script exits. One can manually shutdown \s-1MCE\s0 by simply calling finish after running. This resets the \s-1MCE\s0 instance. .Sp .Vb 1 \& use MCE::Map; \& \& MCE::Map::init { \& chunk_size => 20, max_workers => \*(Aqauto\*(Aq \& }; \& \& my @a = mce_map { ... } 1..100; \& \& MCE::Map::finish; .Ve .SH "INDEX" .IX Header "INDEX" \&\s-1MCE\s0 .SH "AUTHOR" .IX Header "AUTHOR" Mario E. Roy, .SH "LICENSE" .IX Header "LICENSE" This program is free software; you can redistribute it and/or modify it under the terms of either: the \s-1GNU\s0 General Public License as published by the Free Software Foundation; or the Artistic License. .PP See for more information.