.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== .de Sp \" Vertical space (when we can't use .PP) .if t .sp .5v .if n .sp .. .de Vb \" Begin verbatim text .ft CW .nf .ne \\$1 .. .de Ve \" End verbatim text .ft R .fi .. .\" Set up some character translations and predefined strings. \*(-- will .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left .\" double quote, and \*(R" will give a right double quote. \*(C+ will .\" give a nicer C++. Capital omega is used to do unbreakable dashes and .\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, .\" nothing in troff, for use with C<>. .tr \(*W- .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' .ie n \{\ . ds -- \(*W- . ds PI pi . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch . ds L" "" . ds R" "" . ds C` "" . ds C' "" 'br\} .el\{\ . ds -- \|\(em\| . ds PI \(*p . ds L" `` . ds R" '' . ds C` . ds C' 'br\} .\" .\" Escape single quotes in literal strings from groff's Unicode transform. .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" .\" If the F register is >0, we'll generate index entries on stderr for .\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index .\" entries marked with X<> in POD. Of course, you'll have to process the .\" output yourself in some meaningful fashion. .\" .\" Avoid warning from groff about undefined register 'F'. .de IX .. .nr rF 0 .if \n(.g .if rF .nr rF 1 .if (\n(rF:(\n(.g==0)) \{\ . if \nF \{\ . de IX . tm Index:\\$1\t\\n%\t"\\$2" .. . if !\nF==2 \{\ . nr % 0 . nr F 2 . \} . \} .\} .rr rF .\" ======================================================================== .\" .IX Title "MCE::Map 3pm" .TH MCE::Map 3pm "2019-01-30" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l .nh .SH "NAME" MCE::Map \- Parallel map model similar to the native map function .SH "VERSION" .IX Header "VERSION" This document describes MCE::Map version 1.838 .SH "SYNOPSIS" .IX Header "SYNOPSIS" .Vb 2 \& ## Exports mce_map, mce_map_f, and mce_map_s \& use MCE::Map; \& \& ## Array or array_ref \& my @a = mce_map { $_ * $_ } 1..10000; \& my @b = mce_map { $_ * $_ } [ 1..10000 ]; \& \& ## File_path, glob_ref, or scalar_ref \& my @c = mce_map_f { chomp; $_ } "/path/to/file"; \& my @d = mce_map_f { chomp; $_ } $file_handle; \& my @e = mce_map_f { chomp; $_ } \e$scalar; \& \& ## Sequence of numbers (begin, end [, step, format]) \& my @f = mce_map_s { $_ * $_ } 1, 10000, 5; \& my @g = mce_map_s { $_ * $_ } [ 1, 10000, 5 ]; \& \& my @h = mce_map_s { $_ * $_ } { \& begin => 1, end => 10000, step => 5, format => undef \& }; .Ve .SH "DESCRIPTION" .IX Header "DESCRIPTION" This module provides a parallel map implementation via Many-Core Engine. \&\s-1MCE\s0 incurs a small overhead due to passing of data. A fast code block will run faster natively. However, the overhead will likely diminish as the complexity increases for the code. .PP .Vb 2 \& my @m1 = map { $_ * $_ } 1..1000000; ## 0.127 secs \& my @m2 = mce_map { $_ * $_ } 1..1000000; ## 0.304 secs .Ve .PP Chunking, enabled by default, greatly reduces the overhead behind the scene. The time for mce_map below also includes the time for data exchanges between the manager and worker processes. More parallelization will be seen when the code incurs additional \s-1CPU\s0 time. .PP .Vb 3 \& sub calc { \& sqrt $_ * sqrt $_ / 1.3 * 1.5 / 3.2 * 1.07 \& } \& \& my @m1 = map { calc } 1..1000000; ## 0.367 secs \& my @m2 = mce_map { calc } 1..1000000; ## 0.365 secs .Ve .PP Even faster is mce_map_s; useful when input data is a range of numbers. Workers generate sequences mathematically among themselves without any interaction from the manager process. Two arguments are required for mce_map_s (begin, end). Step defaults to 1 if begin is smaller than end, otherwise \-1. .PP .Vb 1 \& my @m3 = mce_map_s { calc } 1, 1000000; ## 0.270 secs .Ve .PP Although this document is about MCE::Map, the MCE::Stream module can write results immediately without waiting for all chunks to complete. This is made possible by passing the reference to an array (in this case \f(CW@m4\fR and \f(CW@m5\fR). .PP .Vb 1 \& use MCE::Stream; \& \& sub calc { \& sqrt $_ * sqrt $_ / 1.3 * 1.5 / 3.2 * 1.07 \& } \& \& my @m4; mce_stream \e@m4, sub { calc }, 1..1000000; \& \& ## Completes in 0.272 secs. This is amazing considering the \& ## overhead for passing data between the manager and workers. \& \& my @m5; mce_stream_s \e@m5, sub { calc }, 1, 1000000; \& \& ## Completed in 0.176 secs. Like with mce_map_s, specifying a \& ## sequence specification turns out to be faster due to lesser \& ## overhead for the manager process. .Ve .SH "OVERRIDING DEFAULTS" .IX Header "OVERRIDING DEFAULTS" The following list options which may be overridden when loading the module. .PP .Vb 3 \& use Sereal qw( encode_sereal decode_sereal ); \& use CBOR::XS qw( encode_cbor decode_cbor ); \& use JSON::XS qw( encode_json decode_json ); \& \& use MCE::Map \& max_workers => 4, # Default \*(Aqauto\*(Aq \& chunk_size => 100, # Default \*(Aqauto\*(Aq \& tmp_dir => "/path/to/app/tmp", # $MCE::Signal::tmp_dir \& freeze => \e&encode_sereal, # \e&Storable::freeze \& thaw => \e&decode_sereal # \e&Storable::thaw \& ; .Ve .PP From \s-1MCE 1.8\s0 onwards, Sereal 3.015+ is loaded automatically if available. Specify \f(CW\*(C`Sereal =\*(C'\fR 0> to use Storable instead. .PP .Vb 1 \& use MCE::Map Sereal => 0; .Ve .SH "CUSTOMIZING MCE" .IX Header "CUSTOMIZING MCE" .IP "MCE::Map\->init ( options )" 3 .IX Item "MCE::Map->init ( options )" .PD 0 .IP "MCE::Map::init { options }" 3 .IX Item "MCE::Map::init { options }" .PD The init function accepts a hash of \s-1MCE\s0 options. The gather option, if specified, is ignored due to being used internally by the module. .Sp .Vb 1 \& use MCE::Map; \& \& MCE::Map::init { \& chunk_size => 1, max_workers => 4, \& \& user_begin => sub { \& print "## ", MCE\->wid, " started\en"; \& }, \& \& user_end => sub { \& print "## ", MCE\->wid, " completed\en"; \& } \& }; \& \& my @a = mce_map { $_ * $_ } 1..100; \& \& print "\en", "@a", "\en"; \& \& \-\- Output \& \& ## 2 started \& ## 1 started \& ## 3 started \& ## 4 started \& ## 1 completed \& ## 4 completed \& ## 2 completed \& ## 3 completed \& \& 1 4 9 16 25 36 49 64 81 100 121 144 169 196 225 256 289 324 361 \& 400 441 484 529 576 625 676 729 784 841 900 961 1024 1089 1156 \& 1225 1296 1369 1444 1521 1600 1681 1764 1849 1936 2025 2116 2209 \& 2304 2401 2500 2601 2704 2809 2916 3025 3136 3249 3364 3481 3600 \& 3721 3844 3969 4096 4225 4356 4489 4624 4761 4900 5041 5184 5329 \& 5476 5625 5776 5929 6084 6241 6400 6561 6724 6889 7056 7225 7396 \& 7569 7744 7921 8100 8281 8464 8649 8836 9025 9216 9409 9604 9801 \& 10000 .Ve .SH "API DOCUMENTATION" .IX Header "API DOCUMENTATION" .IP "MCE::Map\->run ( sub { code }, list )" 3 .IX Item "MCE::Map->run ( sub { code }, list )" .PD 0 .IP "mce_map { code } list" 3 .IX Item "mce_map { code } list" .PD Input data may be defined using a list or an array reference. Unlike MCE::Loop, Flow, and Step, specifying a hash reference as input data isn't allowed. .Sp .Vb 2 \& my @a = mce_map { $_ * 2 } 1..1000; \& my @b = mce_map { $_ * 2 } \e@list; \& \& my @z = mce_map { $_ * 2 } \e%hash; # not supported .Ve .IP "MCE::Map\->run_file ( sub { code }, file )" 3 .IX Item "MCE::Map->run_file ( sub { code }, file )" .PD 0 .IP "mce_map_f { code } file" 3 .IX Item "mce_map_f { code } file" .PD The fastest of these is the /path/to/file. Workers communicate the next offset position among themselves with zero interaction by the manager process. .Sp .Vb 3 \& my @c = mce_map_f { chomp; $_ . "\er\en" } "/path/to/file"; # faster \& my @d = mce_map_f { chomp; $_ . "\er\en" } $file_handle; \& my @e = mce_map_f { chomp; $_ . "\er\en" } \e$scalar; .Ve .ie n .IP "MCE::Map\->run_seq ( sub { code }, $beg, $end [, $step, $fmt ] )" 3 .el .IP "MCE::Map\->run_seq ( sub { code }, \f(CW$beg\fR, \f(CW$end\fR [, \f(CW$step\fR, \f(CW$fmt\fR ] )" 3 .IX Item "MCE::Map->run_seq ( sub { code }, $beg, $end [, $step, $fmt ] )" .PD 0 .ie n .IP "mce_map_s { code } $beg, $end [, $step, $fmt ]" 3 .el .IP "mce_map_s { code } \f(CW$beg\fR, \f(CW$end\fR [, \f(CW$step\fR, \f(CW$fmt\fR ]" 3 .IX Item "mce_map_s { code } $beg, $end [, $step, $fmt ]" .PD Sequence may be defined as a list, an array reference, or a hash reference. The functions require both begin and end values to run. Step and format are optional. The format is passed to sprintf (% may be omitted below). .Sp .Vb 1 \& my ($beg, $end, $step, $fmt) = (10, 20, 0.1, "%4.1f"); \& \& my @f = mce_map_s { $_ } $beg, $end, $step, $fmt; \& my @g = mce_map_s { $_ } [ $beg, $end, $step, $fmt ]; \& \& my @h = mce_map_s { $_ } { \& begin => $beg, end => $end, \& step => $step, format => $fmt \& }; .Ve .IP "MCE::Map\->run ( sub { code }, iterator )" 3 .IX Item "MCE::Map->run ( sub { code }, iterator )" .PD 0 .IP "mce_map { code } iterator" 3 .IX Item "mce_map { code } iterator" .PD An iterator reference may be specified for input_data. Iterators are described under section \*(L"\s-1SYNTAX\s0 for \s-1INPUT_DATA\*(R"\s0 at MCE::Core. .Sp .Vb 1 \& my @a = mce_map { $_ * 2 } make_iterator(10, 30, 2); .Ve .SH "MANUAL SHUTDOWN" .IX Header "MANUAL SHUTDOWN" .IP "MCE::Map\->finish" 3 .IX Item "MCE::Map->finish" .PD 0 .IP "MCE::Map::finish" 3 .IX Item "MCE::Map::finish" .PD Workers remain persistent as much as possible after running. Shutdown occurs automatically when the script terminates. Call finish when workers are no longer needed. .Sp .Vb 1 \& use MCE::Map; \& \& MCE::Map::init { \& chunk_size => 20, max_workers => \*(Aqauto\*(Aq \& }; \& \& my @a = mce_map { ... } 1..100; \& \& MCE::Map::finish; .Ve .SH "INDEX" .IX Header "INDEX" \&\s-1MCE\s0, MCE::Core .SH "AUTHOR" .IX Header "AUTHOR" Mario E. Roy,