Skip to content

Commit a39497a

Browse files
xavierguihotxavierguihot
committed
Change function name partialMap to update
1 parent 590d6da commit a39497a

File tree

10 files changed

+58
-44
lines changed

10 files changed

+58
-44
lines changed

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ sc.textFileWithFileName("/my/input/folder/path")
117117
rdd.saveAsTextFileByKey("/my/output/folder/path")
118118

119119
// Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):
120-
rdd.partialMap { case a if a % 2 == 0 => 2 * a }
120+
rdd.update { case a if a % 2 == 0 => 2 * a }
121121

122122
// Concept mapper (the following example transforms RDD((1, "a"), (2, "b")) into RDD(1, (1, "a")), (2, (2, "b"))):
123123
rdd.withKey(_._1)
@@ -301,7 +301,7 @@ With sbt:
301301
```scala
302302
resolvers += "jitpack" at "https://jitpack.io"
303303

304-
libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "2.0.1"
304+
libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "2.0.2"
305305
```
306306

307307
With maven:
@@ -317,7 +317,7 @@ With maven:
317317
<dependency>
318318
<groupId>com.github.xavierguihot</groupId>
319319
<artifactId>spark_helper</artifactId>
320-
<version>2.0.1</version>
320+
<version>2.0.2</version>
321321
</dependency>
322322
```
323323

@@ -331,7 +331,7 @@ allprojects {
331331
}
332332
333333
dependencies {
334-
compile 'com.github.xavierguihot:spark_helper:2.0.1'
334+
compile 'com.github.xavierguihot:spark_helper:2.0.2'
335335
}
336336
```
337337

‎build.sbt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name := "spark_helper"
22

3-
version := "2.0.1"
3+
version := "2.0.2"
44

55
scalaVersion := "2.11.12"
66

‎docs/com/spark_helper/SparkHelper$$RDDExtensions.html‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -364,34 +364,6 @@ <h4 class="signature">
364364
</a>
365365
</span>
366366
<div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
367-
</li><li name="com.spark_helper.SparkHelper.RDDExtensions#partialMap" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
368-
<a id="partialMap(pf:PartialFunction[T,T]):org.apache.spark.rdd.RDD[T]"></a>
369-
<a id="partialMap(PartialFunction[T,T]):RDD[T]"></a>
370-
<h4 class="signature">
371-
<span class="modifier_kind">
372-
<span class="modifier"></span>
373-
<span class="kind">def</span>
374-
</span>
375-
<span class="symbol">
376-
<span class="name">partialMap</span><span class="params">(<span name="pf">pf: <span class="extype" name="scala.PartialFunction">PartialFunction</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>, <span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>
377-
</span>
378-
</h4><span class="permalink">
379-
<a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@partialMap(pf:PartialFunction[T,T]):org.apache.spark.rdd.RDD[T]" title="Permalink" target="_top">
380-
<img src="../../lib/permalink.png" alt="Permalink" />
381-
</a>
382-
</span>
383-
<p class="shortcomment cmt">Map an RDD to the same type, by applying a partial function and the
384-
identity otherwise.</p><div class="fullcomment"><div class="comment cmt"><p>Map an RDD to the same type, by applying a partial function and the
385-
identity otherwise.</p><p>Avoids having <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">case x => x</code>.</p><p>Similar idea to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">.collect</code>,
386-
but instead of skipping non-matching items, it keeps them as-is.</p><pre>sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">2</span>, <span class="num">7</span>, <span class="num">8</span>)).partialMap { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
387-
<span class="cmt">// is equivalent to:</span>
388-
sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">2</span>, <span class="num">7</span>, <span class="num">8</span>)).map {
389-
<span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a
390-
<span class="kw">case</span> a <span class="kw">=&gt;</span> a
391-
}
392-
<span class="cmt">// in order to map to:</span>
393-
sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">4</span>, <span class="num">7</span>, <span class="num">16</span>))</pre></div><dl class="paramcmts block"><dt class="param">pf</dt><dd class="cmt"><p>the partial function to apply</p></dd><dt>returns</dt><dd class="cmt"><p>an rdd of the same type, for which each element is either the
394-
application of the partial function where defined or the identity.</p></dd></dl></div>
395367
</li><li name="com.spark_helper.SparkHelper.RDDExtensions#rdd" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
396368
<a id="rdd:org.apache.spark.rdd.RDD[T]"></a>
397369
<a id="rdd:RDD[T]"></a>
@@ -443,6 +415,34 @@ <h4 class="signature">
443415
</a>
444416
</span>
445417
<div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
418+
</li><li name="com.spark_helper.SparkHelper.RDDExtensions#update" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
419+
<a id="update(pf:PartialFunction[T,T]):org.apache.spark.rdd.RDD[T]"></a>
420+
<a id="update(PartialFunction[T,T]):RDD[T]"></a>
421+
<h4 class="signature">
422+
<span class="modifier_kind">
423+
<span class="modifier"></span>
424+
<span class="kind">def</span>
425+
</span>
426+
<span class="symbol">
427+
<span class="name">update</span><span class="params">(<span name="pf">pf: <span class="extype" name="scala.PartialFunction">PartialFunction</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>, <span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>
428+
</span>
429+
</h4><span class="permalink">
430+
<a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@update(pf:PartialFunction[T,T]):org.apache.spark.rdd.RDD[T]" title="Permalink" target="_top">
431+
<img src="../../lib/permalink.png" alt="Permalink" />
432+
</a>
433+
</span>
434+
<p class="shortcomment cmt">Map an RDD to the same type, by applying a partial function and the
435+
identity otherwise.</p><div class="fullcomment"><div class="comment cmt"><p>Map an RDD to the same type, by applying a partial function and the
436+
identity otherwise.</p><p>Avoids having <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">case x => x</code>.</p><p>Similar idea to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">.collect</code>,
437+
but instead of skipping non-matching items, it keeps them as-is.</p><pre>sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">2</span>, <span class="num">7</span>, <span class="num">8</span>)).update { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
438+
<span class="cmt">// is equivalent to:</span>
439+
sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">2</span>, <span class="num">7</span>, <span class="num">8</span>)).map {
440+
<span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a
441+
<span class="kw">case</span> a <span class="kw">=&gt;</span> a
442+
}
443+
<span class="cmt">// in order to map to:</span>
444+
sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">4</span>, <span class="num">7</span>, <span class="num">16</span>))</pre></div><dl class="paramcmts block"><dt class="param">pf</dt><dd class="cmt"><p>the partial function to apply</p></dd><dt>returns</dt><dd class="cmt"><p>an rdd of the same type, for which each element is either the
445+
application of the partial function where defined or the identity.</p></dd></dl></div>
446446
</li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
447447
<a id="wait():Unit"></a>
448448
<a id="wait():Unit"></a>
@@ -529,7 +529,7 @@ <h4 class="signature">
529529
</a>
530530
</span>
531531
<p class="shortcomment cmt">Map an RDD of A to an RDD of (B, A) where B is determined from A.</p><div class="fullcomment"><div class="comment cmt"><p>Map an RDD of A to an RDD of (B, A) where B is determined from A.</p><p>Replaces for example <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.map(x => (x.smthg, x))</code>
532-
with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.withKey(_.smthg)</code>.</p><pre>RDD((<span class="num">1</span>, <span class="lit">"a"</span>), (<span class="num">2</span>, <span class="lit">"b"</span>)).withKey(_._1) <span class="cmt">// RDD(1, (1, "a")), (2, (2, "b"))</span></pre></div><dl class="paramcmts block"><dt class="param">toKey</dt><dd class="cmt"><p>the function to apply to extract the key</p></dd><dt>returns</dt><dd class="cmt"><p>an rdd of (K, V) from an RDD of V where K is determined by
532+
with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.withKey(_.smthg)</code>.</p><pre>RDD((<span class="num">1</span>, <span class="lit">"a"</span>), (<span class="num">2</span>, <span class="lit">"b"</span>)).withKey(_._1) <span class="cmt">// RDD(1, (1, "a")), (2, (2, "b"))</span></pre></div><dl class="paramcmts block"><dt class="param">toKey</dt><dd class="cmt"><p>the function to apply to extract the key</p></dd><dt>returns</dt><dd class="cmt"><p>an RDD of (K, V) from an RDD of V where K is determined by
533533
applying toKey on V.</p></dd></dl></div>
534534
</li></ol>
535535
</div>

‎docs/com/spark_helper/SparkHelper$.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ <h4 id="signature" class="signature">
7878
rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>)
7979

8080
<span class="cmt">// Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):</span>
81-
rdd.partialMap { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
81+
rdd.update { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
8282

8383
<span class="cmt">// For when input files contain commas and textFile can't handle it:</span>
8484
sc.textFile(<span class="std">Seq</span>(<span class="lit">"path/hello,world.txt"</span>, <span class="lit">"path/hello_world.txt"</span>))</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src

‎docs/com/spark_helper/package.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ <h4 class="signature">
332332
rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>)
333333

334334
<span class="cmt">// Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):</span>
335-
rdd.partialMap { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
335+
rdd.update { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
336336

337337
<span class="cmt">// For when input files contain commas and textFile can't handle it:</span>
338338
sc.textFile(<span class="std">Seq</span>(<span class="lit">"path/hello,world.txt"</span>, <span class="lit">"path/hello_world.txt"</span>))</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src

‎docs/index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
<div class="ui-west-center">
2626
<div id="filter">
2727
<div id="textfilter"></div>
28-
<div id="letters"><span>#</span><a target="template" href="index/index-a.html">A</a><span>B</span><a target="template" href="index/index-c.html">C</a><a target="template" href="index/index-d.html">D</a><a target="template" href="index/index-e.html">E</a><a target="template" href="index/index-f.html">F</a><span>G</span><a target="template" href="index/index-h.html">H</a><a target="template" href="index/index-i.html">I</a><a target="template" href="index/index-j.html">J</a><a target="template" href="index/index-k.html">K</a><a target="template" href="index/index-l.html">L</a><a target="template" href="index/index-m.html">M</a><a target="template" href="index/index-n.html">N</a><a target="template" href="index/index-o.html">O</a><a target="template" href="index/index-p.html">P</a><span>Q</span><a target="template" href="index/index-r.html">R</a><a target="template" href="index/index-s.html">S</a><a target="template" href="index/index-t.html">T</a><span>U</span><a target="template" href="index/index-v.html">V</a><a target="template" href="index/index-w.html">W</a><span>X</span><a target="template" href="index/index-y.html">Y</a><span>Z</span> – <span>deprecated</span></div>
28+
<div id="letters"><span>#</span><a target="template" href="index/index-a.html">A</a><span>B</span><a target="template" href="index/index-c.html">C</a><a target="template" href="index/index-d.html">D</a><a target="template" href="index/index-e.html">E</a><a target="template" href="index/index-f.html">F</a><span>G</span><a target="template" href="index/index-h.html">H</a><a target="template" href="index/index-i.html">I</a><a target="template" href="index/index-j.html">J</a><a target="template" href="index/index-k.html">K</a><a target="template" href="index/index-l.html">L</a><a target="template" href="index/index-m.html">M</a><a target="template" href="index/index-n.html">N</a><a target="template" href="index/index-o.html">O</a><a target="template" href="index/index-p.html">P</a><span>Q</span><a target="template" href="index/index-r.html">R</a><a target="template" href="index/index-s.html">S</a><a target="template" href="index/index-t.html">T</a><a target="template" href="index/index-u.html">U</a><a target="template" href="index/index-v.html">V</a><a target="template" href="index/index-w.html">W</a><span>X</span><a target="template" href="index/index-y.html">Y</a><span>Z</span> – <span>deprecated</span></div>
2929
</div>
3030
<div class="pack" id="tpl">
3131

‎docs/index/index-p.html‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@
1616
</div><div class="entry">
1717
<div class="name">PairRDDExtensions</div>
1818
<div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
19-
</div><div class="entry">
20-
<div class="name">partialMap</div>
21-
<div class="occurrences"><a href="../com/spark_helper/SparkHelper$$RDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.RDDExtensions">RDDExtensions</a> </div>
2219
</div><div class="entry">
2320
<div class="name">previousDay</div>
2421
<div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> <a href="../com/spark_helper/DateHelper$$StringExtensions.html" class="extype" name="com.spark_helper.DateHelper.StringExtensions">StringExtensions</a> </div>

‎docs/index/index-u.html‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<!DOCTYPE html >
2+
<html>
3+
<head>
4+
<title></title>
5+
<meta name="description" content="" />
6+
<meta name="keywords" content="" />
7+
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
8+
9+
<link href="../lib/ref-index.css" media="screen" type="text/css" rel="stylesheet" />
10+
<script type="text/javascript" src="../lib/jquery.js"></script>
11+
12+
</head>
13+
<body><div class="entry">
14+
<div class="name">update</div>
15+
<div class="occurrences"><a href="../com/spark_helper/SparkHelper$$RDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.RDDExtensions">RDDExtensions</a> </div>
16+
</div></body>
17+
</html>

‎src/main/scala/com/spark_helper/SparkHelper.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ import scala.util.Random
5353
* rdd.saveAsTextFileByKey("/my/output/folder/path")
5454
*
5555
* // Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):
56-
* rdd.partialMap { case a if a % 2 == 0 => 2 * a }
56+
* rdd.update { case a if a % 2 == 0 => 2 * a }
5757
*
5858
* // For when input files contain commas and textFile can't handle it:
5959
* sc.textFile(Seq("path/hello,world.txt", "path/hello_world.txt"))
@@ -79,7 +79,7 @@ object SparkHelper extends Serializable {
7979
* but instead of skipping non-matching items, it keeps them as-is.
8080
*
8181
* {{{
82-
* sc.parallelize(Array(1, 3, 2, 7, 8)).partialMap { case a if a % 2 == 0 => 2 * a }
82+
* sc.parallelize(Array(1, 3, 2, 7, 8)).update { case a if a % 2 == 0 => 2 * a }
8383
* // is equivalent to:
8484
* sc.parallelize(Array(1, 3, 2, 7, 8)).map {
8585
* case a if a % 2 == 0 => 2 * a
@@ -93,7 +93,7 @@ object SparkHelper extends Serializable {
9393
* @return an rdd of the same type, for which each element is either the
9494
* application of the partial function where defined or the identity.
9595
*/
96-
def partialMap(pf: T ~> T): RDD[T] =
96+
def update(pf: T ~> T): RDD[T] =
9797
rdd.map {
9898
case x if pf.isDefinedAt(x) => pf(x)
9999
case x => x

‎src/test/scala/com/spark_helper/SparkHelperTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ class SparkHelperTest
400400

401401
test("Partial map") {
402402
val in = sc.parallelize(Array(1, 3, 2, 7, 8))
403-
val computedOut = in.partialMap { case a if a % 2 == 0 => 2 * a }
403+
val computedOut = in.update { case a if a % 2 == 0 => 2 * a }
404404
val expectedOut = sc.parallelize(Array(1, 3, 4, 7, 16))
405405
assertRDDEquals(computedOut, expectedOut)
406406
}

0 commit comments

Comments
 (0)