From 024a343da2b6f39f5929f976187b94eca6091e90 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 11 Sep 2024 17:18:33 +0200 Subject: [PATCH 1/3] chore: update tests and examples to spark 3.5.2 --- .../examples/example-history-app.yaml | 2 +- .../examples/example-history-server.yaml | 2 +- .../examples/example-sparkapp-configmap.yaml | 2 +- .../examples/example-sparkapp-image.yaml | 2 +- .../examples/example-sparkapp-pvc.yaml | 2 +- .../examples/example-sparkapp-s3-private.yaml | 2 +- .../examples/example-sparkapp-streaming.yaml | 2 +- .../getting_started/getting_started.sh | 2 +- .../getting_started/getting_started.sh.j2 | 2 +- .../pages/usage-guide/job-dependencies.adoc | 9 ++++----- .../partials/supported-versions.adoc | 5 ++--- examples/README-examples.md | 4 ++-- ....5.1.jar => ny-tlc-report-1.1.0-3.5.2.jar} | Bin 135330 -> 135192 bytes .../ny-tlc-report-external-dependencies.yaml | 2 +- examples/ny-tlc-report-image.yaml | 2 +- examples/ny-tlc-report.yaml | 4 ++-- rust/crd/src/affinity.rs | 2 +- rust/crd/src/history.rs | 2 +- tests/test-definition.yaml | 5 ++--- 19 files changed, 25 insertions(+), 28 deletions(-) rename examples/{ny-tlc-report-1.1.0-3.5.1.jar => ny-tlc-report-1.1.0-3.5.2.jar} (90%) diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index 93945ce2..e3062f96 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index 11132fd8..4b7e1efb 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.1 + productVersion: 3.5.2 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index b94200b6..c95898dc 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index b2540c41..06dfe88a 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index 8a8fc1cf..cfd8addd 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 12869a16..04535100 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 60e0c244..37c09637 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index 7dc9fac9..2c04e4ac 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -73,7 +73,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index 9cbda5a2..0a34b9b3 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -73,7 +73,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index f22b8840..ee366676 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -57,7 +57,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable24.3.0 # <1> +FROM docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable24.11.0 # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" ---- @@ -74,8 +74,8 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev" # <1> - productVersion: "3.5.1" # <2> + custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1> + productVersion: "3.5.2" # <2> pullPolicy: IfNotPresent # <3> ... ---- @@ -160,9 +160,8 @@ spec: deps: requirements: - tabulate==0.8.9 # <2> -... +... ---- <1> The main application file. In this example it is assumed that the file is part of a custom image. <2> A Python package that is used by the application and installed when the application is submitted. - diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 4ec5fbdb..3928356f 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,6 +3,5 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) -- 3.4.3 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) -- 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) +- 3.5.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) +- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) diff --git a/examples/README-examples.md b/examples/README-examples.md index 6dbffb7b..4466c7ea 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ````text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp examples/ny-tlc-report-1.1.0-3.5.1.jar minio-mc-0:/tmp +kubectl cp examples/ny-tlc-report-1.1.0-3.5.2.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.1.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.2.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ```` diff --git a/examples/ny-tlc-report-1.1.0-3.5.1.jar b/examples/ny-tlc-report-1.1.0-3.5.2.jar similarity index 90% rename from examples/ny-tlc-report-1.1.0-3.5.1.jar rename to examples/ny-tlc-report-1.1.0-3.5.2.jar index 7ea1fbf8457971237418050af59e1ae7fcecb99c..c88b521ec73c936c2ebfbcf7e3ef7afa1cf93038 100644 GIT binary patch delta 4867 zcmZu#2{=^i8$aI}%V@?JVPvv2N)ltORAgznma$9}CAV^=MWiCPQsLHZLE%n(T;*1@ zDAjdGjk1(3#igV@AzDuP;1AJ{16 zZJ2zs)~&wBz50QwxvS?zA4}^OFQ2`{F`+=2?vqp^osVG{agT7(ZzXdfvI2JG!+PLE zJ`{ot!h_2YTXJ`k6FRXb{X}kxwq{!z3nZ;88Yp3|&Z@D-0JZ zoI#dErG`$i_ZKxq+_hxKzB*6WQ0rnBckXub-Mi~LV>Ppor{FEhD#|Oru(LDjEL&F4WHb{SQ4h&rl!6v9^QI2hnvo$xAS$MxBveR)$;ug0}ij4D6#K0G_r7mK)k zI(K@!YW==(N2~Ac=P9zvOjGSb^$?ci*`>kdW%SN|c#S zU*qMaomi0*5}g`G#TS8D_x&O^%}<0@(-V@-JEx|p{8<5JgImL;((Q~|)?=9e8n|<7dlcN${66Z?If@3zZZA;cfoO*a|g?UP*plQR?(^Fcq` zY0>zG#We$ZKS@`=p3{GFb=ktCq1#@0m|ZZWd% z-%a1{l2trA!DCs-u1o5tnxa12d1_o=n0)c5ZNql!=A}6ck0!rmRo&R2TY5m(B4E9G z2;MR4S^~X*?zYcA^78)X(z2x575lTx=B(9RQ(*f1{bJ1Y!v3nH9^lm{d}}IAq%Dx)V&r-Y=}dJAwDO%|9=A{fYKz z)1s1=y{sa}3LnOzlU3XFrzuAiz}K|R&I#_!Mw9rrvmHPFd##aIUgOdKV%31*BZubh z!Cg;G4E9FPouXo&vMKp-_}WJg_dNgoZs*3z8KuGA0i||5x$j(KHa1229(eax@BOaW ziw;jrGWJDI^Ln#9axf^9|Feho|>r`;}{b#HQ>5@W~jOw{`a zGz{w+J#y?2V?<CjJ$|*@1HL=I%3^Y>yTv~id@P|&`*_E3 zzX@%e7r7hLj&rnDwY_@GZP#hnSr25B?do`|efG$h9lGTwn`qG3njAV^e0{ie5U7iT zC+$-tXH*wIjhaygx!U3*R*rxawer>%3G7thGAS4(*0h;R3PJBxKwDJez=r3fKt-H1 z?XbdEak0S8ZJ+kGRet zqmPa`s6#%n(oQjlIkPafiwdqN$@d4;mrmlg1e8(hmbU;0?nA^S=F5N{oy-9u zm`2hkxSB}R*MNxO?`K271~Nyobr`|Gl_fsm;06T}rG~$U0;a@qJ?P-dj1u+dNxTz5 z`!^w;Wa$BrKm6Sy71%40B#``8vP6>3EWjnLNEjEvJT;ksh?T~XgjND?l9{CRIuL&g znj=a02OJ~cB`8QO%9Pt3&`&A2-2<1ENngs1Ac&F;dkX3)6zz;`3Vq%V0GS+-K`uu! z_#TKznlSAPuvI0F7GF;&@bMQg)pqE#ffIuQU5exn_4hXVjS2}~Z6Oo@Toe0tV1irw z0T(F3<-I_E+(;XmayI4y5l^3EHqrtiGvt&jr?8h$(1rcIfa;hleSnMKAy8AO)DK3J zh0;X~@Bs!wtzl^YkSvpstx%I$$nFCS=-mf2WDMtZ%uIGgGqOqn!%XEIs_GAk>yf63 z;2h8KFwAq_0%yLP*AEuqLD3=WzRSY!7{*u&~%19yT(8!w0ghoKodP!p%G~7VaS0lq-!fEHXG^ebjd$Krgh8A2Noa zgMd$JWD&?BzG#rogZVhllx1(#m*l%+0EX>Dw|srOEun%G7?Qe^rASv`E$X%&57$Tm zsoQWj|H@93^=cIi8z;xKVk(l<9cvyO78DsdGC+}|@dXoTAq84^+Ds&;1N)_bV?gXS z|L|#l$N#5A)if*ILdMuw`iM3O)w^10q}RsTX>qu@OemCLpb&=)RU&Eza@2f&Ktf~F z-r(eROOxVU+(3XZj_8Si1swrS?M`oiRx-Va6&Me2a^Ayi0$uGZVl5a48@|f66DZwZ zbk5(vbY!@N+3O#l8HHh^*%+oFM}-H4IBqBl{KK32KVGA@k)puH$O);!R2(-Zt5AX? zH+G?-)=)i|G>WQV^P81AzNmV(NL_xjmaT<68g4@BdeM+>KseIn5Z^WpK9O%@MuzSm z3=CUlII{W!>xOogw=ig9B&=7!wM+;Vd$$$t{yG@eF=J#n&0D{!z&{!I7}76S0r`zS zDAG;S{Vp|)?{4cDAlen^mXF#HZdbsKWu-i~+w0UZA%>-QkL0)~2UaNH)DCSY{H*;~ zq~4N=CtE^S2ev6u zyAk#ym{w^AV5=|CB0lD$Wj6nuaj*9$PomH4xM99AEFny$!22#j zV5HYygFcX9=Zz<+< zzjVhib-7>{LMam;C)=?;9^DLv8v3rF7C?h<*cAyKRB;2+QIIO?f|3fjT@{}sqq#kE zn_dX2yD2K=gkegk>#hS>dmR|T4peedyN}RrtcKbw+=fKFScrJWO#mHl0q%EAd<<9{ z|BX7+3uOyY8+H}c!vYqrC)?bO&vV&Zko#^#TYH!ch85g}?^w8zEWXUVkAR1WCZmrG zM@}jNTC3qkB%dW}h|j%d0Arp4TXB*-nQ#Gu9h*bl~<}4o_F4M;Ea}=RuyMO@sUU)DcxM>?Ji}@ z%a&uFd>KZczaBFh!?3;RD=q*5^e3Z_Cz3)a0efoU1steF7H|IOU}wPMNrAB7QR+O&%y?fIfT!bQhN@}IznrNl`chlCD)essHy;_v_Zh40bW%$}%z zr-qy8>yS|%nQ_+Hi_`Ay;C=O;ua1w`#z@Y{+g_(%7W!XNR@wJT+V-!1P7JzO>Zm`@ ziLWUb@b!1C{z)a}RBbV+?Vi|*Jy$eYU%3_UtcYXjR~bxNF{$oVrdHaA)>iL<*fYH~ zr<+wbMhTj}$AnwPr%Hba*O>Jmz>raIr}m~i&@I$yr} z{<>Tue@0uSueF|Xk(N|XQQf|>PDQue4f!T_2cB|MZpqExvek$AHgWON!kE81dN@vr zNyDB~%*5H`Y31&*nbu^)Zo5Ti$sh z#<_oiP4aL`f^w=)!U|za)+FN0Iv*I)Q`#9C<@)?+S5exY=zUX%;GRnKmIG-G~m z+tkel+DC8gGxd2kx@2ay+Z*p=6PSZ-%|$-T3VnAUS7%?l+;+G#b9?`xQXDO+t7S0N1?nfuWuyYy-$79Mt65>TVAqzqhN0Ij^!U}&N)R7C$yiop8e)o&WUv! z-Py-;iW_CiPrEn1e5wE9LsQ)#KljfaQSKb`M)Ndt=b`|ob+CEPivHYTPD}o#%Elvw}A1{~6A% z^W4H)srJ*us++swHM6dDEKdj{u6@ov5&dxg8ppW@oo!hcB6fq<#=SeIuNt~=+Nj4# zFu2^yLNCd6U338vd}YdEJY^-u*of%luPX=ufo@ zO%KvsM|a;cN_8!Dw6{8A)YM@8(lj^y53_ke!Ml6swge4TD@nI%Y&ogK9u0gGSL20lix*s2GT>&+Z|L#*)bNLdMG8kwKC!m`eoK9mL)D|1^QvkqYxf$J z6zEx+m|wqbS#4G9CDh~j3k^!ODhl=*l^axC#SF58jugRwjHAoaiF%)`fF(J`2uQE`de(GFc=sPREQgPih{wJXC*0Xe+!!(Z!7ML#Yqh>f^mZu`H z2vee6+>6kIZ$be>uxYI~E&X1^QEEk?O+ji@9DOxvrXtXZu3=F*m@ZclNW{fYrqbPe zq(F{ZndODx6ocd!p04x!%1{4LO$+n?yn|v19{o2+=k;(@b%uxwzxV=8SSp2ch0|t$ zH&l3tQqvULS%Ev$wNT0i)KlJst@dE`SC*bIXg1hKZIP8!gKzBtLn!YE4$*w%>O8e1 zf?ZT#zdeu_u3rH5e~)P`0y}9x$V3lDGH|A_$pggFL=(P>FvuGmqb`L=b~(MQ2mrs+ z%fvw7z@YL%Vx|dEwIk^i&^29ZQs*?G?pm;cDhqX86$u*YVxm(MUWf*tRHElkTnRb<;hp5@pl~PF$iW_lMwl4gOh*IM~eEP|Gq)Lv1?X^ zn|ksA&cVLk(r{o1r~+O1a1dx{eeZ)K=3^xwf&mvA{s*Y$hIan}Rs1XfVQZfu!~e8r9q+u6K6yHA)rLwEwHn}I zSj?*E^|3#kl2fYrwPCE3f;%gT0;`q zPRNILl4JgZ5duEKxN#tl1RlULN!*dr1f-CS&k$6X!u7OC?aDUkcI}oJ7M_S<%3|@n zFaTFd;W{E&2csS;-ba<1kIrMN7;XIwm@kD}QtsQ4dkJU-prIAe9B-Y65qoIs(?r_D?~U6q zkAa(u3h|dQKVlss86lE*t+)G${7r%^SzJ}bz4^Q3W0Gr%xcXwr+ApKkEtAFRsTq%~ zBt`;7I0qKV;uA$@YrXIGM?NY#8?~Hx!5qV2w=6zg#M0?V+Ti~Gvf9E)))aUN$LV<; z%tT$b8Qn%C44Gqa5eajmD7beV9wuS-dJ^V9V-~Hd$CvO(T9IXr;hF7OpQ+ zH8EvrRSII+|G+Xm1oIJ#I?P5E3V~(7{xQ3@Y(IA!G=SL5@4bgWXeKvi7a)i3pDwsb z4%Z4K?-s8`?^rB0h8;ljNKvfZBc*^I`(A58R!?6Q84?ro{gbIRe3Pp-D&aBo74LAD zYS=B0E5p~f(a2%TkF}YmJd#~pL+yFWkfqe_)Dd6g&HhSEXuec4h&^wfrikO}KeE=I3Fs3RES%X1X zpny-M^fV(tBCrd Date: Wed, 11 Sep 2024 17:22:33 +0200 Subject: [PATCH 2/3] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17960ea0..eef5b1f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file. - `affinity` - `volumes` - `volumeMounts` +- Update tests and docs to Spark version 3.5.2 ([#459]) ### Fixed @@ -18,6 +19,7 @@ All notable changes to this project will be documented in this file. [#450]: https://github.com/stackabletech/spark-k8s-operator/pull/450 [#451]: https://github.com/stackabletech/spark-k8s-operator/pull/451 +[#459]: https://github.com/stackabletech/spark-k8s-operator/pull/459 ## [24.7.0] - 2024-07-24 From 714d76d5b53bd043ed291e08174e1842d2e5d291 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 12 Sep 2024 03:54:43 -0400 Subject: [PATCH 3/3] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eef5b1f1..0b9f308c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ All notable changes to this project will be documented in this file. - Fix `envOverrides` for SparkApplication and SparkHistoryServer ([#451]). +### Removed + +- Support for Spark versions 3.4.2 and 3.4.3 has been dropped ([#459]). + [#450]: https://github.com/stackabletech/spark-k8s-operator/pull/450 [#451]: https://github.com/stackabletech/spark-k8s-operator/pull/451 [#459]: https://github.com/stackabletech/spark-k8s-operator/pull/459