Skip to content

Commit 7eceedd

Browse files
JolanrensenJolanrensen
committed
Updated intellij examples: added example responses for youtube, added fake movies csv, fixed pclass in titanic example
1 parent e8ce4b1 commit 7eceedd

File tree

9 files changed

+241
-54
lines changed

9 files changed

+241
-54
lines changed

‎examples/idea-examples/movies/src/main/resources/movies.csv

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
movieId,title,genres
2+
9b30aff7943f44579e92c261f3adc193,Women in Black (1997),Fantasy|Suspenseful|Comedy
3+
2a1ba1fc5caf492a80188e032995843e,Bumblebee Movie (2007),Comedy|Jazz|Family|Animation
4+
f44ceb4771504342bb856d76c112d5a6,Magical School Boy and the Rock of Wise Men (2001),Fantasy|Growing up|Magic
5+
43d02fb064514ff3bd30d1e3a7398357,Master of the Jewlery: The Company of the Jewel (2001),Fantasy|Magic|Suspenseful
6+
6aa0d26a483148998c250b9c80ddf550,Sun Conflicts: Part IV: A Novel Espair (1977),Fantasy
7+
eace16e59ce24eff90bf8924eb6a926c,The Outstanding Bulk (2008),Fantasy|Superhero|Family
8+
ae916bc4844a4bb7b42b70d9573d05cd,In Automata (2014),Horror|Existential
9+
c1f0a868aeb44c5ea8d154ec3ca295ac,Interplanetary (2014),Sci-fi|Futuristic
10+
9595b771f87f42a3b8dd07d91e7cb328,Woods Run (1994),Family|Drama
11+
aa9fc400e068443488b259ea0802a975,Anthropod-Dude (2002),Superhero|Fantasy|Family|Growing up
12+
22d20c2ba11d44cab83aceea39dc00bd,The Chamber (2003),Comedy|Drama
13+
8cf4d0c1bd7b41fab6af9d92c892141f,That Thing About an Iceberg (1997),Drama|History|Family|Romance
14+
c2f3e7588da84684a7d78d6bd8d8e1f4,Vehicles (2006),Animation|Family
15+
ce06175106af4105945f245161eac3c7,Playthings Tale (1995),Animation|Family
16+
ee28d7e69103485c83e10b8055ef15fb,Metal Man 2 (2010),Fantasy|Superhero|Family
17+
c32bdeed466f4ec09de828bb4b6fc649,Surgeon Odd in the Omniverse of Crazy (2022),Fantasy|Superhero|Family|Horror
18+
d4a325ab648a42c4a2d6f35dfabb387f,Bad Dream on Pine Street (1984),Horror
19+
60ebe74947234ddcab49dea1a958faed,The Shimmering (1980),Horror
20+
f24327f2b05147b197ca34bf13ae3524,Krubit: Societal Teachings for Do Many Good Amazing Country of Uzbekistan (2006),Comedy
21+
2bb29b3a245e434fa80542e711fd2cee,This is No Movie (1950),(no genres listed)

‎examples/idea-examples/titanic/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/titanic/ml/titanic.kt

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,19 +30,14 @@ private val model = Sequential.of(
3030

3131
fun main() {
3232

33-
// Set working directory for correct path resolution
34-
System.setProperty("user.dir", System.getProperty("user.dir") + "/examples/idea-examples/titanic")
35-
3633
// Set Locale for correct number parsing
3734
Locale.setDefault(Locale.FRANCE)
3835

39-
val df = Passenger.readCSV()
40-
41-
val pclass by column<Int>()
36+
// Set path for correct resolution (https://github.com/Kotlin/dataframe/issues/139)
37+
val df = Passenger.readCSV("examples/idea-examples/titanic/src/main/resources/titanic.csv")
4238

4339
// Calculating imputing values
4440
val (train, test) = df
45-
.rename { it.`pclass`}.into(pclass)
4641
// imputing
4742
.fillNulls { sibsp and parch and age and fare }.perCol { it.mean() }
4843
.fillNulls { sex }.withValue("female")

‎examples/idea-examples/titanic/src/main/resources/titanic.csv

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
pclass;survived;name;sex;age;sibsp;parch;ticket;fare;cabin;embarked;boat;body;homedest
1+
pclass;survived;name;sex;age;sibsp;parch;ticket;fare;cabin;embarked;boat;body;homedest
22
1;1;Allen, Miss. Elisabeth Walton;;29;;;24160;211,3375;B5;;2;;St Louis, MO
33
1;1;Allison, Master. Hudson Trevor;male;0,9167;1;2;113781;151,55;C22 C26;AA;11;;Montreal, PQ / Chesterville, ON
44
1;0;Allison, Miss. Helen Loraine;female;2;1;2;113781;151,55;C22 C26;S;;;Montreal, PQ / Chesterville, ON

‎examples/idea-examples/youtube/build.gradle.kts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
33
plugins {
44
application
55
kotlin("jvm")
6-
kotlin("plugin.dataframe")
6+
id("org.jetbrains.kotlinx.dataframe")
77
}
88

99
repositories {

‎examples/idea-examples/youtube/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/youtube/apiKey.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
package org.jetbrains.kotlinx.dataframe.examples.youtube
22

3-
const val apiKey = "<PUT YOUR API KEY HERE>"
3+
val apiKey: String = TODO("Insert your API key here")
44
const val basePath = "https://www.googleapis.com/youtube/v3"

‎examples/idea-examples/youtube/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/youtube/youtube.kt

Lines changed: 11 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,24 @@
11
@file:ImportDataSchema(
22
"SearchResponse",
3-
"$basePath/search?q=cats&part=snippet&key=$apiKey"
3+
"src/main/resources/searchResponse.json",
44
)
55
@file:ImportDataSchema(
66
"StatisticsResponse",
7-
"$basePath/videos?part=statistics&id=uHKfrz65KSU&key=$apiKey"
7+
"src/main/resources/statisticsResponse.json",
88
)
99

1010
package org.jetbrains.kotlinx.dataframe.examples.youtube
1111

12+
import kotlinx.datetime.Instant
1213
import org.jetbrains.kotlinx.dataframe.AnyFrame
1314
import org.jetbrains.kotlinx.dataframe.AnyRow
1415
import org.jetbrains.kotlinx.dataframe.DataRow
1516
import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema
16-
import org.jetbrains.kotlinx.dataframe.api.add
17-
import org.jetbrains.kotlinx.dataframe.api.asColumnGroup
18-
import org.jetbrains.kotlinx.dataframe.api.cast
19-
import org.jetbrains.kotlinx.dataframe.api.chunked
20-
import org.jetbrains.kotlinx.dataframe.api.column
21-
import org.jetbrains.kotlinx.dataframe.api.columnGroup
22-
import org.jetbrains.kotlinx.dataframe.api.concat
23-
import org.jetbrains.kotlinx.dataframe.api.convert
24-
import org.jetbrains.kotlinx.dataframe.api.dfsOf
25-
import org.jetbrains.kotlinx.dataframe.api.distinct
26-
import org.jetbrains.kotlinx.dataframe.api.dropNulls
27-
import org.jetbrains.kotlinx.dataframe.api.flatten
28-
import org.jetbrains.kotlinx.dataframe.api.getValueOrNull
29-
import org.jetbrains.kotlinx.dataframe.api.groupBy
30-
import org.jetbrains.kotlinx.dataframe.api.join
31-
import org.jetbrains.kotlinx.dataframe.api.map
32-
import org.jetbrains.kotlinx.dataframe.api.maxBy
33-
import org.jetbrains.kotlinx.dataframe.api.move
34-
import org.jetbrains.kotlinx.dataframe.api.parse
35-
import org.jetbrains.kotlinx.dataframe.api.remove
36-
import org.jetbrains.kotlinx.dataframe.api.select
37-
import org.jetbrains.kotlinx.dataframe.api.sortByDesc
38-
import org.jetbrains.kotlinx.dataframe.api.sum
39-
import org.jetbrains.kotlinx.dataframe.api.toTop
40-
import org.jetbrains.kotlinx.dataframe.api.under
41-
import org.jetbrains.kotlinx.dataframe.api.with
17+
import org.jetbrains.kotlinx.dataframe.api.*
4218
import org.jetbrains.kotlinx.dataframe.dataTypes.IFRAME
4319
import org.jetbrains.kotlinx.dataframe.dataTypes.IMG
4420
import org.jetbrains.kotlinx.dataframe.io.read
4521
import java.net.URL
46-
import kotlinx.datetime.Instant
47-
import org.jetbrains.kotlinx.dataframe.api.cumSum
48-
import org.jetbrains.kotlinx.dataframe.api.print
49-
import org.jetbrains.kotlinx.dataframe.api.sortBy
50-
import org.jetbrains.kotlinx.dataframe.api.sumFor
51-
import org.jetbrains.kotlinx.dataframe.api.toLong
52-
import org.jetbrains.kotlinx.dataframe.io.toHTML
53-
import java.awt.Desktop
54-
import java.io.File
5522

5623
fun load(path: String) = DataRow.read("$basePath/$path&key=$apiKey")
5724

@@ -62,14 +29,13 @@ fun load(path: String, maxPages: Int): AnyFrame {
6229
val row = load(pagePath)
6330
rows.add(row)
6431
val next = row.getValueOrNull<String>("nextPageToken")
65-
pagePath = path + "&pageToken=" + next
32+
pagePath = "$path&pageToken=$next"
6633

6734
} while (next != null && rows.size < maxPages)
6835
return rows.concat()
6936
}
7037

7138
fun main() {
72-
7339
val searchRequest = "cute%20cats"
7440
val resultsPerPage = 50
7541
val maxPages = 5
@@ -78,7 +44,10 @@ fun main() {
7844
val channel by columnGroup()
7945

8046
val videos = load("search?q=$searchRequest&maxResults=$resultsPerPage&part=snippet", maxPages)
81-
.cast<SearchResponse>()
47+
.convertTo<SearchResponse> {
48+
convert<String?>().with { it.toString() }
49+
convert<Int?>().with { it ?: 0 }
50+
}
8251
.items.concat()
8352
.dropNulls { id.videoId }
8453
.select { id.videoId into videoId and snippet }
@@ -113,13 +82,13 @@ fun main() {
11382
.sortByDesc { viewCount }
11483
.flatten()
11584

116-
channels.print()
85+
channels.print(borders = true, columnTypes = true)
11786

11887
val growth = withStat
11988
.select { publishTime and viewCount }
12089
.convert { publishTime and viewCount }.toLong()
12190
.sortBy { publishTime }
12291
.cumSum { viewCount }
12392

124-
growth.print()
93+
growth.print(borders = true, columnTypes = true)
12594
}

‎examples/idea-examples/youtube/src/main/resources/searchResponse.json

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
{
2+
"kind": "youtube#searchListResponse",
3+
"etag": "nl77cg-yrK-TW2q2RtoGXrdkkfo",
4+
"nextPageToken": "CAUQAA",
5+
"regionCode": "NL",
6+
"pageInfo": {
7+
"totalResults": 1000000,
8+
"resultsPerPage": 5
9+
},
10+
"items": [
11+
{
12+
"kind": "youtube#searchResult",
13+
"etag": "gsRtDXx5RZlp-qILhP65o2oF-go",
14+
"id": {
15+
"kind": "youtube#video",
16+
"videoId": "Dix58mO0Pbc"
17+
},
18+
"snippet": {
19+
"publishedAt": "2022-08-10T14:30:04Z",
20+
"channelId": "UC7wafFu5c8AO0YF5U7R7xFA",
21+
"title": "Cat TV for Cats to Watch 😺 Summer birds and ducks by the lake 🐦 Cute squirrels 🐿 8 Hours(4K HDR)",
22+
"description": "8 hours of pleasing video for cats, dogs, parrots, or other nature lovers to enjoy. It can relax your kitten or puppy and minimize ...",
23+
"thumbnails": {
24+
"default": {
25+
"url": "https://i.ytimg.com/vi/Dix58mO0Pbc/default.jpg",
26+
"width": 120,
27+
"height": 90
28+
},
29+
"medium": {
30+
"url": "https://i.ytimg.com/vi/Dix58mO0Pbc/mqdefault.jpg",
31+
"width": 320,
32+
"height": 180
33+
},
34+
"high": {
35+
"url": "https://i.ytimg.com/vi/Dix58mO0Pbc/hqdefault.jpg",
36+
"width": 480,
37+
"height": 360
38+
}
39+
},
40+
"channelTitle": "Birder King",
41+
"liveBroadcastContent": "none",
42+
"publishTime": "2022-08-10T14:30:04Z"
43+
}
44+
},
45+
{
46+
"kind": "youtube#searchResult",
47+
"etag": "_7QEwCZHKtgnPTcYmsxNaol-I0Q",
48+
"id": {
49+
"kind": "youtube#video",
50+
"videoId": "bGsN7jzp5DE"
51+
},
52+
"snippet": {
53+
"publishedAt": "2022-08-09T17:00:30Z",
54+
"channelId": "UCINb0wqPz-A0dV9nARjJlOQ",
55+
"title": "Cat Is Obsessed With His Tiny Love Bird | The Dodo Odd Couples",
56+
"description": "This cat is glued to his favorite little love bird and even climbs inside her cage to hang out longer This video is dedicated to ...",
57+
"thumbnails": {
58+
"default": {
59+
"url": "https://i.ytimg.com/vi/bGsN7jzp5DE/default.jpg",
60+
"width": 120,
61+
"height": 90
62+
},
63+
"medium": {
64+
"url": "https://i.ytimg.com/vi/bGsN7jzp5DE/mqdefault.jpg",
65+
"width": 320,
66+
"height": 180
67+
},
68+
"high": {
69+
"url": "https://i.ytimg.com/vi/bGsN7jzp5DE/hqdefault.jpg",
70+
"width": 480,
71+
"height": 360
72+
}
73+
},
74+
"channelTitle": "The Dodo",
75+
"liveBroadcastContent": "none",
76+
"publishTime": "2022-08-09T17:00:30Z"
77+
}
78+
},
79+
{
80+
"kind": "youtube#searchResult",
81+
"etag": "IHNyBgppiApI3KGzkUV5AuPMftM",
82+
"id": {
83+
"kind": "youtube#video",
84+
"videoId": "U1OxDRxNEMM"
85+
},
86+
"snippet": {
87+
"publishedAt": "2022-08-10T14:45:00Z",
88+
"channelId": "UCcnThqTwvub5ykbII9WkR5g",
89+
"title": "Funny animals - Funny cats / dogs - Funny animal videos 218",
90+
"description": "Funny animals! Compilation number 218. Only the best! Sit back and charge positively Funny animal videos (funny cats, dogs ...",
91+
"thumbnails": {
92+
"default": {
93+
"url": "https://i.ytimg.com/vi/U1OxDRxNEMM/default.jpg",
94+
"width": 120,
95+
"height": 90
96+
},
97+
"medium": {
98+
"url": "https://i.ytimg.com/vi/U1OxDRxNEMM/mqdefault.jpg",
99+
"width": 320,
100+
"height": 180
101+
},
102+
"high": {
103+
"url": "https://i.ytimg.com/vi/U1OxDRxNEMM/hqdefault.jpg",
104+
"width": 480,
105+
"height": 360
106+
}
107+
},
108+
"channelTitle": "Happy Dog",
109+
"liveBroadcastContent": "none",
110+
"publishTime": "2022-08-10T14:45:00Z"
111+
}
112+
},
113+
{
114+
"kind": "youtube#searchResult",
115+
"etag": "0OYjMrtwAzJH7yo_jOPvF2hwSao",
116+
"id": {
117+
"kind": "youtube#video",
118+
"videoId": "ByH9LuSILxU"
119+
},
120+
"snippet": {
121+
"publishedAt": "2020-06-19T02:18:53Z",
122+
"channelId": "UC8hC-augAnujJeprhjI0YkA",
123+
"title": "Baby Cats - Cute and Funny Cat Videos Compilation #34 | Aww Animals",
124+
"description": "Baby cats are amazing creature because they are the cutest and most funny. Watching funny baby cats is the hardest try not to ...",
125+
"thumbnails": {
126+
"default": {
127+
"url": "https://i.ytimg.com/vi/ByH9LuSILxU/default.jpg",
128+
"width": 120,
129+
"height": 90
130+
},
131+
"medium": {
132+
"url": "https://i.ytimg.com/vi/ByH9LuSILxU/mqdefault.jpg",
133+
"width": 320,
134+
"height": 180
135+
},
136+
"high": {
137+
"url": "https://i.ytimg.com/vi/ByH9LuSILxU/hqdefault.jpg",
138+
"width": 480,
139+
"height": 360
140+
}
141+
},
142+
"channelTitle": "Aww Animals",
143+
"liveBroadcastContent": "none",
144+
"publishTime": "2020-06-19T02:18:53Z"
145+
}
146+
},
147+
{
148+
"kind": "youtube#searchResult",
149+
"etag": "S1UukOVi_sofJQLHSU0jX5GSv2M",
150+
"id": {
151+
"kind": "youtube#video",
152+
"videoId": "VkqVsCPAIag"
153+
},
154+
"snippet": {
155+
"publishedAt": "2022-08-10T11:03:15Z",
156+
"channelId": "UCHBnS9TR-4h2nvuiiq3XCAA",
157+
"title": "Awesome SO Cute Cat ! Cute and Funny Cat Videos to Keep You Smiling! 🐱",
158+
"description": "The featured clips in our video are used with permission from the original video owners. The highlight clips can be done by our ...",
159+
"thumbnails": {
160+
"default": {
161+
"url": "https://i.ytimg.com/vi/VkqVsCPAIag/default.jpg",
162+
"width": 120,
163+
"height": 90
164+
},
165+
"medium": {
166+
"url": "https://i.ytimg.com/vi/VkqVsCPAIag/mqdefault.jpg",
167+
"width": 320,
168+
"height": 180
169+
},
170+
"high": {
171+
"url": "https://i.ytimg.com/vi/VkqVsCPAIag/hqdefault.jpg",
172+
"width": 480,
173+
"height": 360
174+
}
175+
},
176+
"channelTitle": "Best awesome",
177+
"liveBroadcastContent": "none",
178+
"publishTime": "2022-08-10T11:03:15Z"
179+
}
180+
}
181+
]
182+
}

‎examples/idea-examples/youtube/src/main/resources/statisticsResponse.json

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"kind": "youtube#videoListResponse",
3+
"etag": "rHk7psLWXLIjjx8rGeiNKUxrD-s",
4+
"items": [
5+
{
6+
"kind": "youtube#video",
7+
"etag": "hiKGiry1Gc19FmHigb3sMfjnzP8",
8+
"id": "uHKfrz65KSU",
9+
"statistics": {
10+
"viewCount": "67715094",
11+
"likeCount": "641192",
12+
"favoriteCount": "0",
13+
"commentCount": "22174"
14+
}
15+
}
16+
],
17+
"pageInfo": {
18+
"totalResults": 1,
19+
"resultsPerPage": 1
20+
}
21+
}

‎settings.gradle.kts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@ include("dataframe-arrow")
1111

1212
include("examples:idea-examples:titanic")
1313
include("examples:idea-examples:movies")
14-
// TODO: replace url in ImportDataSchema with a sample of response from youtube api
15-
// include("examples:idea-examples:youtube")
14+
include("examples:idea-examples:youtube")
1615

1716
val jupyterApiTCRepo: String by settings
1817

0 commit comments

Comments
 (0)