Skip to content

Commit a87b440

Browse files
JolanrensenJolanrensen
authored
Merge pull request #191 from Kotlin/path-fixes
Absolute path fixes in plugins
2 parents bd120da + df54aa1 commit a87b440

File tree

4 files changed

+86
-35
lines changed

4 files changed

+86
-35
lines changed

‎examples/idea-examples/movies/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/movies/moviesWithInterface.kt

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,23 @@ package org.jetbrains.kotlinx.dataframe.examples.movies
22

33
import org.jetbrains.kotlinx.dataframe.DataFrame
44
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
5-
import org.jetbrains.kotlinx.dataframe.api.*
5+
import org.jetbrains.kotlinx.dataframe.api.by
6+
import org.jetbrains.kotlinx.dataframe.api.convertTo
7+
import org.jetbrains.kotlinx.dataframe.api.count
8+
import org.jetbrains.kotlinx.dataframe.api.explode
9+
import org.jetbrains.kotlinx.dataframe.api.filter
10+
import org.jetbrains.kotlinx.dataframe.api.groupBy
11+
import org.jetbrains.kotlinx.dataframe.api.inplace
12+
import org.jetbrains.kotlinx.dataframe.api.into
13+
import org.jetbrains.kotlinx.dataframe.api.mean
14+
import org.jetbrains.kotlinx.dataframe.api.pivot
15+
import org.jetbrains.kotlinx.dataframe.api.print
16+
import org.jetbrains.kotlinx.dataframe.api.sortBy
17+
import org.jetbrains.kotlinx.dataframe.api.split
618
import org.jetbrains.kotlinx.dataframe.io.read
719

820
@DataSchema
9-
interface Movie{
21+
interface Movie {
1022
val movieId: String
1123
val title: String
1224
val genres: String

‎examples/idea-examples/titanic/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/titanic/ml/titanic.kt

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,24 @@ package org.jetbrains.kotlinx.dataframe.examples.titanic.ml
22

33
import org.jetbrains.kotlinx.dataframe.ColumnSelector
44
import org.jetbrains.kotlinx.dataframe.DataFrame
5-
import org.jetbrains.kotlinx.dataframe.api.*
5+
import org.jetbrains.kotlinx.dataframe.api.by
66
import org.jetbrains.kotlinx.dataframe.api.column
7+
import org.jetbrains.kotlinx.dataframe.api.convert
8+
import org.jetbrains.kotlinx.dataframe.api.dfsOf
9+
import org.jetbrains.kotlinx.dataframe.api.fillNulls
10+
import org.jetbrains.kotlinx.dataframe.api.getColumn
11+
import org.jetbrains.kotlinx.dataframe.api.into
12+
import org.jetbrains.kotlinx.dataframe.api.mean
13+
import org.jetbrains.kotlinx.dataframe.api.merge
14+
import org.jetbrains.kotlinx.dataframe.api.perCol
15+
import org.jetbrains.kotlinx.dataframe.api.pivotMatches
16+
import org.jetbrains.kotlinx.dataframe.api.remove
17+
import org.jetbrains.kotlinx.dataframe.api.select
18+
import org.jetbrains.kotlinx.dataframe.api.shuffle
19+
import org.jetbrains.kotlinx.dataframe.api.toFloat
20+
import org.jetbrains.kotlinx.dataframe.api.toFloatArray
21+
import org.jetbrains.kotlinx.dataframe.api.toTypedArray
22+
import org.jetbrains.kotlinx.dataframe.api.withValue
723
import org.jetbrains.kotlinx.dl.api.core.Sequential
824
import org.jetbrains.kotlinx.dl.api.core.activation.Activations
925
import org.jetbrains.kotlinx.dl.api.core.initializer.HeNormal
@@ -14,7 +30,7 @@ import org.jetbrains.kotlinx.dl.api.core.loss.Losses
1430
import org.jetbrains.kotlinx.dl.api.core.metric.Metrics
1531
import org.jetbrains.kotlinx.dl.api.core.optimizer.Adam
1632
import org.jetbrains.kotlinx.dl.dataset.OnHeapDataset
17-
import java.util.*
33+
import java.util.Locale
1834

1935
private const val SEED = 12L
2036
private const val TEST_BATCH_SIZE = 100
@@ -33,8 +49,7 @@ fun main() {
3349
// Set Locale for correct number parsing
3450
Locale.setDefault(Locale.FRANCE)
3551

36-
// Set path for correct resolution (https://github.com/Kotlin/dataframe/issues/139)
37-
val df = Passenger.readCSV("examples/idea-examples/titanic/src/main/resources/titanic.csv")
52+
val df = Passenger.readCSV()
3853

3954
// Calculating imputing values
4055
val (train, test) = df
@@ -44,7 +59,7 @@ fun main() {
4459
// one hot encoding
4560
.pivotMatches { pclass and sex }
4661
// feature extraction
47-
.select { survived and pclass and sibsp and parch and age and fare and sex}
62+
.select { survived and pclass and sibsp and parch and age and fare and sex }
4863
.shuffle()
4964
.toTrainTest(0.7) { survived }
5065

@@ -64,18 +79,23 @@ fun main() {
6479
}
6580
}
6681

67-
fun <T> DataFrame<T>.toTrainTest(trainRatio: Double, yColumn: ColumnSelector<T, Number>): Pair<OnHeapDataset, OnHeapDataset> =
68-
toOnHeapDataset(yColumn).split(trainRatio)
82+
fun <T> DataFrame<T>.toTrainTest(
83+
trainRatio: Double,
84+
yColumn: ColumnSelector<T, Number>,
85+
): Pair<OnHeapDataset, OnHeapDataset> =
86+
toOnHeapDataset(yColumn)
87+
.split(trainRatio)
6988

70-
private fun <T> DataFrame<T>.toOnHeapDataset(yColumn: ColumnSelector<T, Number>): OnHeapDataset {
71-
return OnHeapDataset.create(
89+
private fun <T> DataFrame<T>.toOnHeapDataset(yColumn: ColumnSelector<T, Number>): OnHeapDataset =
90+
OnHeapDataset.create(
7291
dataframe = this,
73-
yColumn = yColumn
92+
yColumn = yColumn,
7493
)
75-
}
76-
77-
private fun <T> OnHeapDataset.Companion.create(dataframe: DataFrame<T>, yColumn: ColumnSelector<T, Number>): OnHeapDataset {
7894

95+
private fun <T> OnHeapDataset.Companion.create(
96+
dataframe: DataFrame<T>,
97+
yColumn: ColumnSelector<T, Number>,
98+
): OnHeapDataset {
7999
val x by column<FloatArray>("X")
80100

81101
fun extractX(): Array<FloatArray> =
@@ -88,6 +108,6 @@ private fun <T> OnHeapDataset.Companion.create(dataframe: DataFrame<T>, yColumn:
88108

89109
return create(
90110
::extractX,
91-
::extractY
111+
::extractY,
92112
)
93113
}

‎plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import org.jetbrains.kotlinx.dataframe.io.Excel
2222
import org.jetbrains.kotlinx.dataframe.io.JSON
2323
import org.jetbrains.kotlinx.dataframe.io.OpenApi
2424
import org.jetbrains.kotlinx.dataframe.io.TSV
25+
import org.jetbrains.kotlinx.dataframe.io.isURL
2526
import java.io.File
2627
import java.net.URL
2728
import java.nio.file.Paths
@@ -128,14 +129,22 @@ abstract class GenerateDataSchemaTask : DefaultTask() {
128129
schemaFile.writeText(codeGenResult.toStandaloneSnippet(escapedPackageName, readDfMethod.additionalImports))
129130
}
130131

131-
private fun stringOf(data: Any): String {
132-
return when (data) {
133-
is File -> data.toRelativeString(base = project.projectDir)
132+
private fun stringOf(data: Any): String =
133+
when (data) {
134+
is File -> data.absolutePath
134135
is URL -> data.toExternalForm()
135-
is String -> data
136+
is String ->
137+
when {
138+
isURL(data) -> stringOf(URL(data))
139+
else -> {
140+
val relativeFile = project.file(data)
141+
val absoluteFile = File(data)
142+
stringOf(if (relativeFile.exists()) relativeFile else absoluteFile)
143+
}
144+
}
145+
136146
else -> unsupportedType()
137147
}
138-
}
139148

140149
private fun escapePackageName(packageName: String): String {
141150
// See RegexExpectationsTest
@@ -147,20 +156,26 @@ abstract class GenerateDataSchemaTask : DefaultTask() {
147156
}
148157
}
149158

150-
private fun urlOf(data: Any): URL {
151-
fun isURL(fileOrUrl: String): Boolean = listOf("http:", "https:", "ftp:").any { fileOrUrl.startsWith(it) }
152-
153-
return when (data) {
159+
private fun urlOf(data: Any): URL =
160+
when (data) {
154161
is File -> data.toURI()
155162
is URL -> data.toURI()
156163
is String -> when {
157164
isURL(data) -> URL(data).toURI()
158-
else -> project.file(data).toURI()
165+
else -> {
166+
val relativeFile = project.file(data)
167+
val absoluteFile = File(data)
168+
169+
if (relativeFile.exists()) {
170+
relativeFile
171+
} else {
172+
absoluteFile
173+
}.toURI()
174+
}
159175
}
160176

161177
else -> unsupportedType()
162178
}.toURL()
163-
}
164179

165180
private fun unsupportedType(): Nothing =
166181
throw IllegalArgumentException("data for schema \"${interfaceName.get()}\" must be File, URL or String")

‎plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/DataSchemaGenerator.kt

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import org.jetbrains.kotlinx.dataframe.io.Excel
2626
import org.jetbrains.kotlinx.dataframe.io.JSON
2727
import org.jetbrains.kotlinx.dataframe.io.OpenApi
2828
import org.jetbrains.kotlinx.dataframe.io.TSV
29+
import org.jetbrains.kotlinx.dataframe.io.isURL
2930
import java.io.File
3031
import java.net.MalformedURLException
3132
import java.net.URL
@@ -63,30 +64,33 @@ class DataSchemaGenerator(
6364
}
6465

6566
private fun ImportDataSchema.toStatement(file: KSFile, logger: KSPLogger): ImportDataSchemaStatement? {
66-
val protocols = listOf("http", "https", "ftp")
67-
val url = if (protocols.any { path.startsWith(it, ignoreCase = true) }) {
67+
val url = if (isURL(path)) {
6868
try {
6969
URL(this.path)
7070
} catch (exception: MalformedURLException) {
7171
logger.error("'${this.path}' is not valid URL: ${exception.message}", file)
72-
null
72+
return null
7373
}
7474
} else {
7575
val resolutionDir: String = resolutionDir ?: run {
7676
reportMissingKspArgument(file)
7777
return null
7878
}
79-
val data = File(resolutionDir, path)
79+
80+
val relativeFile = File(resolutionDir, path)
81+
val absoluteFile = File(path)
82+
val data = if (relativeFile.exists()) relativeFile else absoluteFile
8083
try {
81-
data.toURI().toURL()
84+
data.toURI().toURL() ?: return null
8285
} catch (exception: MalformedURLException) {
8386
logger.error(
84-
"Failed to convert resolved path '${data.absolutePath}' to URL: ${exception.message}",
87+
"Failed to convert resolved path '${relativeFile.absolutePath}' or '${absoluteFile.absolutePath}' to URL: ${exception.message}",
8588
file
8689
)
87-
null
90+
return null
8891
}
89-
} ?: return null
92+
}
93+
9094
return ImportDataSchemaStatement(
9195
origin = file,
9296
name = name,

0 commit comments

Comments
 (0)