Skip to content
Open
Show file tree
Hide file tree
Changes from 37 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
6d10538
update to Gradle 7.5
aSemy Jul 23, 2022
329e08d
merge .gitignore
aSemy Jul 23, 2022
64cd016
minor readme fixes
aSemy Jul 23, 2022
ae0ffb2
set up buildSrc and convention plugins (with config mostly copied fro…
aSemy Jul 23, 2022
3ad80da
update settings plugin to use shared repo config, and update Gradle p…
aSemy Jul 23, 2022
04928e2
enable Gradle caching options (faster IDE, build, tests speeds)
aSemy Jul 23, 2022
240e39d
update projects to use convention plugins, and remove custom dir sett…
aSemy Jul 23, 2022
3cb0751
set snapshot repo as 'snapshots only' (better performance)
aSemy Jul 23, 2022
c0e9329
buildsrc - format plugins
aSemy Jul 23, 2022
e54e338
fix explicitApi, and add task-config avoidance on test tasks
aSemy Jul 23, 2022
cd9b95b
gradle does NOT like parallel! Lots of OOM exceptions
aSemy Jul 23, 2022
9bc840c
enable detekt for Kotlin projects
aSemy Jul 23, 2022
df43ed3
move snapshot repo to common repo definitions
aSemy Jul 23, 2022
0fdab96
apply version-management.gradle.kts plugin, update some log dependencies
aSemy Jul 23, 2022
e47e882
fix Maven publishing for JVM
aSemy Jul 24, 2022
6bda350
add dokka to Javadoc Jar
aSemy Jul 24, 2022
1c0de97
rm jsExecution feature TODO (it's Gradle config to create a feature v…
aSemy Jul 24, 2022
0babfef
update project dependencies to use typesafe-accessors
aSemy Jul 24, 2022
c88edbc
rm idea plugin from buildSrc/build.gradle.kts
aSemy Jul 24, 2022
edc8596
initial migration to Kotlin Multiplatform
aSemy Jul 24, 2022
4c6dca6
bump Kotlin, Ktor
aSemy Jul 24, 2022
cdc7fd6
set up test-utils
aSemy Jul 24, 2022
d33e4d5
Merge remote-tracking branch 'origin/master' into feat/kotlin-multipl…
aSemy Jul 26, 2022
9ab5806
bump versions again (overwritten during merge)
aSemy Jul 26, 2022
8abdfc6
update Gradle caching for GitHub actions
aSemy Jul 26, 2022
6b2803b
add concurrency, timeout, and disable fail-fast for tests
aSemy Jul 26, 2022
14a2c33
Merge branch 'github-actions-update' into feat/kotlin-multiplatform
aSemy Jul 26, 2022
7f674fa
Merge branch 'github-actions-update' into feat/kotlin-multiplatform
aSemy Jul 26, 2022
6c3bb86
Merge remote-tracking branch 'origin/master' into feat/kotlin-multipl…
aSemy Aug 26, 2022
4854cbe
bump gradle
aSemy Aug 26, 2022
685cb89
add jvm test dependencies
aSemy Aug 26, 2022
53f5518
toLowerCase() -> lowercase()
aSemy Aug 26, 2022
2b18c73
spelling scoope -> scope
aSemy Aug 26, 2022
0a64903
fix code warnings
aSemy Aug 26, 2022
b429225
bump kover plugin version
aSemy Aug 26, 2022
6bc29c3
try bumping kover version
aSemy Aug 26, 2022
444aeaa
revert kover, try to fix the other bug by manually setting the engine…
aSemy Aug 26, 2022
04c9d48
fix Kover coverage
aSemy Sep 7, 2022
ef935bd
fix Kover coverage
aSemy Sep 7, 2022
09de5c7
remove jsExecution feature variant, publish browser-fetcher directly
aSemy Sep 7, 2022
a104400
configure kotlin-multiplatform publishing
aSemy Sep 7, 2022
f393237
all -> configureEach
aSemy Sep 7, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions assertions/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,10 +1,27 @@
plugins {
buildsrc.convention.`kotlin-jvm`
buildsrc.convention.`publish-jvm`
buildsrc.convention.`kotlin-multiplatform`
buildsrc.convention.`publish-kmp`
}

dependencies {
api(projects.htmlParser)
api(projects.dsl)
api(projects.fetcher.baseFetcher)
kotlin {
jvm {}

sourceSets {
val commonMain by getting {
dependencies {
api(projects.htmlParser)
api(projects.dsl)
api(projects.fetcher.baseFetcher)
}
}

val jvmTest by getting {
dependencies {
implementation(Deps.jUnit)
implementation(Deps.strikt)
implementation(Deps.Mockk.mockk)
implementation(Deps.Mockk.dslJvm)
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,5 @@ public infix fun ContentType.toContain(expected: ContentTypes): ContentType /* =
this.apply { generalAssertion(raw().contains(expected.value), expected) }

private fun ContentType.raw() = (this as String)
.toLowerCase()
.lowercase()
.replace("\\s".toRegex(), "")
33 changes: 20 additions & 13 deletions build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
plugins {
buildsrc.convention.base
buildsrc.convention.`kotlin-jvm`
buildsrc.convention.`publish-jvm`
buildsrc.convention.`kotlin-multiplatform`
buildsrc.convention.`publish-kmp`

id("org.jetbrains.kotlinx.kover")

Expand All @@ -13,22 +12,30 @@ val release_version: String by project
version = release_version
group = "it.skrape"

dependencies {
api(projects.assertions)
api(projects.fetcher.asyncFetcher)
api(projects.fetcher.baseFetcher)
api(projects.fetcher.browserFetcher)
api(projects.dsl)
api(projects.fetcher.httpFetcher)
api(projects.htmlParser)
kotlin {
jvm {}

sourceSets {
val commonMain by getting {
dependencies {
api(projects.assertions)
api(projects.fetcher.asyncFetcher)
api(projects.fetcher.baseFetcher)
api(projects.fetcher.browserFetcher)
api(projects.dsl)
api(projects.fetcher.httpFetcher)
api(projects.htmlParser)
}
}
}
}

tasks.withType<Test>().configureEach {
finalizedBy(tasks.koverReport, tasks.koverCollectReports)
}

kover {
coverageEngine.set(kotlinx.kover.api.CoverageEngine.INTELLIJ)
intellijEngineVersion.set("1.0.680")
}

nexusPublishing {
Expand All @@ -38,6 +45,6 @@ nexusPublishing {
}

tasks.wrapper {
gradleVersion = "7.5"
gradleVersion = "7.5.1"
distributionType = Wrapper.DistributionType.ALL
}
6 changes: 3 additions & 3 deletions buildSrc/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ plugins {
}

// set the versions of Gradle plugins that the subprojects will use here
val kotlinVersion = "1.6.21"
val kotlinVersion = "1.7.10"

val detektPlugin = "1.19.0"
val gradleNexusPublishPlugin = "1.1.0"
val gradleTestLoggerPlugin = "3.1.0"
val gradleVersionsPlugin = "0.39.0"
val kotlinDokkaPlugin = "1.6.21"
val kotlinxKoverPlugin = "0.5.0"
val kotlinDokkaPlugin = "1.7.10"
val kotlinxKoverPlugin = "0.5.1"
val useLatestVersionsPlugin = "0.2.18"

dependencies {
Expand Down
4 changes: 2 additions & 2 deletions buildSrc/src/main/kotlin/Deps.kt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
object Versions {
const val kotlin = "1.6.21"
const val kotlin = "1.7.10"
const val coroutines = "1.6.1"
const val ktor = "2.0.3" //2.0.1
const val ktor = "2.0.3"
const val serialization = "1.0.1"
const val datetime = "0.1.1"
const val jsoup = "1.13.1" // 1.14.3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ dependencies {
kotlin {
explicitApi()
jvmToolchain {
(this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of("8"))
languageVersion.set(JavaLanguageVersion.of("8"))
}
}

Expand All @@ -40,8 +40,8 @@ tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.apply {
jvmTarget = "1.8"
freeCompilerArgs += listOf("-Xjsr305=strict")
apiVersion = "1.4"
languageVersion = "1.4"
apiVersion = "1.5"
languageVersion = "1.7"
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,16 @@ plugins {
id("buildsrc.convention.base")
id("buildsrc.convention.detekt")
}

kotlin {
explicitApi()

targets.all {
compilations.all {
kotlinOptions {
apiVersion = "1.5"
languageVersion = "1.7"
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package buildsrc.convention


plugins {
}

// placeholder
7 changes: 5 additions & 2 deletions dsl/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
plugins {
buildsrc.convention.`kotlin-jvm`
buildsrc.convention.`publish-jvm`
buildsrc.convention.`kotlin-multiplatform`
}

kotlin {
jvm {}
}
37 changes: 27 additions & 10 deletions fetcher/async-fetcher/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,15 +1,32 @@
plugins {
buildsrc.convention.`kotlin-jvm`
buildsrc.convention.`publish-jvm`
buildsrc.convention.`kotlin-multiplatform`
buildsrc.convention.`publish-kmp`
}

dependencies {
implementation(projects.fetcher.baseFetcher)
implementation(Deps.Ktor.client)
implementation(Deps.Ktor.clientApache)
implementation(Deps.Ktor.clientLogging)
implementation(Deps.logback)
implementation(Deps.log4jOverSlf4j)
kotlin {
jvm {}

testImplementation(projects.testUtils)
sourceSets {
val commonMain by getting {
dependencies {
implementation(projects.fetcher.baseFetcher)
implementation(Deps.Ktor.client)
implementation(Deps.Ktor.clientApache)
implementation(Deps.Ktor.clientLogging)
}
}

val jvmMain by getting {
dependencies {
implementation(Deps.logback)
implementation(Deps.log4jOverSlf4j)
}
}

val jvmTest by getting {
dependencies {
implementation(projects.testUtils)
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ internal fun String?.toExpires(): Expires {
}
}

internal fun String?.toSameSite(): SameSite = when (this?.toLowerCase()) {
internal fun String?.toSameSite(): SameSite = when (this?.lowercase()) {
"strict" -> SameSite.STRICT
"lax" -> SameSite.LAX
"none" -> SameSite.NONE
Expand Down
46 changes: 31 additions & 15 deletions fetcher/base-fetcher/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,20 +1,36 @@
@file:Suppress("PropertyName")

val kotlin_version: String by project

plugins {
buildsrc.convention.`kotlin-jvm`
buildsrc.convention.`publish-jvm`
buildsrc.convention.`kotlin-multiplatform`
buildsrc.convention.`publish-kmp`
}

dependencies {
implementation(projects.dsl)
implementation(Deps.Kotlin.reflect) {
because("to support Result#extractIt by creating instance of a class")
}
api(Deps.KotlinX.Coroutines.jdk8)
kotlin {
jvm {}
sourceSets {
val commonMain by getting {
dependencies {
implementation(projects.dsl)
api(Deps.KotlinX.Coroutines.core)
}
}

val jvmMain by getting {
dependencies {
api(Deps.jsoup)
implementation(Deps.Kotlin.reflect) {
because("to support Result#extractIt by creating instance of a class")
}
api(Deps.KotlinX.Coroutines.jdk8)
}
}

testImplementation(projects.testUtils)
testImplementation(Deps.Ktor.client)
testImplementation(Deps.Ktor.clientApache)
val jvmTest by getting {

dependencies {
implementation(projects.testUtils)

implementation(Deps.Ktor.client)
implementation(Deps.Ktor.clientApache)
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -54,16 +54,16 @@ public fun String.toCookie(origin: String): Cookie {
else -> Domain(domainUrl, true)
}
val sameSite = attributes.getAttribute("samesite").toSameSite()
val secure = attributes.any { it.toLowerCase() == "secure" }
val httpOnly = attributes.any { it.toLowerCase() == "httponly" }
val secure = attributes.any { it.lowercase() == "secure" }
val httpOnly = attributes.any { it.lowercase() == "httponly" }
return Cookie(name, value, expires, maxAge, domain, path, sameSite, secure, httpOnly)
}

private fun List<String>.getAttribute(attributeName: String) =
this.find { it.startsWith("${attributeName}=", ignoreCase = true) }?.takeLastWhile { it != '=' }

private fun String?.toSameSite(): SameSite {
return when (this?.toLowerCase()) {
return when (this?.lowercase()) {
"strict" -> SameSite.STRICT
"lax" -> SameSite.LAX
"none" -> SameSite.NONE
Expand Down
40 changes: 29 additions & 11 deletions fetcher/browser-fetcher/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,16 +1,34 @@
plugins {
buildsrc.convention.`kotlin-jvm`
buildsrc.convention.`kotlin-multiplatform`
}

dependencies {
api(projects.fetcher.baseFetcher)
api(Deps.htmlUnit) {
exclude("org.eclipse.jetty.websocket") // avoid android crash; see #93
}
val log4jOverSlf4jVersion = "1.7.36"
val logbackVersion = "1.2.11"
api("ch.qos.logback:logback-classic:$logbackVersion")
api("org.slf4j:log4j-over-slf4j:$log4jOverSlf4jVersion")
kotlin {
jvm {}
sourceSets {

val commonMain by getting {
dependencies {
api(projects.fetcher.baseFetcher)
}
}

val jvmMain by getting {
dependencies {
api(Deps.htmlUnit) {
exclude("org.eclipse.jetty.websocket") // avoid android crash; see #93
}

testImplementation(projects.testUtils)
val log4jOverSlf4jVersion = "1.7.36"
val logbackVersion = "1.2.11"
api("ch.qos.logback:logback-classic:$logbackVersion")
api("org.slf4j:log4j-over-slf4j:$log4jOverSlf4jVersion")
}
}

val jvmTest by getting {
dependencies {
implementation(projects.testUtils)
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package it.skrape.fetcher

public expect object BrowserFetcher : BlockingFetcher<Request> {

public fun render(html: String): String

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import com.gargoylesoftware.htmlunit.util.NameValuePair
import java.net.Proxy
import java.net.URL

public object BrowserFetcher : BlockingFetcher<Request> {
public actual object BrowserFetcher : BlockingFetcher<Request> {
override val requestBuilder: Request get() = Request()

override fun fetch(request: Request): Result {
Expand Down Expand Up @@ -41,7 +41,7 @@ public object BrowserFetcher : BlockingFetcher<Request> {
return result
}

public fun render(html: String): String =
public actual fun render(html: String): String =
WebClient(BrowserVersion.BEST_SUPPORTED).loadHtmlCodeIntoCurrentWindow(html).asXml()

private fun WebClient.withOptions(request: Request) = apply {
Expand Down
Loading