Skip to content

#6 unused headerFields & remove logic to unshortUrl #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
naushad-madakiya wants to merge 2 commits into
base: master
Choose a base branch
from
Open

#6 unused headerFields & remove logic to unshortUrl #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3702473
fix: remove unused headerFiled & unshort url related code
Aug 10, 2019
6e314d3
ui: update sample with more example
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions app/src/main/java/net/vrgsoft/rxurlparser/MainActivity.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ class MainActivity : AppCompatActivity() {
private lateinit var mBinding: ActivityMainBinding
private val crawler = LinkCrawler()

private var parseSubscription: Disposable? = null
private var githubSubscription: Disposable? = null
private var youtubeSubscription: Disposable? = null

override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
Expand All @@ -24,15 +25,22 @@ class MainActivity : AppCompatActivity() {
.show()
}

parseSubscription =
githubSubscription =
crawler.parseUrl("https://github.com")
.subscribe { t ->
mBinding.content = t.result
mBinding.contentGithub = t.result
}

youtubeSubscription =
crawler.parseUrl("https://youtu.be/X1RVYt2QKQE")
.subscribe { t ->
mBinding.contentYoutube = t.result
}
}

override fun onPause() {
parseSubscription?.dispose()
githubSubscription?.dispose()
youtubeSubscription?.dispose()
super.onPause()
}
}
64 changes: 52 additions & 12 deletions app/src/main/res/layout/activity_main.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@
<data>

<variable
name="content"
name="content_github"
type="net.vrgsoft.library.ParseContent" />

<variable
name="content_youtube"
type="net.vrgsoft.library.ParseContent" />
</data>

Expand All @@ -16,45 +20,81 @@
tools:context="net.vrgsoft.rxurlparser.MainActivity">

<TextView
android:id="@+id/title"
android:id="@+id/github_title"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="@dimen/small_margin"
android:layout_marginRight="@dimen/small_margin"
android:layout_marginTop="8dp"
android:text="@{content.title}"
android:text="@{content_github.title}"
android:textAlignment="center"
android:textStyle="bold"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintRight_toRightOf="parent"
app:layout_constraintTop_toTopOf="parent"
tools:text="Title" />

<TextView
android:id="@+id/description"
android:id="@+id/github_description"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="@dimen/small_margin"
android:layout_marginRight="@dimen/small_margin"
android:layout_marginTop="@dimen/small_margin"
android:text="@{content.description}"
android:text="@{content_github.description}"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/github_title"
tools:text="Description" />

<TextView
android:id="@+id/github_url"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="@dimen/small_margin"
android:layout_marginTop="@dimen/small_margin"
android:layout_marginRight="@dimen/small_margin"
android:text="@{content_github.finalUrl}"
android:textAlignment="center"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/github_description"
tools:text="www.vrgsoft.net" />

<TextView
android:id="@+id/youtube_title"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="@dimen/small_margin"
android:layout_marginTop="16dp"
android:layout_marginRight="@dimen/small_margin"
android:text="@{content_youtube.title}"
android:textAlignment="center"
android:textStyle="bold"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/github_url"
tools:text="Title" />

<TextView
android:id="@+id/youtube_description"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="@dimen/small_margin"
android:layout_marginTop="@dimen/small_margin"
android:layout_marginRight="@dimen/small_margin"
android:text="@{content_youtube.description}"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintRight_toRightOf="parent"
app:layout_constraintTop_toBottomOf="@+id/title"
app:layout_constraintTop_toBottomOf="@+id/youtube_title"
tools:text="Description" />

<TextView
android:id="@+id/url"
android:id="@+id/youtube_url"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginLeft="@dimen/small_margin"
android:layout_marginRight="@dimen/small_margin"
android:layout_marginTop="@dimen/small_margin"
android:text="@{content.finalUrl}"
android:text="@{content_youtube.finalUrl}"
android:textAlignment="center"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintRight_toRightOf="parent"
app:layout_constraintTop_toBottomOf="@+id/description"
app:layout_constraintTop_toBottomOf="@+id/youtube_description"
tools:text="www.vrgsoft.net" />
</android.support.constraint.ConstraintLayout>
</layout>
68 changes: 15 additions & 53 deletions library/src/main/java/net/vrgsoft/library/LinkCrawler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
package net.vrgsoft.library

import io.reactivex.*
import io.reactivex.Flowable
import io.reactivex.Single
import io.reactivex.processors.PublishProcessor
import io.reactivex.schedulers.Schedulers
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
import java.io.IOException
import java.net.MalformedURLException
import java.net.URL
import java.net.URLConnection
import io.reactivex.Flowable
import io.reactivex.processors.PublishProcessor

typealias LinkPreloadCallback = (String) -> Unit

Expand Down Expand Up @@ -67,24 +63,25 @@ class LinkCrawler {
return Single.fromCallable {
val urls: List<String> = SearchUrls.matches(url)
when {
urls.isNotEmpty() -> content.finalUrl = unshortenUrl(extendedTrim(urls[0]))
else -> content.finalUrl = ""
urls.isNotEmpty() -> content.url = extendedTrim(urls[0])
else -> content.url = ""
}
if (content.finalUrl != "") {
if (content.url != "") {
when {
isImage(content.finalUrl) && !content.finalUrl.contains("dropbox") -> {
isImage(content.url) && !content.url.contains("dropbox") -> {
content.success = true
content.images.add(content.finalUrl)
content.images.add(content.url)
content.title = ""
content.description = ""
}
else -> try {
val doc: Document = Jsoup.connect(content.finalUrl).userAgent("Mozzila").get()
val doc: Document = Jsoup.connect(content.url).userAgent("Mozzila").get()
content.htmlCode = extendedTrim(doc.toString())
val metaTags: Map<String, String> = getMetaTags(content.htmlCode)
content.metaTags = metaTags
content.title = metaTags["title"]!!
content.description = metaTags["description"]!!
content.title = metaTags["title"] ?: ""
content.description = metaTags["description"] ?: ""
content.finalUrl = metaTags["url"] ?: ""

when {
content.title == "" -> {
Expand All @@ -109,9 +106,9 @@ class LinkCrawler {
}
}
}
val linksSet = content.finalUrl.split("&")
content.url = linksSet[0]
content.canonicalUrl = canonicalPage(content.finalUrl)
val linksSet = content.url.split("&")
content.finalUrl = linksSet[0]
content.canonicalUrl = canonicalPage(content.url)
content.description = trimTags(content.description)
//return content
content
Expand Down Expand Up @@ -247,41 +244,6 @@ class LinkCrawler {
return matches
}

private fun unshortenUrl(url: String): String {
if (!url.startsWith(HTTP_PROTOCOL) && !url.startsWith(HTTPS_PROTOCOL)) {
return ""
}

var urlConn = connectURL(url)
urlConn?.headerFields

var finalResult = urlConn?.url.toString()

urlConn = connectURL(finalResult)
urlConn?.headerFields

while (urlConn?.url.toString() != finalResult) {
finalResult = unshortenUrl(finalResult)
}

return finalResult

}

private fun connectURL(strURL: String): URLConnection? {
var conn: URLConnection? = null
try {
val inputURL = URL(strURL)
conn = inputURL.openConnection()
} catch (e: MalformedURLException) {
println("Please input a valid URL")
} catch (ioe: IOException) {
println("Can not connect to the URL")
}

return conn
}

private fun htmlDecode(content: String): String = Jsoup.parse(content).text()

private fun trimTags(content: String): String = Jsoup.parse(content).text()
Expand Down