Added way to test HTML & forcing all Instagram request with header

Udhayarajan · Udhayarajan · commit 12a8eadd7f1f · 2023-06-27T05:36:10.000+05:30
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -27,7 +27,7 @@ plugins {
 }
 
 group = "io.github.udhayarajan"
-version = "5.3.9"
+version = "5.3.10"
 //Version Naming incremented if "<NEW_FEATURE_ADDED>.<WORKED_ON_BUG>.<BETA_VERSION_COUNT>"
 //Priority on incrementing Feature > BugFix > Beta
 
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt
@@ -107,6 +107,10 @@ class DailyMotion(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun extractFromM3U8(response: String) {
         fun valueForKey(key: String, line: String): String? {
             val matcher = Pattern.compile("$key=(?:\"(.*?)\"|(.*?),)").matcher(line)
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt
@@ -119,11 +119,12 @@ abstract class Extractor(
 
     private suspend fun safeAnalyze() {
         try {
-            if (inputUrl.contains("instagram"))
+            if (inputUrl.contains("instagram")) {
                 inputUrl = if (cookies == null) {
                     inputUrl.replace("/reels/", "/reel/")
                 } else inputUrl.replace("/reel/", "/reels/")
-
+                headers["User-Agent"] = "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"
+            }
             if (HttpRequest(inputUrl, headers).isAvailable())
                 analyze()
             else if (inputUrl.contains("instagram") && cookies != null) {
@@ -180,7 +181,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (videoData in format.videoData) {
-                sizes.add(async { HttpRequest(videoData.url).getSize() })
+                sizes.add(async { HttpRequest(videoData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -190,7 +191,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (audioData in format.audioData) {
-                sizes.add(async { HttpRequest(audioData.url).getSize() })
+                sizes.add(async { HttpRequest(audioData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -200,7 +201,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (imageData in format.imageData) {
-                sizes.add(async { HttpRequest(imageData.url).getSize() })
+                sizes.add(async { HttpRequest(imageData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -213,4 +214,6 @@ abstract class Extractor(
     protected inline fun internalError(msg: String, e: Exception? = null) {
         onProgress(Result.Failed(Error.InternalError(msg, e)))
     }
+
+    abstract suspend fun testWebpage(string: String)
 }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt
@@ -523,7 +523,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {
         return null
     }
 
-    suspend fun testWebpage(string: String) {
+    override suspend fun testWebpage(string: String) {
         onProgress = {
 
         }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt
@@ -156,14 +156,14 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 inputUrl = inputUrl.replace("https://instagram.com", "https://www.instagram.com")
                 logger.info("The new url is $inputUrl&__a=1&__d=dis")
                 val items =
-                    HttpRequest(inputUrl.plus("&__a=1&__d=dis"), getHeadersWithUserAgent()).getResponse(true) ?: run {
+                    HttpRequest(inputUrl.plus("&__a=1&__d=dis"), headers).getResponse(true) ?: run {
                         clientRequestError("unable to get post event with __a=1&__d=dis")
                         return
                     }
                 extractFromItems(items.toJSONObject().getJSONArray("items"))
                 return
             }
-            extractInfoShared(HttpRequest(inputUrl, getHeadersWithUserAgent()).getResponse() ?: run {
+            extractInfoShared(HttpRequest(inputUrl, headers).getResponse() ?: run {
                 clientRequestError()
                 return
             })
@@ -182,7 +182,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
     }
 
     private suspend fun extractHighlights(highlightsId: String) {
-        val highlights = HttpRequest(HIGHLIGHTS_API.format("%3A$highlightsId"), getHeadersWithUserAgent()).getResponse()
+        val highlights = HttpRequest(HIGHLIGHTS_API.format("%3A$highlightsId"), headers).getResponse()
             ?.toJSONObjectOrNull()
         highlights?.let {
             if (it.getNullable("login_required") == "true") {
@@ -197,8 +197,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
 
     @Suppress("UNCHECKED_CAST")
     private suspend fun extractStories(userId: String) {
-        val dupHeader = getHeadersWithUserAgent()
-        val stories = HttpRequest(STORIES_API.format(userId), dupHeader).getResponse()
+        val stories = HttpRequest(STORIES_API.format(userId), headers).getResponse()
         val reel = JSONObject(stories).getNullableJSONObject("reel")
         reel?.let { extractFromItems(it.getJSONArray("items")) } ?: onProgress(
             Result.Failed(
@@ -209,12 +208,6 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         )
     }
 
-    private fun getHeadersWithUserAgent(): Hashtable<String, String> {
-        val dupHeader: Hashtable<String, String> = headers.clone() as Hashtable<String, String>
-        dupHeader["User-Agent"] =
-            "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"
-        return dupHeader
-    }
 
 
     private suspend fun extractInfoShared(page: String) {
@@ -228,7 +221,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 if (mediaId == null) throw JSONException("mediaId is null purposely thrown wrong error")
                 val url = POST_API.format(mediaId)
                 extractFromItems(
-                    HttpRequest(url, getHeadersWithUserAgent())
+                    HttpRequest(url, headers)
                         .getResponse()
                         .toString()
                         .toJSONObject()
@@ -378,7 +371,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         val matcher = Pattern.compile("<link rel=\"preload\" href=\"(.*?)\" as=\"script\"").matcher(page)
         while (matcher.find()) {
             ids.addAll(
-                getQueryHash(HttpRequest(matcher.group(1)).getResponse() ?: run {
+                getQueryHash(HttpRequest(matcher.group(1), headers).getResponse() ?: run {
                     return null
                 })
             )
@@ -560,4 +553,12 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         } else
             finalize()
     }
+
+    override suspend fun testWebpage(string: String) {
+        onProgress = {
+            println(it)
+        }
+        extractInfoShared(string)
+    }
+
 }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt
@@ -57,6 +57,10 @@ class Likee internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun extractVideoList(jsonArray: JSONArray) {
         for (i in 0 until jsonArray.length()) {
             val localFormats = formats.copy(title = "", videoData = mutableListOf(), imageData = mutableListOf())
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt
@@ -42,6 +42,10 @@ class LinkedIn internal constructor(url: String) : Extractor(url) {
         })
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun scratchWebpage(page: String) {
         val matcher = Pattern.compile("data-sources=\"(.*?)\"").matcher(page)
         if (matcher.find()) {
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt
@@ -43,6 +43,10 @@ class Periscope internal constructor(url: String) : Extractor(url) {
         TODO("Not yet implemented")
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     var manifest: ArrayList<ArrayList<String>>? = ArrayList()
 
 
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt
@@ -42,6 +42,10 @@ class ShareChat internal constructor(url: String) : Extractor(url) {
         })
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun scratchWebPage(response: String) {
         val matcher =
             Pattern.compile("""<script data-rh="true" type="application\/ld\+json">(\{"@context":"http:\/\/schema\.org","@type":"(?:Image|Video)Object".*?\})<\/script>""")
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt
@@ -80,7 +80,7 @@ class TikTok internal constructor(url: String) : Extractor(url) {
             onProgress(Result.Failed(Error.MethodMissingLogic))
     }
 
-    suspend fun testWithWebPage(string: String){
+    override suspend fun testWebpage(string: String) {
         onProgress = {
             println(it)
         }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt
@@ -38,6 +38,10 @@ class Twitch internal constructor(url: String) : Extractor(url) {
         TODO("Not yet implemented")
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private fun getId(s: String?): String? {
         return s?.run {
             val matcher =
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt
@@ -235,6 +235,10 @@ class Twitter internal constructor(url: String) : Extractor(url) {
         super.finalize()
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
 
     fun resolution(url: String?): String {
         val pattern: Pattern = Pattern.compile("/(\\d+x\\d+)/")
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt
@@ -64,6 +64,10 @@ class Vimeo internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun parseConfigRequest(response: String) {
         val json = response.toJSONObject()
         val hls = json.getJSONObject("request").getJSONObject("files").getJSONObject("dash")
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt
@@ -42,13 +42,11 @@ class HttpRequest(
     companion object {
         private var prefixUrl = ""
         private var additionHeader: Hashtable<String, String>? = null
-        private fun defaultClient(requiresRedirection: Boolean = true) = HttpClient(Android).also {
-            it.config {
+        private fun defaultClient(requiresRedirection: Boolean = true) = HttpInterfaceImpl(
+            HttpClient(Android).config {
                 followRedirects = requiresRedirection
             }
-        }.run {
-            HttpInterfaceImpl(this)
-        }
+        )
 
         private var clientGenerator: () -> HttpClient = {
             HttpClient(Android)
@@ -104,7 +102,7 @@ class HttpRequest(
      * @return bytes count of given [url]
      */
     suspend fun getSize(useCustomClient: Boolean = true) =
-        (if (useCustomClient) createClient() else defaultClient()).getSize(url)
+        (if (useCustomClient) createClient() else defaultClient()).getSize(url, getHeader())
 
     suspend fun postRequest(postData: Hashtable<String, Any>? = null, useCustomClient: Boolean = true): String =
         withContext(Dispatchers.IO) {

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ plugins {`
`27`	`27`	`}`
`28`	`28`
`29`	`29`	`group = "io.github.udhayarajan"`
`30`		`-version = "5.3.9"`
	`30`	`+version = "5.3.10"`
`31`	`31`	`//Version Naming incremented if "<NEW_FEATURE_ADDED>.<WORKED_ON_BUG>.<BETA_VERSION_COUNT>"`
`32`	`32`	`//Priority on incrementing Feature > BugFix > Beta`
`33`	`33`
Original file line number	Diff line number	Diff line change
`@@ -107,6 +107,10 @@ class DailyMotion(url: String) : Extractor(url) {`
`107`	`107`	`}`
`108`	`108`	`}`
`109`	`109`
	`110`	`+ override suspend fun testWebpage(string: String) {`
	`111`	`+ TODO("Not yet implemented")`
	`112`	`+ }`
	`113`	`+`
`110`	`114`	`private suspend fun extractFromM3U8(response: String) {`
`111`	`115`	`fun valueForKey(key: String, line: String): String? {`
`112`	`116`	`val matcher = Pattern.compile("$key=(?:\"(.?)\"\|(.?),)").matcher(line)`
Original file line number	Diff line number	Diff line change
`@@ -119,11 +119,12 @@ abstract class Extractor(`
`119`	`119`
`120`	`120`	`private suspend fun safeAnalyze() {`
`121`	`121`	`try {`
`122`		`- if (inputUrl.contains("instagram"))`
	`122`	`+ if (inputUrl.contains("instagram")) {`
`123`	`123`	`inputUrl = if (cookies == null) {`
`124`	`124`	`inputUrl.replace("/reels/", "/reel/")`
`125`	`125`	`} else inputUrl.replace("/reel/", "/reels/")`
`126`		`-`
	`126`	`+ headers["User-Agent"] = "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"`
	`127`	`+ }`
`127`	`128`	`if (HttpRequest(inputUrl, headers).isAvailable())`
`128`	`129`	`analyze()`
`129`	`130`	`else if (inputUrl.contains("instagram") && cookies != null) {`
`@@ -180,7 +181,7 @@ abstract class Extractor(`
`180`	`181`	`val sizes = mutableListOf<Deferred<Long>>()`
`181`	`182`	`coroutineScope {`
`182`	`183`	`for (videoData in format.videoData) {`
`183`		`- sizes.add(async { HttpRequest(videoData.url).getSize() })`
	`184`	`+ sizes.add(async { HttpRequest(videoData.url, headers).getSize() })`
`184`	`185`	`}`
`185`	`186`	`}`
`186`	`187`	`return sizes.awaitAll()`
`@@ -190,7 +191,7 @@ abstract class Extractor(`
`190`	`191`	`val sizes = mutableListOf<Deferred<Long>>()`
`191`	`192`	`coroutineScope {`
`192`	`193`	`for (audioData in format.audioData) {`
`193`		`- sizes.add(async { HttpRequest(audioData.url).getSize() })`
	`194`	`+ sizes.add(async { HttpRequest(audioData.url, headers).getSize() })`
`194`	`195`	`}`
`195`	`196`	`}`
`196`	`197`	`return sizes.awaitAll()`
`@@ -200,7 +201,7 @@ abstract class Extractor(`
`200`	`201`	`val sizes = mutableListOf<Deferred<Long>>()`
`201`	`202`	`coroutineScope {`
`202`	`203`	`for (imageData in format.imageData) {`
`203`		`- sizes.add(async { HttpRequest(imageData.url).getSize() })`
	`204`	`+ sizes.add(async { HttpRequest(imageData.url, headers).getSize() })`
`204`	`205`	`}`
`205`	`206`	`}`
`206`	`207`	`return sizes.awaitAll()`
`@@ -213,4 +214,6 @@ abstract class Extractor(`
`213`	`214`	`protected inline fun internalError(msg: String, e: Exception? = null) {`
`214`	`215`	`onProgress(Result.Failed(Error.InternalError(msg, e)))`
`215`	`216`	`}`
	`217`	`+`
	`218`	`+ abstract suspend fun testWebpage(string: String)`
`216`	`219`	`}`
Original file line number	Diff line number	Diff line change
`@@ -523,7 +523,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {`
`523`	`523`	`return null`
`524`	`524`	`}`
`525`	`525`
`526`		`- suspend fun testWebpage(string: String) {`
	`526`	`+ override suspend fun testWebpage(string: String) {`
`527`	`527`	`onProgress = {`
`528`	`528`
`529`	`529`	`}`
Original file line number	Diff line number	Diff line change
`@@ -57,6 +57,10 @@ class Likee internal constructor(url: String) : Extractor(url) {`
`57`	`57`	`}`
`58`	`58`	`}`
`59`	`59`
	`60`	`+ override suspend fun testWebpage(string: String) {`
	`61`	`+ TODO("Not yet implemented")`
	`62`	`+ }`
	`63`	`+`
`60`	`64`	`private suspend fun extractVideoList(jsonArray: JSONArray) {`
`61`	`65`	`for (i in 0 until jsonArray.length()) {`
`62`	`66`	`val localFormats = formats.copy(title = "", videoData = mutableListOf(), imageData = mutableListOf())`
Original file line number	Diff line number	Diff line change
`@@ -43,6 +43,10 @@ class Periscope internal constructor(url: String) : Extractor(url) {`
`43`	`43`	`TODO("Not yet implemented")`
`44`	`44`	`}`
`45`	`45`
	`46`	`+ override suspend fun testWebpage(string: String) {`
	`47`	`+ TODO("Not yet implemented")`
	`48`	`+ }`
	`49`	`+`
`46`	`50`	`var manifest: ArrayList<ArrayList<String>>? = ArrayList()`
`47`	`51`
`48`	`52`
Original file line number	Diff line number	Diff line change
`@@ -80,7 +80,7 @@ class TikTok internal constructor(url: String) : Extractor(url) {`
`80`	`80`	`onProgress(Result.Failed(Error.MethodMissingLogic))`
`81`	`81`	`}`
`82`	`82`
`83`		`- suspend fun testWithWebPage(string: String){`
	`83`	`+ override suspend fun testWebpage(string: String) {`
`84`	`84`	`onProgress = {`
`85`	`85`	`println(it)`
`86`	`86`	`}`