feat: 加入404链接筛选

Leon406 · Apr 17, 2024 · 4fb32b1 · 4fb32b1
1 parent cba3124
commit 4fb32b1
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 29 deletions.
diff --git a/src/main/java/me/leon/support/Ext.kt b/src/main/java/me/leon/support/Ext.kt
@@ -3,35 +3,49 @@ package me.leon.support
 import java.net.HttpURLConnection
 import java.net.URL
 import java.text.SimpleDateFormat
-import java.util.*
+import java.util.Calendar
+import java.util.TimeZone
 
 private const val DEFAULT_READ_TIME_OUT = 30_000
 private const val DEFAULT_CONNECT_TIME_OUT = 30_000
 
+fun String.httpRequest(timeout:Int = DEFAULT_READ_TIME_OUT) = (URL(this).openConnection().apply {
+    //                setRequestProperty("Referer",
+    // "https://pc.woozooo.com/mydisk.php")
+    connectTimeout = DEFAULT_CONNECT_TIME_OUT
+    readTimeout = timeout
+    setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
+    setRequestProperty(
+        "user-agent",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " +
+                "Chrome/86.0.4240.198 Safari/537.36 Clash"
+    )
+} as HttpURLConnection)
+
 fun String.readFromNet() =
     runCatching {
-            String(
-                (URL(this).openConnection().apply {
-                        //                setRequestProperty("Referer",
-                        // "https://pc.woozooo.com/mydisk.php")
-                        connectTimeout = DEFAULT_CONNECT_TIME_OUT
-                        readTimeout = DEFAULT_READ_TIME_OUT
-                        setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
-                        setRequestProperty(
-                            "user-agent",
-                            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " +
-                                "Chrome/86.0.4240.198 Safari/537.36 Clash"
-                        )
-                    } as HttpURLConnection)
-                    .takeIf {
-                        //            println("$this __ ${it.responseCode}")
-                        it.responseCode == RESPONSE_OK
-                    }
-                    ?.inputStream
-                    ?.readBytes()
-                    ?: "".toByteArray()
-            )
-        }
+        String(
+            (URL(this).openConnection().apply {
+                //                setRequestProperty("Referer",
+                // "https://pc.woozooo.com/mydisk.php")
+                connectTimeout = DEFAULT_CONNECT_TIME_OUT
+                readTimeout = DEFAULT_READ_TIME_OUT
+                setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
+                setRequestProperty(
+                    "user-agent",
+                    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " +
+                            "Chrome/86.0.4240.198 Safari/537.36 Clash"
+                )
+            } as HttpURLConnection)
+                .takeIf {
+                    //            println("$this __ ${it.responseCode}")
+                    it.responseCode == RESPONSE_OK
+                }
+                ?.inputStream
+                ?.readBytes()
+                ?: "".toByteArray()
+        )
+    }
         .getOrElse {
             println("read err ${it.message}")
             ""

diff --git a/src/test/java/me/leon/ConnectTest.kt b/src/test/java/me/leon/ConnectTest.kt
@@ -1,7 +1,6 @@
 package me.leon
 
-import kotlinx.coroutines.async
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.*
 import me.leon.support.*
 import org.junit.jupiter.api.Test
 
@@ -38,4 +37,24 @@ class ConnectTest {
                 .forEach { println(it.first.info() + ":" + it.second) }
         }
     }
+
+    @Test
+    fun url404() {
+        runBlocking {
+            ("$ROOT/pool/sublists".readLines() + "$ROOT/pool/subs")
+                .filterNot { it.startsWith("#") }
+                .map {
+                    async(DISPATCHER) {
+                        it to runCatching { it.httpRequest(10000).responseCode == 404 }.getOrDefault(false)
+                    }
+                }
+                .awaitAll()
+                .filter { it.second }
+                .forEach {
+                    println(it)
+                }
+        }
+
+
+    }
 }
diff --git a/sub/pool/subs b/sub/pool/subs
@@ -1,4 +1,3 @@
-https://raw.githubusercontent.com/AzadNetCH/Clash/main/AzadNet.yml
 https://raw.githubusercontent.com/adiwzx/freenode/main/adispeed.yml
 https://raw.githubusercontent.com/ccnndwa/SSR/main/SSR
 https://raw.githubusercontent.com/freefq/free/master/v2
@@ -7,15 +6,12 @@ https://raw.githubusercontent.com/ssrsub/ssr/master/Clash.yml
 https://raw.githubusercontent.com/zyzmzyz/free-nodes/master/Clash.yml
 https://raw.githubusercontent.com/pojiezhiyuanjun/freev2/master/0727clash.yml
 https://raw.githubusercontent.com/xiyaowong/freeFQ/main/v2ray
-https://raw.githubusercontent.com/oslook/clash-freenode/main/clash.yaml
 #https://raw.githubusercontent.com/yu-steven/openit/main/sub/nodes.yaml
-https://raw.githubusercontent.com/alanbobs999/TopFreeProxies/main/Eternity
 https://raw.githubusercontent.com/iwxf/free-v2ray/master/index.html
 https://raw.githubusercontent.com/ermaozi/get_subscribe/main/subscribe/v2ray.txt
 https://raw.githubusercontent.com/wrfree/free/main/v2
 https://raw.githubusercontent.com/GreenFishStudio/GreenFish/master/Subscription/GreenFishYYDS
 https://raw.githubusercontent.com/ronghuaxueleng/get_v2/main/pub/changfengoss.yaml
 https://raw.githubusercontent.com/anaer/Sub/main/clash.yaml
 https://v2ray.neocities.org/v2ray.txt
-https://raw.githubusercontent.com/songtao1873/providers/main/sub/base64
 https://raw.githubusercontent.com/mahdibland/SSAggregator/master/sub/sub_merge_yaml.yml