pushcode

src/main/java/top/lvzhiqiang/mapper/IcodePoolMapper.java

@@ -39,7 +39,7 @@ public interface IcodePoolMapper {
     /**
      * 根据状态查询识别码
      */
-    @Select("SELECT identification_code FROM icode_pool WHERE delete_flag = 1 and status = #{status} and retry_count <= 3")
+    @Select("SELECT identification_code FROM icode_pool WHERE delete_flag = 1 and status = #{status} and retry_count < 3")
     List<String> findIcodeByStatus(Integer status);
 
     /**

src/main/java/top/lvzhiqiang/service/impl/BgServiceImpl.java

@@ -20,7 +20,6 @@ import top.lvzhiqiang.util.StringUtils;
 
 import javax.annotation.Resource;
 import java.io.*;
-import java.net.Proxy;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -153,6 +152,7 @@ public class BgServiceImpl implements BgService {
         Document document;
         VideoInfoPool videoInfoPool;
         String javbusUrl;
+        int j = 0;
         for (int i = 0; i < icodePoolList.size(); i++) {
 
             long start = System.currentTimeMillis();
@@ -172,7 +172,7 @@ public class BgServiceImpl implements BgService {
             while (retryCount <= 3) {
                 javbusUrl = javbusUrlList.get((int) (0 + Math.random() * (javbusUrlList.size() - 0)));
                 try {
-                    document = Jsoup.connect(javbusUrl.concat(identificationCode)).timeout(50000).userAgent(getUserAgent()).get();
+                    document = Jsoup.connect(javbusUrl.concat(identificationCode)).timeout(50000).header("referer", "https://www.javbus.com/".concat(identificationCode)).userAgent(getUserAgent()).get();
 
                     videoInfoPool = new VideoInfoPool();
                     long picTime = parseDocument(document, identificationCode, machiPath, videoInfoPool);
@@ -181,6 +181,8 @@ public class BgServiceImpl implements BgService {
                         icodePool.setRetryCount(retryCount);
                         icodePoolMapper.updateStatus(icodePool);
                         videoInfoPoolMapper.insert(videoInfoPool);
+
+                        j++;
                     }
 
                     log.info("jsoupIcodePool success：i={}，picTime={}，time={}，identificationCode={}", i, picTime, System.currentTimeMillis() - start, identificationCode);
@@ -192,7 +194,7 @@ public class BgServiceImpl implements BgService {
                         log.error("jsoupIcodePool error重试：i={}，retryCount={}，time={}，identificationCode={}", i, retryCount, System.currentTimeMillis() - start, identificationCode, e);
                     } else if (retryCount == 4) {
                         icodePool.setStatus(3);
-                        icodePool.setRetryCount(retryCount);
+                        icodePool.setRetryCount(retryCount - 1);
                         icodePool.setFailureCause(e.getMessage().length() <= 200 ? e.getMessage() : e.getMessage().substring(0, 200));
                         icodePoolMapper.updateStatus(icodePool);
                         log.error("jsoupIcodePool error：i={}，time={}，identificationCode={}", i, System.currentTimeMillis() - start, identificationCode, e);
@@ -200,6 +202,7 @@ public class BgServiceImpl implements BgService {
                 }
             }
         }
+        log.info("jsoupIcodePool 统计：total={}，success={}", icodePoolList.size(), j);
     }
 
     private String getUserAgent() {
@@ -218,8 +221,9 @@ public class BgServiceImpl implements BgService {
                 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
                 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
                 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
-                "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0"};
-        int i = r.nextInt(14);
+                "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36"};
+        int i = r.nextInt(15);
         return ua[i];
     }
 

src/main/resources/static/bg.html

@@ -48,7 +48,7 @@
         <span class="font">jsoupIcodePool</span>
         <form method="post" action="bg/jsoupIcodePool" enctype="multipart/form-data">
             <span>status</span>
-            <input type="text" name="status" placeholder="1：未爬取，2：爬取失败。默认是未爬取" style="width: 300px;"/>
+            <input type="text" name="status" placeholder="1：未爬取，3：爬取失败。默认是未爬取" style="width: 300px;"/>
             <span>isDel</span>
             <input type="text" name="isDel" placeholder="1：是，2：否。默认是"/>
             <input type="submit" value="提交">

src/test/java/Test2.java

@@ -14,13 +14,12 @@ import java.time.LocalDateTime;
 
 public class Test2 {
     public static void main(String[] args) {
-        String javbusUrl = "https://www.javbus.com/";
-        String identificationCode = "VEO-049";// DTT-049  HAWA-243  HISN-011 DANDY-745  VOSS-172  PFES-024 VIDA-005  SHKD-843  CAWD-176  BLK-467
+        String javbusUrl = "https://www.seejav.work/";
+        String identificationCode = "MGMQ-053";// DTT-049  HAWA-243  HISN-011 DANDY-745  VOSS-172  PFES-024 VIDA-005  SHKD-843  CAWD-176  BLK-467
         Document document;
         VideoInfoPool videoInfoPool;
         try {
-            Proxy proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("trojan.lvzhiqiang.top", Integer.parseInt("4430")));
-            document = Jsoup.connect(javbusUrl.concat(identificationCode)).timeout(50000).proxy(proxy).get();
+            document = Jsoup.connect(javbusUrl.concat(identificationCode)).timeout(50000).get();
 
             videoInfoPool = new VideoInfoPool();
             parseDocument(document, identificationCode, videoInfoPool);