update:javbus搜索接口验证待解决v1

src/main/java/top/lvzhiqiang/controller/BgController.java

@@ -4,7 +4,6 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Controller;
-import org.springframework.ui.Model;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
@@ -153,12 +152,12 @@ public class BgController {
      */
     @RequestMapping("/jsoupVideoInfo")
     @ResponseBody
-    public R jsoupVideoInfo(Integer status) {
+    public R jsoupVideoInfo(Integer status, String website, String url, String identificationCode) {
         if (null == status) {
             status = 1;
         }
 
-        bgService.jsoupVideoInfo(status);
+        bgService.jsoupVideoInfo(status, website, url, identificationCode);
         return R.ok();
     }
 

src/main/java/top/lvzhiqiang/service/BgService.java

@@ -109,7 +109,7 @@ public interface BgService {
      * @author lvzhiqiang
      * 2022/5/5 18:21
      */
-    void jsoupVideoInfo(Integer status);
+    void jsoupVideoInfo(Integer status, String website, String url, String identificationCode);
 
     /**
      * 删除影片

src/main/java/top/lvzhiqiang/service/Crawler4JavbusService.java

@@ -1,5 +1,7 @@
 package top.lvzhiqiang.service;
 
+import java.util.Map;
+
 /**
  * Crawler Javbus Service
  *
@@ -13,4 +15,6 @@ public interface Crawler4JavbusService {
     void handleJavbusLog(Integer status) throws Exception;
 
     String findJavbusProfile(String keyword, Integer timeDay, Integer pic, String orderField, String order, Integer pageNo, Integer pageSize);
+
+    Map<String,String> getJavbusCookiesMap() throws Exception;
 }

src/main/java/top/lvzhiqiang/service/impl/BgServiceImpl.java

@@ -22,6 +22,7 @@ import top.lvzhiqiang.exception.BusinessException;
 import top.lvzhiqiang.exception.ParameterException;
 import top.lvzhiqiang.mapper.*;
 import top.lvzhiqiang.service.BgService;
+import top.lvzhiqiang.service.Crawler4JavbusService;
 import top.lvzhiqiang.util.DateUtils;
 import top.lvzhiqiang.util.JsoupUtil;
 import top.lvzhiqiang.util.StringUtils;
@@ -30,6 +31,7 @@ import javax.annotation.Resource;
 import java.io.*;
 import java.lang.reflect.Field;
 import java.math.BigDecimal;
+import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
@@ -75,12 +77,16 @@ public class BgServiceImpl implements BgService {
     private VideoInfoInfantryMapper videoInfoInfantryMapper;
     @Resource
     private CrawlerLoveFootMapper crawlerLoveFootMapper;
+    @Resource
+    private Crawler4JavbusService crawler4JavbusService;
 
     @Value("${spring.profiles.active}")
     private String env;
 
     private String ignoreFileTypeSuffix = "";
 
+    private Proxy proxy = null;
+
     /**
      * findDicCode
      *
@@ -415,20 +421,39 @@ public class BgServiceImpl implements BgService {
      */
     @Async
     @Override
-    public void jsoupVideoInfo(Integer status) {
+    public void jsoupVideoInfo(Integer status, String website, String url, String identificationCodeP) {
         log.warn("jsoupVideoInfo 开始：status={}", status);
 
         // 获取待抓取码列表
-        List<String> icodeList = videoInfoMapper.findIcodeByStatus(status);
-        if (icodeList.size() == 0) {
-            log.warn("status={}的icodeList为空", status);
-            return;
+        List<String> icodeList = new ArrayList<>();
+        if (StringUtils.isEmpty(identificationCodeP)) {
+            icodeList = videoInfoMapper.findIcodeByStatus(status);
+            if (icodeList.size() == 0) {
+                log.warn("status={}的icodeList为空", status);
+                return;
+            }
+        } else {
+            icodeList.add(identificationCodeP);
         }
+
         // 获取javbus防屏蔽地址
-        List<String> javbusUrlList = videoSitePoolMapper.findUrlByTypeAndDeleteFlag(1, 1);
-        if (javbusUrlList.size() == 0) {
-            log.warn("javbusUrlList为空");
-            return;
+        List<String> javbusUrlList = new ArrayList<>();
+        if (StringUtils.isEmpty(url)) {
+            javbusUrlList = videoSitePoolMapper.findUrlByTypeAndDeleteFlag(1, 1);
+            if (javbusUrlList.size() == 0) {
+                log.warn("javbusUrlList为空");
+                return;
+            }
+        } else {
+            javbusUrlList.add(url);
+        }
+
+        if (null == proxy) {
+            if ("dev".equals(env)) {
+                proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 1080));
+            } else {
+                proxy = Proxy.NO_PROXY;
+            }
         }
 
         // 获取图片保存路径
@@ -449,10 +474,12 @@ public class BgServiceImpl implements BgService {
                 try {
                     document = Jsoup.connect(javbusUrl.concat("/").concat(identificationCode))
                             .timeout(50000)
-                            //.proxy()
-                            //.data()
-                            .ignoreContentType(true)
+                            .proxy(proxy)
                             .userAgent(getUserAgent())
+                            .followRedirects(true)
+                            .ignoreContentType(true)
+                            .ignoreHttpErrors(true)
+                            .cookies(crawler4JavbusService.getJavbusCookiesMap())
                             .header("referer", "https://www.javbus.com/".concat(identificationCode))
                             .get();
 
@@ -490,7 +517,7 @@ public class BgServiceImpl implements BgService {
     public void getMaleCast(VideoInfo videoInfo) {
         Document document;
         try {
-            Map<String,String> headerMap = new HashMap<>();
+            Map<String, String> headerMap = new HashMap<>();
             headerMap.put("referer", "avdanyuwiki.com");
 
             document = JsoupUtil.requestDocument("https://avdanyuwiki.com/?s=".concat(videoInfo.getName()), JsoupUtil.HTTP_GET, Proxy.NO_PROXY, null, headerMap, null);

src/main/java/top/lvzhiqiang/service/impl/Crawler4JavbusServiceImpl.java

@@ -82,6 +82,20 @@ public class Crawler4JavbusServiceImpl implements Crawler4JavbusService {
         }
     }
 
+    @Override
+    public Map<String, String> getJavbusCookiesMap() throws Exception {
+        // 获取javbus常量MAP
+        if (javbusConstantMap == null) {
+            javbusConstantMap = dicCodeMapper.findAll().stream()
+                    .filter(x -> "javbus".equals(x.getCodeDesc()) && x.getEnv().contains(env))
+                    .collect(Collectors.toMap(DicCode::getCodeKey, DicCode::getCodeValue, (key1, key2) -> key1));
+        }
+        // 代理及TOKEN设置
+        beforeJavbus();
+
+        return javbusCookiesMap;
+    }
+
     @Async
     @Override
     public void jsoupJavbusProfile(Long start, Integer limit) throws Exception {

src/main/resources/static/bg.html

@@ -188,7 +188,19 @@
         <span class="font">jsoupVideoInfo</span>
         <form method="post" action="bg/jsoupVideoInfo">
             <span>status</span>
-            <input type="text" name="status" placeholder="1：未爬取，3：爬取失败。默认是未爬取" style="width: 300px;"/>
+            <select name="status" style="height: 21.43px;">
+                <option value="1">未爬取</option>
+                <option value="3">爬取失败</option>
+            </select>
+            <span>website</span>
+            <select name="website" style="height: 21.43px;">
+                <option value="javbus">javbus</option>
+                <option value="javdb">javdb</option>
+            </select>
+            <span>url</span>
+            <input type="text" name="url" placeholder="https://www.javbus.com" style="width: 100px;"/>
+            <span>identificationCode</span>
+            <input type="text" name="identificationCode"/>
             <input type="submit" value="提交">
         </form>
     </div>