add:增加jsoupIkoaMovie4VideoInfo功能v1

src/main/java/top/lvzhiqiang/controller/CrawlerController.java

@@ -82,4 +82,21 @@ public class CrawlerController {
 
         return crawlerService.searchIkoaMovie(keywords, movieType, by, page);
     }
+
+    /**
+     * jsoupIkoaMovie4VideoInfo
+     *
+     * @author lvzhiqiang
+     * 2022/9/25 0:37
+     */
+    @RequestMapping("/jsoupIkoaMovie4VideoInfo")
+    @ResponseBody
+    public String jsoupIkoaMovie4VideoInfo(String bitrate) throws Exception {
+        if (StringUtils.isEmpty(bitrate)) {
+            throw new ParameterException("bitrate不能为空");
+        }
+
+        crawlerService.jsoupIkoaMovie4VideoInfo(bitrate);
+        return "success";
+    }
 }

src/main/java/top/lvzhiqiang/entity/Temp4k.java

@@ -0,0 +1,49 @@
+package top.lvzhiqiang.entity;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+/**
+ * 字典码
+ *
+ * @author lvzhiqiang
+ * 2022/9/25 0:37
+ */
+@Data
+public class Temp4k implements Serializable {
+
+    /**
+     * 主键
+     */
+    private Long id;
+
+    /**
+     * name
+     */
+    private String name;
+
+    /**
+     * cost
+     */
+    private Integer cost;
+
+    /**
+     * success
+     */
+    private Integer success;
+
+    /**
+     * 创建时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
+    private LocalDateTime createTime;
+
+    /**
+     * 最后修改时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
+    private LocalDateTime modifyTime;
+}

src/main/java/top/lvzhiqiang/mapper/TempMapper.java

@@ -0,0 +1,28 @@
+package top.lvzhiqiang.mapper;
+
+import org.apache.ibatis.annotations.Insert;
+import org.apache.ibatis.annotations.Select;
+import top.lvzhiqiang.entity.Temp4k;
+
+import java.util.List;
+
+/**
+ * Temp Mapper
+ *
+ * @author lvzhiqiang
+ * 2022/9/25 0:37
+ */
+public interface TempMapper {
+
+    @Select("select distinct vi.identification_code " +
+            "from video_info vi" +
+            "         left join video_info_other vio on vi.identification_code = vio.identification_code " +
+            "where vio.resolution_ratio != #{bitrate} " +
+            "   or vio.resolution_ratio is null")
+    List<String> findVideoInfo4BitrateNo4k(String bitrate);
+
+    @Insert("INSERT INTO temp_4k(name, cost, success, create_time, modify_time) " +
+            "VALUES (#{name}, #{cost}, 2, now(), now()) " +
+            "ON DUPLICATE KEY UPDATE cost=values(cost),modify_time=now()")
+    void insertOrUpdate4k(Temp4k temp4k);
+}

src/main/java/top/lvzhiqiang/service/CrawlerService.java

@@ -31,4 +31,6 @@ public interface CrawlerService {
      * 2022/9/17 12:55
      */
     String searchIkoaMovie(String keywords, String movieType, String by, String page) throws Exception;
+
+    void jsoupIkoaMovie4VideoInfo(String bitrate) throws Exception;
 }

src/main/java/top/lvzhiqiang/service/impl/CrawlerServiceImpl.java

@@ -7,15 +7,18 @@ import org.jsoup.Connection;
 import org.jsoup.HttpStatusException;
 import org.jsoup.Jsoup;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.StopWatch;
 import top.lvzhiqiang.entity.CrawlerXiaoeknowCourse;
 import top.lvzhiqiang.entity.DicCode;
+import top.lvzhiqiang.entity.Temp4k;
 import top.lvzhiqiang.exception.BusinessException;
 import top.lvzhiqiang.mapper.CrawlerXiaoeknowCourseMapper;
 import top.lvzhiqiang.mapper.DicCodeMapper;
+import top.lvzhiqiang.mapper.TempMapper;
 import top.lvzhiqiang.service.CrawlerService;
 import top.lvzhiqiang.util.DateUtils;
 import top.lvzhiqiang.util.StringUtils;
@@ -43,6 +46,8 @@ public class CrawlerServiceImpl implements CrawlerService {
     private CrawlerXiaoeknowCourseMapper crawlerXiaoeknowCourseMapper;
     @Resource
     private DicCodeMapper dicCodeMapper;
+    @Resource
+    private TempMapper tempMapper;
 
     @Value("${spring.profiles.active}")
     private String env;
@@ -317,6 +322,70 @@ public class CrawlerServiceImpl implements CrawlerService {
         return sb.toString();
     }
 
+    @Override
+    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
+    @Async
+    public void jsoupIkoaMovie4VideoInfo(String bitrate) throws Exception {
+        List<String> list = tempMapper.findVideoInfo4BitrateNo4k(bitrate);
+
+        // 获取ikoa常量MAP
+        Map<String, String> ikoaConstantMap = dicCodeMapper.findAll().stream()
+                .filter(x -> "ikoa".equals(x.getCodeDesc()) && x.getEnv().contains(env))
+                .collect(Collectors.toMap(DicCode::getCodeKey, DicCode::getCodeValue, (key1, key2) -> key1));
+
+        // 代理及TOKEN设置
+        Proxy proxy;
+        if ("dev".equals(env)) {
+            proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 1080));
+        } else {
+            proxy = Proxy.NO_PROXY;
+        }
+        if (StringUtils.isEmpty(ikoaToken)) {
+            generateIkoaToken(ikoaConstantMap, proxy);
+        }
+        Connection.Response response;
+        JSONObject result = null;
+        // List<Temp4k> temp4kList = new ArrayList<>();
+        for (String code : list) {
+            outer:
+            for (int i = 0; i < 3; i++) {
+                try {
+                    response = Jsoup.connect(ikoaConstantMap.get("search_url"))
+                            .header("Authorization", "Token " + ikoaToken)
+                            .timeout(50000)
+                            .proxy(proxy)
+                            .data("page", "1").data("keywords", code).data("movie_type", "*").data("by", "*")
+                            .ignoreContentType(true)
+                            .userAgent(getUserAgent())
+                            .header("referer", ikoaConstantMap.get("referer"))
+                            .method(Connection.Method.GET)
+                            .execute();
+                    result = JSONObject.parseObject(response.body());
+                    JSONArray data = result.getJSONArray("data");
+                    for (int j = 0; j < data.size(); j++) {
+                        JSONObject jsonObject = data.getJSONObject(i);
+                        if (jsonObject.getString("movie_type").contains(bitrate.toLowerCase())) {
+                            Temp4k temp4k = new Temp4k();
+                            temp4k.setName(code);
+                            temp4k.setCost(jsonObject.getInteger("cost"));
+                            // temp4kList.add(temp4k);
+                            tempMapper.insertOrUpdate4k(temp4k);
+                            break outer;
+                        }
+                    }
+                    break;
+                } catch (HttpStatusException hse) {
+                    generateIkoaToken(ikoaConstantMap, proxy);
+                } catch (Exception e) {
+                    log.error("IKOA搜索异常，ikoaConstantMap={},result={}", ikoaConstantMap, result.toString(), e);
+                    if (i == 2) {
+                        throw new Exception("IKOA搜索异常！");
+                    }
+                }
+            }
+        }
+    }
+
     private void parseIkoaSearchResult(JSONArray result, StringBuffer sb) {
         sb.append("<table border=\"1\" cellspacing=\"0\"><tr><th>序号</th><th>影片名称</th><th>cid</th><th>pid</th><th>影片类型</th><th>码率</th><th>总分段数量</th><th>价格</th><th>是否已购</th><th>购买后有效期</th><th>封面</th></tr>");
 

src/main/resources/static/crawler.html

@@ -93,6 +93,16 @@
             <input type="submit" value="提交">
         </form>
     </div>
+    <div style="margin-right:20px;">
+        <span class="font">jsoupIkoaMovie4VideoInfo</span>
+        <form method="post" action="bg/crawler/jsoupIkoaMovie4VideoInfo">
+            <span>码率</span>
+            <select name="bitrate" style="height: 21.43px;">
+                <option value="4k">4k</option>
+            </select>
+            <input type="submit" value="提交">
+        </form>
+    </div>
 </div>
 </body>
 </html>