add:增加javdb自创清单jsoup加入IcodePool v1

src/main/java/top/lvzhiqiang/config/MyJobs.java → src/main/java/top/lvzhiqiang/config/MyJavJobs.java

@@ -29,7 +29,7 @@ import java.util.stream.Collectors;
  */
 @Component
 @Slf4j
-public class MyJobs {
+public class MyJavJobs {
 
     @Resource
     private VideoSitePoolMapper videoSitePoolMapper;

src/main/java/top/lvzhiqiang/entity/VideoMonitorActors.java

@@ -42,6 +42,11 @@ public class VideoMonitorActors implements Serializable {
     private String actorsRemark;
 
     /**
+     * 类型{1：他人，2：自己}
+     */
+    private Integer status;
+
+    /**
      * 创建时间
      */
     @JsonFormat(pattern = DateUtils.PATTERN_TO_SECONDS)

src/main/java/top/lvzhiqiang/mapper/IcodePoolMapper.java

@@ -63,6 +63,9 @@ public interface IcodePoolMapper {
     @Select("SELECT identification_code FROM icode_pool WHERE identification_code = #{identificationCode}")
     List<String> findIcodeByCode(String identificationCode);
 
+    @Select("select 1 from icode_pool where identification_code = #{code} limit 1")
+    Integer existIcodePoolByCode(String code);
+
     /**
      * 更新状态
      *

src/main/java/top/lvzhiqiang/mapper/VideoMonitorActorsMapper.java

@@ -15,21 +15,21 @@ import java.util.List;
  */
 public interface VideoMonitorActorsMapper {
 
-    @Select("select * from video_monitor_actors where website = #{website} and delete_flag = 1")
-    List<VideoMonitorActors> findAllListByWebsite(String website);
+    @Select("select * from video_monitor_actors where website = #{website} and type = #{type} and delete_flag = 1")
+    List<VideoMonitorActors> findAllListByWebsite(String website, Integer type);
 
-    @Select("select identification_code from video_monitor_actors_log where symbol = #{symbol}")
-    List<String> findAllLogListBySymbol(String symbol);
+    @Select("select identification_code from video_monitor_actors_log where symbol = #{symbol} and type = #{type}")
+    List<String> findAllLogListBySymbol(String symbol, Integer type);
 
     @Insert({"<script>" +
-            "INSERT ignore INTO video_monitor_actors_log(identification_code,symbol,create_time)" +
+            "INSERT ignore INTO video_monitor_actors_log(identification_code,symbol,type,create_time)" +
             " VALUES " +
             "<foreach collection='codeList' item='code' index=\"index\" separator=\",\">" +
-            "   (#{code}, #{symbol}, now())" +
+            "   (#{code}, #{symbol}, #{type}, now())" +
             "</foreach>" +
             "</script>"})
-    void insertLogList(@Param("codeList") List<String> codeList, @Param("symbol") String symbol);
+    void insertLogList(@Param("codeList") List<String> codeList, @Param("symbol") String symbol, @Param("type") Integer type);
 
-    @Select("select identification_code from video_monitor_actors_log where identification_code = #{code} limit 1")
-    String findAllLogListByCode(String code);
+    @Select("select identification_code from video_monitor_actors_log where identification_code = #{code} and type = #{type} limit 1")
+    String findAllLogListByCode(String code, Integer type);
 }

src/main/java/top/lvzhiqiang/service/impl/Crawler4JavdbServiceImpl.java

@@ -13,7 +13,9 @@ import org.jsoup.select.Elements;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import top.lvzhiqiang.entity.VideoMonitorActors;
+import top.lvzhiqiang.mapper.IcodePoolMapper;
 import top.lvzhiqiang.mapper.VideoMonitorActorsMapper;
+import top.lvzhiqiang.service.BgService;
 import top.lvzhiqiang.service.Crawler4JavdbService;
 import top.lvzhiqiang.util.JsoupUtil;
 import top.lvzhiqiang.util.StringUtils;
@@ -41,6 +43,10 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
 
     @Resource
     private VideoMonitorActorsMapper videoMonitorActorsMapper;
+    @Resource
+    private IcodePoolMapper icodePoolMapper;
+    @Resource
+    private BgService bgService;
 
     @Value("${spring.profiles.active}")
     private String env;
@@ -78,14 +84,14 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
             beforeJavbus();
 
             String website = "javdb";
-            List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website);
+            List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website, 1);
 
             Document document;
             Elements itembSelects;
             for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
                 try {
                     List<String> insertCodeList = new ArrayList<>();
-                    List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol());
+                    List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol(), 1);
 
                     Map<String, String> headerMap = new HashMap<>();
                     headerMap.put("referer", videoMonitorActors.getHttpUrl());
@@ -98,7 +104,7 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
                             continue;
                         }
 
-                        String identificationCode = videoMonitorActorsMapper.findAllLogListByCode(code);
+                        String identificationCode = videoMonitorActorsMapper.findAllLogListByCode(code, 1);
                         if (StringUtils.isNotEmpty(identificationCode)) {
                             continue;
                         }
@@ -128,7 +134,7 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
                     }
 
                     if (insertCodeList.size() > 0) {
-                        videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol());
+                        videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol(), 1);
                     }
                 } catch (Exception e) {
                     e.printStackTrace();
@@ -136,6 +142,62 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
             }
             log.warn("monitorActors结束==============================");
         }, 0, 2, TimeUnit.HOURS);
+
+        scheduler.scheduleAtFixedRate(() -> {
+            log.warn("monitorActors4Own开始==============================");
+            beforeJavbus();
+
+            String website = "javdb";
+            List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website, 2);
+
+            Document document;
+            Elements itembSelects;
+            for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
+                Map<String, String> unIcodePoolMap = new HashMap<>();
+                List<String> insertCodeList = new ArrayList<>();
+                try {
+                    List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol(), 2);
+                    Map<String, String> headerMap = new HashMap<>();
+                    headerMap.put("referer", videoMonitorActors.getHttpUrl());
+                    document = JsoupUtil.requestDocument(videoMonitorActors.getHttpUrl(), JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+                    itembSelects = document.select("div.movie-list").select("div.item");
+
+                    for (Element itembSelect : itembSelects) {
+                        String code = itembSelect.select("a.box").get(0).select("div.video-title").select("strong").text().trim().toUpperCase();
+                        if (codeList.contains(code)) {
+                            break;
+                        }
+
+                        Integer exist = icodePoolMapper.existIcodePoolByCode(code);
+                        if (exist != null) {
+                            insertCodeList.add(code);
+                            continue;
+                        }
+
+                        String url = itembSelect.select("a.box").get(0).attr("abs:href");
+                        unIcodePoolMap.put(code, url);
+                        insertCodeList.add(code);
+
+                        Thread.sleep(2000);
+                    }
+
+                    // insert to IcodePool
+                    if (unIcodePoolMap.size() > 0) {
+                        unIcodePoolMap.forEach((k, v) -> icodePoolMapper.insert(k, v));
+                    }
+                    // insert to video_monitor_actors_log
+                    if (insertCodeList.size() > 0) {
+                        videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol(), 2);
+                    }
+                    // jsoupIcodePool
+                    bgService.jsoupIcodePool("javdb", 1, 2, 2);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+
+            log.warn("monitorActors4Own结束==============================");
+        }, 0, 3, TimeUnit.HOURS);
     }
 
     @Override