add:足舐类型添加jpfoot站点爬取v1

src/main/java/top/lvzhiqiang/config/MyJobs.java

@@ -199,6 +199,8 @@ public class MyJobs {
         log.warn("jsoupLoveFoot开始==============================");
 
         crawler4LoveFootService.jsoupLoveFoot4avnoashi(null, 2, 2);
+
+        crawler4LoveFootService.jsoupLoveFoot4jpfoot(null, 2, 2);
     }
 
     /**

src/main/java/top/lvzhiqiang/controller/CrawlerController.java

@@ -248,8 +248,12 @@ public class CrawlerController {
 
         if (4 == status) {
             crawler4LoveFootService.jsoupLoveFoot4CrawingFail(status, ignoreRetryCount, website);
-        } else {
-            crawler4LoveFootService.jsoupLoveFoot4avnoashi(status, isDel, ignoreRetryCount);
+        } else if (status > 10) {
+            if (status == 11) {
+                crawler4LoveFootService.jsoupLoveFoot4avnoashi(status, isDel, ignoreRetryCount);
+            } else if (status == 12) {
+                crawler4LoveFootService.jsoupLoveFoot4jpfoot(status, isDel, ignoreRetryCount);
+            }
         }
 
         return R.ok();

src/main/java/top/lvzhiqiang/entity/CrawlerLoveFoot.java

@@ -76,7 +76,8 @@ public class CrawlerLoveFoot implements Serializable {
     /**
      * 原始URL
      */
-    private String orginUrl;
+    private String orginAvnoashiUrl;
+    private String orginJpfootUrl;
 
     private String javdbUrl;
 

src/main/java/top/lvzhiqiang/entity/VideoInfo.java

@@ -120,7 +120,8 @@ public class VideoInfo implements Serializable {
     /**
      * 原始URL
      */
-    private String orginUrl;
+    private String orginAvnoashiUrl;
+    private String orginJpfootUrl;
 
     private String score;
     private String comment;
@@ -139,4 +140,6 @@ public class VideoInfo implements Serializable {
     private String videoInfoPoolFlag;
     private String videoLoveFootFlag;
     private String javdbUrl;
+
+    private String source;
 }

src/main/java/top/lvzhiqiang/mapper/CrawlerLoveFootMapper.java

@@ -29,19 +29,26 @@ public interface CrawlerLoveFootMapper {
     /**
      * 查询最新的一条
      */
-    @Select("SELECT * FROM crawler_lovefoot_info WHERE delete_flag = 1 order by update_date desc limit 1")
-    CrawlerLoveFoot findLatestInfo();
+    @Select("SELECT * FROM crawler_lovefoot_info WHERE orgin_avnoashi_url is not null and delete_flag = 1 order by update_date desc limit 1")
+    CrawlerLoveFoot findLatestInfo4avnoashi();
+    @Select("SELECT * FROM crawler_lovefoot_info WHERE orgin_jpfoot_url is not null and delete_flag = 1 order by update_date desc limit 1")
+    CrawlerLoveFoot findLatestInfo4jpfoot();
 
     /**
      * 新增
      *
      * @param crawlerLoveFoot
      */
-    @Insert("INSERT INTO crawler_lovefoot_info(name, identification_code, issue_date, length, director, maker, issuer, genres, cast, img_url, orgin_url, type, status, failure_cause, clock_date, update_date, create_time, modify_time) " +
-            "VALUES (#{name}, #{identificationCode}, #{issueDate}, #{length}, #{director}, #{maker}, #{issuer}, #{genres}, #{cast}, #{imgUrl}, #{orginUrl}, #{type}, #{status}, #{failureCause}, #{clockDate}, #{updateDate}, #{createTime}, now()) " +
+    @Insert("INSERT INTO crawler_lovefoot_info(name, identification_code, issue_date, length, director, maker, issuer, genres, cast, img_url, orgin_avnoashi_url, type, status, failure_cause, clock_date, update_date, create_time, modify_time) " +
+            "VALUES (#{name}, #{identificationCode}, #{issueDate}, #{length}, #{director}, #{maker}, #{issuer}, #{genres}, #{cast}, #{imgUrl}, #{orginAvnoashiUrl}, #{type}, #{status}, #{failureCause}, #{clockDate}, #{updateDate}, #{createTime}, now()) " +
             "ON DUPLICATE KEY UPDATE name=values(name),issue_date=values(issue_date),length=values(length),director=values(director),maker=values(maker),issuer=values(issuer),genres=values(genres),cast=values(cast)," +
-            "img_url=values(img_url),orgin_url=values(orgin_url),type=values(type),status=values(status),failure_cause=values(failure_cause),clock_date=values(clock_date),update_date=values(update_date),modify_time=now()")
-    int insertOrUpdate(CrawlerLoveFoot crawlerLoveFoot);
+            "img_url=values(img_url),orgin_avnoashi_url=values(orgin_avnoashi_url),type=values(type),status=values(status),failure_cause=values(failure_cause),clock_date=values(clock_date),update_date=values(update_date),modify_time=now()")
+    int insertOrUpdate4avnoashi(CrawlerLoveFoot crawlerLoveFoot);
+    @Insert("INSERT INTO crawler_lovefoot_info(name, identification_code, issue_date, length, director, maker, issuer, genres, cast, img_url, orgin_jpfoot_url, type, status, failure_cause, clock_date, update_date, create_time, modify_time) " +
+            "VALUES (#{name}, #{identificationCode}, #{issueDate}, #{length}, #{director}, #{maker}, #{issuer}, #{genres}, #{cast}, #{imgUrl}, #{orginJpfootUrl}, #{type}, #{status}, #{failureCause}, #{clockDate}, #{updateDate}, #{createTime}, now()) " +
+            "ON DUPLICATE KEY UPDATE name=values(name),issue_date=values(issue_date),length=values(length),director=values(director),maker=values(maker),issuer=values(issuer),genres=values(genres),cast=values(cast)," +
+            "img_url=values(img_url),orgin_jpfoot_url=values(orgin_jpfoot_url),type=values(type),status=values(status),failure_cause=values(failure_cause),clock_date=values(clock_date),update_date=values(update_date),modify_time=now()")
+    int insertOrUpdate4jpfoot(CrawlerLoveFoot crawlerLoveFoot);
 
     /**
      * 获取类型

src/main/java/top/lvzhiqiang/service/Crawler4LoveFootService.java

@@ -11,4 +11,6 @@ public interface Crawler4LoveFootService {
     void jsoupLoveFoot4avnoashi(Integer status, Integer isDel, Integer ignoreRetryCount) throws Exception;
 
     void jsoupLoveFoot4CrawingFail(Integer status, Integer ignoreRetryCount, String website);
+
+    void jsoupLoveFoot4jpfoot(Integer status, Integer isDel, Integer ignoreRetryCount) throws Exception;
 }

src/main/java/top/lvzhiqiang/service/impl/Crawler4LoveFootServiceImpl.java

@@ -109,6 +109,38 @@ public class Crawler4LoveFootServiceImpl implements Crawler4LoveFootService {
     @Async
     @Override
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
+    public void jsoupLoveFoot4jpfoot(Integer status, Integer isDel, Integer ignoreRetryCount) throws Exception {
+        log.warn("jsoupLoveFoot4jpfoot 开始：status={}，isDel={}，ignoreRetryCount={}", status, isDel, ignoreRetryCount);
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+        if (isDel == 1) {
+            crawlerLoveFootMapper.deleteAll();
+        }
+
+        List<DicCode> dicCodeList = dicCodeMapper.findAll();
+        // 获取常量MAP
+        footConstantMap = dicCodeList.stream()
+                .filter(x -> "foot".equals(x.getCodeDesc()) && x.getEnv().contains(env))
+                .collect(Collectors.toMap(DicCode::getCodeKey, DicCode::getCodeValue, (key1, key2) -> key1));
+        javbusConstantMap = dicCodeList.stream()
+                .filter(x -> x.getType() != null && 1 == x.getType() && x.getEnv().contains(env))
+                .collect(Collectors.toMap(DicCode::getCodeKey, DicCode::getCodeValue, (key1, key2) -> key1));
+        // 获取javbus防屏蔽地址
+        javbusUrlList = videoSitePoolMapper.findUrlByTypeAndDeleteFlag(1, 1);
+        if (javbusUrlList.size() == 0) {
+            log.warn("javbusUrlList为空");
+            return;
+        }
+        // 代理及TOKEN设置
+        beforeProxy();
+        // 解析原始站点
+        jsoupLoveFoot4jpfootSub(status, ignoreRetryCount);
+        log.warn("jsoupLoveFoot4jpfoot 结束：time={}", stopWatch.getTotalTimeSeconds());
+    }
+
+    @Async
+    @Override
+    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
     public void jsoupLoveFoot4CrawingFail(Integer status, Integer ignoreRetryCount, String website) {
         log.warn("jjsoupLoveFoot4CrawingFail 开始");
         StopWatch stopWatch = new StopWatch();
@@ -448,7 +480,7 @@ public class Crawler4LoveFootServiceImpl implements Crawler4LoveFootService {
 
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
     public void jsoupLoveFoot4avnoashiSub(Integer status, Integer ignoreRetryCount) throws Exception {
-        CrawlerLoveFoot latestLoveFoot = crawlerLoveFootMapper.findLatestInfo();
+        CrawlerLoveFoot latestLoveFoot = crawlerLoveFootMapper.findLatestInfo4avnoashi();
 
         LocalDate latestDate;
         if (latestLoveFoot == null) {
@@ -500,7 +532,7 @@ public class Crawler4LoveFootServiceImpl implements Crawler4LoveFootService {
                     CrawlerLoveFoot crawlerLoveFoot = new CrawlerLoveFoot();
                     crawlerLoveFoot.setClockDate(clockDate);
                     crawlerLoveFoot.setUpdateDate(updateDate);
-                    crawlerLoveFoot.setOrginUrl(sourceUrl);
+                    crawlerLoveFoot.setOrginAvnoashiUrl(sourceUrl);
                     crawlerLoveFoot.setType(2);
                     crawlerLoveFoot.setStatus(3);
                     crawlerLoveFoot.setCreateTime(LocalDateTime.now());
@@ -510,12 +542,12 @@ public class Crawler4LoveFootServiceImpl implements Crawler4LoveFootService {
                         throw new Exception(message);
                     }
 
-                    crawlerLoveFootMapper.insertOrUpdate(crawlerLoveFoot);
+                    crawlerLoveFootMapper.insertOrUpdate4avnoashi(crawlerLoveFoot);
                 } catch (Exception e) {
                     log.error("jsoupLoveFoot4avnoashiSub detail fail，sourceUrl={}", sourceUrl, e);
                     CrawlerLoveFoot crawlerLoveFoot = new CrawlerLoveFoot();
                     crawlerLoveFoot.setIdentificationCode(UUID.randomUUID().toString());
-                    crawlerLoveFoot.setOrginUrl(sourceUrl);
+                    crawlerLoveFoot.setOrginAvnoashiUrl(sourceUrl);
                     crawlerLoveFoot.setClockDate(clockDate);
                     crawlerLoveFoot.setUpdateDate(updateDate);
                     crawlerLoveFoot.setName(keywords);
@@ -523,7 +555,7 @@ public class Crawler4LoveFootServiceImpl implements Crawler4LoveFootService {
                     crawlerLoveFoot.setStatus(statusInt);
                     crawlerLoveFoot.setCreateTime(LocalDateTime.now());
                     crawlerLoveFoot.setFailureCause(e.getMessage());
-                    crawlerLoveFootMapper.insertOrUpdate(crawlerLoveFoot);
+                    crawlerLoveFootMapper.insertOrUpdate4avnoashi(crawlerLoveFoot);
                 }
             }
 
@@ -537,6 +569,96 @@ public class Crawler4LoveFootServiceImpl implements Crawler4LoveFootService {
         }
     }
 
+    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
+    public void jsoupLoveFoot4jpfootSub(Integer status, Integer ignoreRetryCount) throws Exception {
+        CrawlerLoveFoot latestLoveFoot = crawlerLoveFootMapper.findLatestInfo4jpfoot();
+
+        LocalDate latestDate;
+        if (latestLoveFoot == null) {
+            latestDate = LocalDate.of(1970, 1, 1);
+        } else {
+            latestDate = latestLoveFoot.getUpdateDate();
+        }
+
+        String jpfootUrl = footConstantMap.get("jpfoot_url");
+        headerMap.put("referer", jpfootUrl);
+        Document loveFootDocument;
+        Document loveFootDetailDocument;
+        outer:
+        while (true) {
+            loveFootDocument = JsoupUtil.requestDocument(jpfootUrl, JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+            log.warn("jsoupLoveFoot4jpfootSub page success：url={}", jpfootUrl);
+
+            Elements sourceSelects = loveFootDocument.select("article.mainContainer > div.av_itemGrid").select("article.av_item");
+            for (Element sourceSelect : sourceSelects) {
+                Thread.sleep(1000L);
+                String sourceUrl = sourceSelect.select("a.av_itemLink").attr("abs:href");
+
+                Integer statusInt = 2;
+                Integer typeInt = 1;
+                LocalDate clockDate = null;
+                LocalDate updateDate = null;
+                String keywords = null;
+                try {
+                    loveFootDetailDocument = JsoupUtil.requestDocument(sourceUrl, JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+                    String dateStr = loveFootDetailDocument.select("div.avdetail_date").select("span.avdetail_dateText").text();
+                    clockDate = LocalDate.parse(dateStr, DateUtils.dateFormatter4);
+                    updateDate = clockDate;
+
+                    if (updateDate.isBefore(latestDate) || updateDate.isEqual(latestDate)) {
+                        break outer;
+                    }
+
+                    // 获取关键词
+                    keywords = loveFootDetailDocument.select("div.avdetail_detailTop").select("p.avdetail_detailTopTitle").text().trim();
+                    if (StringUtils.isNotEmpty(keywords)) {
+                        statusInt = 1;
+                        log.warn("jsoupLoveFoot4jpfootSub parseDetailToKeywords success，sourceUrl={}，keywords={}", sourceUrl, keywords);
+                    } else {
+                        throw new Exception("keywords is null");
+                    }
+
+                    // 通过关键词获取识别码
+                    CrawlerLoveFoot crawlerLoveFoot = new CrawlerLoveFoot();
+                    crawlerLoveFoot.setClockDate(clockDate);
+                    crawlerLoveFoot.setUpdateDate(updateDate);
+                    crawlerLoveFoot.setOrginJpfootUrl(sourceUrl);
+                    crawlerLoveFoot.setType(2);
+                    crawlerLoveFoot.setStatus(3);
+                    crawlerLoveFoot.setCreateTime(LocalDateTime.now());
+                    String message = parseKeywordsToCode(crawlerLoveFoot, keywords);
+                    if (StringUtils.isNotEmpty(message)) {
+                        statusInt = 4;
+                        throw new Exception(message);
+                    }
+
+                    crawlerLoveFootMapper.insertOrUpdate4jpfoot(crawlerLoveFoot);
+                } catch (Exception e) {
+                    log.error("jsoupLoveFoot4jpfootSub detail fail，sourceUrl={}", sourceUrl, e);
+                    CrawlerLoveFoot crawlerLoveFoot = new CrawlerLoveFoot();
+                    crawlerLoveFoot.setIdentificationCode(UUID.randomUUID().toString());
+                    crawlerLoveFoot.setOrginJpfootUrl(sourceUrl);
+                    crawlerLoveFoot.setClockDate(clockDate);
+                    crawlerLoveFoot.setUpdateDate(updateDate);
+                    crawlerLoveFoot.setName(keywords);
+                    crawlerLoveFoot.setType(typeInt);
+                    crawlerLoveFoot.setStatus(statusInt);
+                    crawlerLoveFoot.setCreateTime(LocalDateTime.now());
+                    crawlerLoveFoot.setFailureCause(e.getMessage());
+                    crawlerLoveFootMapper.insertOrUpdate4jpfoot(crawlerLoveFoot);
+                }
+            }
+
+            // 继续下一页
+            Elements nextSelects = loveFootDocument.select("nav.pagination > div.nav-links").select("a.next");
+            if (nextSelects.size() > 0) {
+                jpfootUrl = nextSelects.get(0).attr("abs:href");
+            } else {
+                break;
+            }
+        }
+    }
+
     private String parseKeywordsToCode(CrawlerLoveFoot crawlerLoveFoot, String keywords) {
         int retryCount = 0;
         Document javbusSearchDocument = null;

src/main/java/top/lvzhiqiang/util/DateUtils.java

@@ -37,6 +37,7 @@ public class DateUtils {
     public static final String PATTERN_TO_DAYS2 = "yyyy.MM.dd";
 
     public static final String PATTERN_TO_DAYS3 = "yyyy年M月d日";
+    public static final String PATTERN_TO_DAYS4 = "yyyy/M/d";
 
     /**
      * （精确到秒的）日期样式
@@ -47,6 +48,7 @@ public class DateUtils {
     public static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern(PATTERN_TO_DAYS);
     public static final DateTimeFormatter dateFormatter2 = DateTimeFormatter.ofPattern(PATTERN_TO_DAYS2);
     public static final DateTimeFormatter dateFormatter3 = DateTimeFormatter.ofPattern(PATTERN_TO_DAYS3);
+    public static final DateTimeFormatter dateFormatter4 = DateTimeFormatter.ofPattern(PATTERN_TO_DAYS4);
 
     public static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(PATTERN_TO_SECONDS);
     public static final DateTimeFormatter dateTimeFormatter3 = DateTimeFormatter.ofPattern(PATTERN_TO_MINUTES);

src/main/resources/mapper/VideoInfoLoveFootMapper.xml

@@ -6,7 +6,7 @@
 <mapper namespace="top.lvzhiqiang.mapper.CrawlerLoveFootMapper">
 
     <select id="getVideoInfoLoveFootList" resultType="top.lvzhiqiang.entity.VideoInfo">
-        select vi.id,vi.name,vi.identification_code,vi.issue_date,vi.modify_time,vi.update_date,vi.type,vi.img_url,vi.orgin_url,IFNULL(vi.cast, vi.genres) AS main_who,IFNULL(vio.score, 0) AS score, IFNULL(vio.comment, '暂无评论') AS comment,
+        select vi.id,vi.name,vi.identification_code,vi.issue_date,vi.modify_time,vi.update_date,vi.type,vi.img_url,vi.orgin_avnoashi_url,vi.orgin_jpfoot_url,IFNULL(vi.cast, vi.genres) AS main_who,IFNULL(vio.score, 0) AS score, IFNULL(vio.comment, '暂无评论') AS comment,
             IFNULL(temp1.identification_code, 'false') AS videoInfoFlag,IFNULL(temp2.identification_code, 'false') AS videoInfoPoolFlag
         from crawler_lovefoot_info vi
         left join video_info_other vio on vi.identification_code = vio.identification_code and vio.delete_flag = 1

src/main/resources/static/crawler.html

@@ -245,7 +245,8 @@
             <select name="status" style="height: 21.43px;">
                 <option value="2">获取关键词失败</option>
                 <option value="4">爬取失败</option>
-                <option value="">从0开始</option>
+                <option value="11">从0开始avnoashi</option>
+                <option value="12">从0开始jpfoot</option>
             </select>
             <span>website</span>
             <select name="website" style="height: 21.43px;">

src/main/resources/static/js/my-video.js

@@ -148,7 +148,9 @@ function search(pageNo, startFlag, searchSelectFlag) {
                         scoreStr += (videoInfo.type === 1 ? '爬取失败' : (videoInfo.type === 2 ? '待审查' : (videoInfo.type === 3 ? '审查已通过' : '审查未通过'))) + '|';
                         scoreStr += videoInfo.score;
 
-                        orginUrl = videoInfo.orginUrl;
+                        orginUrl += videoInfo.orginAvnoashiUrl === null ? '' : videoInfo.orginAvnoashiUrl;
+                        orginUrl += (videoInfo.orginAvnoashiUrl != null && videoInfo.orginJpfootUrl != null) ? ',' : '';
+                        orginUrl += videoInfo.orginJpfootUrl === null ? '' : videoInfo.orginJpfootUrl;
                     } else {
                         scoreStr = videoInfo.score;
                     }