l1024-jav/src/main/java/top/lvzhiqiang/config/MyJobs.java

package top.lvzhiqiang.config;

import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import top.lvzhiqiang.entity.DicCode;
import top.lvzhiqiang.entity.VideoSitePool;
import top.lvzhiqiang.mapper.VideoSitePoolMapper;
import top.lvzhiqiang.service.BgService;
import top.lvzhiqiang.service.Crawler4JavbusService;
import top.lvzhiqiang.service.Crawler4LoveFootService;
import top.lvzhiqiang.service.CrawlerService;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * 定时任务
 *
 * @author lvzhiqiang
 * 2022/4/28 15:49
 */
@Component
@Slf4j
public class MyJobs {

    @Resource
    private VideoSitePoolMapper videoSitePoolMapper;

    @Resource
    private BgService bgService;
    @Resource
    private CrawlerService crawlerService;
    @Resource
    private Crawler4JavbusService crawler4JavbusService;
    @Resource
    private Crawler4LoveFootService crawler4LoveFootService;

    private static final String SCHEDULED_ZONE = "Asia/Shanghai";

    /**
     * 每天06:00 校验站点有效性
     */
    @Scheduled(cron = "0 0 6 * * ?", zone = SCHEDULED_ZONE)
    //@Scheduled(cron = "0 10 19 * * ?",zone = SCHEDULED_ZONE)
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void checkVideoSite() {
        log.warn("checkVideoSite开始==============================");

        // 获取javbus官方地址
        DicCode dicCode = WebAppConfig.dicCodeList.stream().filter(x -> 2 == x.getType() && "javbus".equals(x.getCodeKey())).findFirst().get();
        if (dicCode == null) {
            log.warn("javbus官方站点为Null");
            return;
        }

        // 获取javbusUrlList
        List<String> javbusUrlList = videoSitePoolMapper.findUrlByType(1);

        // 获取javbusNewUrlList
        Set<String> javbusNewUrlList = new HashSet<>();
        try {
            Document document = Jsoup.connect(dicCode.getCodeValue()).timeout(50000).ignoreContentType(true)
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36")
                    .header("referer", "https://www.javbus.com/").get();

            Elements ahrefList = document.select("strong:contains(防屏蔽地址)").next("a");
            for (Element element : ahrefList) {
                String text = element.text();
                log.warn("Jsoup获取{}防屏蔽地址：{}", dicCode.getCodeValue(), text);
                javbusNewUrlList.add(text);
            }
        } catch (Exception e) {
            log.error("Jsoup获取{}防屏蔽地址异常", dicCode.getCodeValue(), e);
        }

        if (javbusNewUrlList.size() == 0 && javbusUrlList.size() > 0) {
            for (String javbusUrl : javbusUrlList) {
                try {
                    Document document = Jsoup.connect(javbusUrl).timeout(50000).ignoreContentType(true)
                            .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36")
                            .header("referer", "https://www.javbus.com/").get();

                    Elements ahrefList = document.select("strong:contains(防屏蔽地址)").next("a");
                    for (Element element : ahrefList) {
                        String text = element.text();
                        log.info("Jsoup获取{}防屏蔽地址：{}", javbusUrl, text);
                        javbusNewUrlList.add(text);
                    }
                    if (javbusNewUrlList.size() > 0) {
                        break;
                    }
                } catch (Exception e) {
                    log.error("Jsoup获取{}防屏蔽地址异常", javbusUrl, e);
                }
            }
        }

        if (javbusNewUrlList.size() == 0) {
            log.warn("javbusNewUrlList为空");
        }
        if (javbusNewUrlList.size() == 0 && javbusUrlList.size() == 0) {
            log.warn("javbusUrlList和javbusNewUrlList为空");
            return;
        }

        // 校验新地址
        List<String> javbusNewUrlFinalList = javbusNewUrlList.stream().filter(e -> !javbusUrlList.contains(e)).collect(Collectors.toList());
        List<VideoSitePool> videoSitePoolList = new ArrayList<>();
        VideoSitePool videoSitePool;
        for (String javbusNewUrlFinal : javbusNewUrlFinalList) {
            try {
                Jsoup.connect(javbusNewUrlFinal).timeout(50000);

                videoSitePool = new VideoSitePool();
                videoSitePool.setUrl(javbusNewUrlFinal);
                videoSitePool.setType(1);
                videoSitePoolList.add(videoSitePool);
                log.warn("javbusNewUrlFinalList：javbus防屏蔽地址有效！javbusUrl={}", javbusNewUrlFinal);
            } catch (Exception e) {
                log.error("javbusNewUrlFinalList：javbus防屏蔽地址失效！javbusUrl={}", javbusNewUrlFinal, e);
            }
        }
        if (videoSitePoolList.size() > 0) {
            videoSitePoolMapper.insertList(videoSitePoolList);
        }
        // 校验存量地址
        for (String javbusUrl : javbusUrlList) {
            int deleteFlag = 1;
            try {
                Jsoup.connect(javbusUrl).timeout(50000);
                log.warn("javbusUrlList：javbus防屏蔽地址有效！javbusUrl={}", javbusUrl);
            } catch (Exception e) {
                deleteFlag = 2;
                log.error("javbusUrlList：javbus防屏蔽地址失效！javbusUrl={}", javbusUrl, e);
            }
            videoSitePoolMapper.updateDeleteFlag(javbusUrl, deleteFlag);
        }

        log.warn("checkVideoSite结束==============================");
    }

    /**
     * 每天20:00 Jsoup码池
     */
    @Scheduled(cron = "0 00 20 * * ?", zone = SCHEDULED_ZONE)
    public void jsoupIcodePool4CrawingNo() {
        log.warn("jsoupIcodePool4CrawingNo开始==============================");

        bgService.jsoupIcodePool(1, 2, 2);
    }

    /**
     * 每天20:30 Jsoup码池
     */
    @Scheduled(cron = "0 30 20 * * ?", zone = SCHEDULED_ZONE)
    public void jsoupIcodePool4CrawingFail() {
        log.warn("jsoupIcodePool4CrawingFail开始==============================");

        bgService.jsoupIcodePool(3, 2, 2);
    }

    /**
     * 每天18:00 jsoupXiaoeknowCourse
     */
    @Scheduled(cron = "0 00 18 * * ?", zone = SCHEDULED_ZONE)
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void jsoupXiaoeknowCourse() {
        log.warn("jsoupXiaoeknowCourse开始==============================");

        crawlerService.jsoupXiaoeknowCourse(null, 2, null);
    }

    /**
     * 每隔一小时执行一次 jsoupJavbusProfile
     */
    @Scheduled(cron = "0 0 * * * ?", zone = SCHEDULED_ZONE)
    public void jsoupJavbusProfileJob() throws Exception {
        log.warn("jsoupJavbusProfileJob开始==============================");

        crawler4JavbusService.jsoupJavbusProfile(null, null);
    }

    /**
     * 每天23:50 jsoupLoveFoot
     */
    @Scheduled(cron = "0 50 23 * * ?", zone = SCHEDULED_ZONE)
    public void jsoupLoveFoot() throws Exception {
        log.warn("jsoupLoveFoot开始==============================");

        crawler4LoveFootService.jsoupLoveFoot4avnoashi(null, 2, 2);
    }

    /**
     * 每天23:55 jsoupLoveFoot4CrawingFail
     */
    // @Scheduled(cron = "0 55 23 * * ?", zone = SCHEDULED_ZONE)
    public void jsoupLoveFoot4CrawingFail() {
        log.warn("jsoupLoveFoot4CrawingFail开始==============================");

        crawler4LoveFootService.jsoupLoveFoot4CrawingFail(4, 2);
    }
}