update:crawler youtube live采用xxljob方式v1

src/main/java/top/lvzhiqiang/config/MyCoinJobs.java

@@ -26,9 +26,6 @@ public class MyCoinJobs {
     @Resource
     private CoinService2 coinService2;
 
-    @Resource
-    private CoinYoutubeService coinYoutubeService;
-
     private static final String SCHEDULED_ZONE = "Asia/Shanghai";
 
     /**
@@ -81,34 +78,4 @@ public class MyCoinJobs {
         } catch (Exception e) {
         }
     }
-
-    /**
-     * @throws Exception
-     */
-    @Scheduled(cron = "0 0 1 * * ?", zone = SCHEDULED_ZONE)
-    public void jsoupYoutubeLive4yt2140() throws Exception {
-        log.warn("jsoupYoutubeLive4yt2140开始==============================");
-
-        coinYoutubeService.jsoupYoutubeLive4yt2140();
-    }
-
-    /**
-     * @throws Exception
-     */
-    @Scheduled(cron = "0 30 1 * * ?", zone = SCHEDULED_ZONE)
-    public void jsoupYoutubeLive4yt2140Chapter4No() throws Exception {
-        log.warn("jsoupYoutubeLive4yt2140Chapter4No开始==============================");
-
-        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(3, null, null);
-    }
-
-    /**
-     * @throws Exception
-     */
-    @Scheduled(cron = "0 30 2 * * ?", zone = SCHEDULED_ZONE)
-    public void jsoupYoutubeLive4yt2140Chapter4Fail() throws Exception {
-        log.warn("jsoupYoutubeLive4yt2140Chapter4Fail开始==============================");
-
-        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(2, null, null);
-    }
 }

src/main/java/top/lvzhiqiang/job/CoinYoutubeJob.java

@@ -0,0 +1,87 @@
+package top.lvzhiqiang.job;
+
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import top.lvzhiqiang.service.CoinYoutubeService;
+
+import javax.annotation.Resource;
+
+/**
+ * coin-youtube更新任务
+ *
+ * @author lvzhiqiang
+ * 2025/10/18 11:04
+ */
+@Component
+@Slf4j
+public class CoinYoutubeJob {
+
+    @Resource
+    private CoinYoutubeService coinYoutubeService;
+
+    /**
+     * jsoupYoutubeLive4yt2140任务处理器
+     * 0 0 1 * * ?
+     *
+     * @return com.xxl.job.core.biz.model.ReturnT<java.lang.String>
+     * @author lvzhiqiang
+     * 2025/10/18 11:04
+     */
+    @XxlJob("jsoupYoutubeLive4yt2140JobHandler")
+    public ReturnT<String> jsoupYoutubeLive4yt2140JobHandler() {
+        try {
+            coinYoutubeService.jsoupYoutubeLive4yt2140();
+        } catch (Exception e) {
+            log.error("jsoupYoutubeLive4yt2140JobHandler exception", e);
+            XxlJobHelper.log(e);
+            return ReturnT.FAIL;
+        }
+
+        return ReturnT.SUCCESS;
+    }
+
+    /**
+     * jsoupYoutubeLive4yt2140Chapter4No任务处理器
+     * 0 30 1 * * ?
+     *
+     * @return com.xxl.job.core.biz.model.ReturnT<java.lang.String>
+     * @author lvzhiqiang
+     * 2025/10/18 11:04
+     */
+    @XxlJob("jsoupYoutubeLive4yt2140Chapter4NoJobHandler")
+    public ReturnT<String> jsoupYoutubeLive4yt2140Chapter4NoJobHandler() {
+        try {
+            coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(3, null, null, "4No待爬取");
+        } catch (Exception e) {
+            log.error("jsoupYoutubeLive4yt2140Chapter4NoJobHandler exception", e);
+            XxlJobHelper.log(e);
+            return ReturnT.FAIL;
+        }
+
+        return ReturnT.SUCCESS;
+    }
+
+    /**
+     * jsoupYoutubeLive4yt2140Chapter4Fail任务处理器
+     * 0 30 2 * * ?
+     *
+     * @return com.xxl.job.core.biz.model.ReturnT<java.lang.String>
+     * @author lvzhiqiang
+     * 2025/10/18 11:04
+     */
+    @XxlJob("jsoupYoutubeLive4yt2140Chapter4FailJobHandler")
+    public ReturnT<String> jsoupYoutubeLive4yt2140Chapter4FailJobHandler() {
+        try {
+            coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(2, null, null, "4Fail失败");
+        } catch (Exception e) {
+            log.error("jsoupYoutubeLive4yt2140Chapter4NoJobHandler exception", e);
+            XxlJobHelper.log(e);
+            return ReturnT.FAIL;
+        }
+
+        return ReturnT.SUCCESS;
+    }
+}

src/main/java/top/lvzhiqiang/service/CoinYoutubeService.java

@@ -17,7 +17,7 @@ public interface CoinYoutubeService {
 
     List<String> getChannelLiveStreams(String channelUrl) throws IOException, InterruptedException;
 
-    void jsoupYoutubeLive4yt2140Chapter(Integer status, String originalUrl, String id);
+    void jsoupYoutubeLive4yt2140Chapter(Integer status, String originalUrl, String id, String remark);
 
     void jsoupYoutubeLive4yt2140ChapterSub(CoinYoutubeYt2140Live coinYoutubeYt2140Live);
 

src/main/java/top/lvzhiqiang/service/impl/BgServiceImpl.java

@@ -86,13 +86,16 @@ public class BgServiceImpl implements BgService {
     private String ignoreFileTypeSuffix = "";
 
     private Proxy proxy = null;
+    private Proxy proxy2 = null;
 
     public void beforeProxy() {
         if (null == proxy) {
             if ("dev".equals(env)) {
                 proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 1080));
+                proxy2 = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 1080));
             } else {
                 proxy = Proxy.NO_PROXY;
+                proxy2 = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 10808));
             }
         }
     }
@@ -551,7 +554,7 @@ public class BgServiceImpl implements BgService {
                 try {
                     Map<String, String> headerMap = new HashMap<>();
                     headerMap.put("referer", url);
-                    document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+                    document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy2, null, headerMap, null);
 
                     videoInfo = new VideoInfo();
                     videoInfo.setIdentificationCode(identificationCode);
@@ -632,19 +635,19 @@ public class BgServiceImpl implements BgService {
                     if (StringUtils.isNotEmpty(videoInfoInfantry.getJavdbUrl())) {
                         Map<String, String> headerMap = new HashMap<>();
                         headerMap.put("referer", url);
-                        document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+                        document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy2, null, headerMap, null);
                     } else {
                         String searchUrl = javdbConstantMap.get("javdb").concat("search?q=").concat(videoInfoInfantry.getIdentificationCode()).concat("&f=all");
                         Map<String, String> header3Map = new HashMap<>();
                         header3Map.put("referer", searchUrl);
 
-                        javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
+                        javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy2, null, header3Map, null);
 
                         itembSelects = javdbSearchDocument.select("div.movie-list").select("div.item");
                         if (itembSelects.size() == 0) {
                             String newName = videoInfoInfantry.getName();
                             searchUrl = javdbConstantMap.get("javdb").concat("search?q=").concat(newName).concat("&f=all");
-                            javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
+                            javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy2, null, header3Map, null);
                             itembSelects = javdbSearchDocument.select("div.movie-list").select("div.item");
                         }
 
@@ -674,7 +677,7 @@ public class BgServiceImpl implements BgService {
                             throw new BusinessException(30000, "javdb search result mismatch");
                         }
 
-                        document = JsoupUtil.requestDocument(codeUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
+                        document = JsoupUtil.requestDocument(codeUrl, JsoupUtil.HTTP_GET, proxy2, null, header3Map, null);
                         videoInfoInfantry.setJavdbUrl(codeUrl);
                     }
 
@@ -751,19 +754,19 @@ public class BgServiceImpl implements BgService {
                     if (StringUtils.isNotEmpty(videoInfoUncensored.getJavdbUrl())) {
                         Map<String, String> headerMap = new HashMap<>();
                         headerMap.put("referer", url);
-                        document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+                        document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy2, null, headerMap, null);
                     } else {
                         String searchUrl = javdbConstantMap.get("javdb").concat("search?q=").concat(videoInfoUncensored.getIdentificationCode()).concat("&f=all");
                         Map<String, String> header3Map = new HashMap<>();
                         header3Map.put("referer", searchUrl);
 
-                        javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
+                        javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy2, null, header3Map, null);
 
                         itembSelects = javdbSearchDocument.select("div.movie-list").select("div.item");
                         if (itembSelects.size() == 0) {
                             String newName = videoInfoUncensored.getName();
                             searchUrl = javdbConstantMap.get("javdb").concat("search?q=").concat(newName).concat("&f=all");
-                            javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
+                            javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy2, null, header3Map, null);
                             itembSelects = javdbSearchDocument.select("div.movie-list").select("div.item");
 
                             ignoreCodeFlag = true;
@@ -798,7 +801,7 @@ public class BgServiceImpl implements BgService {
                             throw new BusinessException(30000, "javdb search result mismatch");
                         }
 
-                        document = JsoupUtil.requestDocument(codeUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
+                        document = JsoupUtil.requestDocument(codeUrl, JsoupUtil.HTTP_GET, proxy2, null, header3Map, null);
                         videoInfoUncensored.setJavdbUrl(codeUrl);
                     }
 
@@ -1504,7 +1507,7 @@ public class BgServiceImpl implements BgService {
                         Map<String, String> headerMap = new HashMap<>();
                         headerMap.put("referer", identificationCodeArr[1]);
                         beforeProxy();
-                        document = JsoupUtil.requestDocument(identificationCodeArr[1], JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
+                        document = JsoupUtil.requestDocument(identificationCodeArr[1], JsoupUtil.HTTP_GET, proxy2, null, headerMap, null);
                         picTime = parseDocument4Javdb(document, identificationCodeArr[0], machiPath, videoInfoPool);
                     }
 

src/main/java/top/lvzhiqiang/service/impl/CoinYoutubeServiceImpl.java

@@ -2,9 +2,9 @@ package top.lvzhiqiang.service.impl;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
@@ -47,9 +47,8 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
     private String env;
 
     @Override
-    @Async("coinTaskExecutor")
     public void jsoupYoutubeLive4yt2140() throws IOException, InterruptedException {
-        log.warn("jsoupYoutubeLive4yt2140 开始：");
+        XxlJobHelper.log("jsoupYoutubeLive4yt2140开始==============================：");
         StopWatch stopWatch = new StopWatch();
         stopWatch.start();
 
@@ -60,6 +59,7 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
         } else {
             latestOriginalUrl = coinYoutubeYt2140Live.getOriginalUrl();
         }
+        XxlJobHelper.log("当前库里latestOriginalUrl={}", latestOriginalUrl);
 
         String youtubeYt2140LiveUrl = InitRunner.dicCodeMap.get("youtube_yt2140_live_url").getCodeValue();
         List<String> liveStreams = getChannelLiveStreams(youtubeYt2140LiveUrl);
@@ -76,11 +76,11 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
             live.setStatus(3);
             int count = coinYoutubeMapper.insertIgnoreYt2140Live(live);
             findCount += count;
-            log.warn("jsoupYoutubeLive4yt2140 live success：originalUrl={}，count={}", originalUrl, count);
+            XxlJobHelper.log("jsoupYoutubeLive4yt2140 live success：originalUrl={}，count={}", originalUrl, count);
         }
 
         stopWatch.stop();
-        log.warn("jsoupYoutubeLive4yt2140 结束：findCount={}，time={}", findCount, stopWatch.getTotalTimeMillis());
+        XxlJobHelper.log("jsoupYoutubeLive4yt2140结束==============================：findCount={}，time={}", findCount, stopWatch.getTotalTimeMillis());
     }
 
     @Override
@@ -93,7 +93,7 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
         }
 
         // 构建命令
-        String command = "yt-dlp " + proxyStr + " --flat-playlist --print url " + channelUrl + "/streams";
+        String command = "/usr/local/bin/yt-dlp " + proxyStr + " --flat-playlist --print url " + channelUrl + "/streams";
 
         Process process = Runtime.getRuntime().exec(command);
         BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
@@ -114,9 +114,8 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
     }
 
     @Override
-    //@Async("coinTaskExecutor")
-    public void jsoupYoutubeLive4yt2140Chapter(Integer status, String originalUrl, String id) {
-        log.warn("jsoupYoutubeLive4yt2140Chapter 开始：status={}", status);
+    public void jsoupYoutubeLive4yt2140Chapter(Integer status, String originalUrl, String id, String remark) {
+        XxlJobHelper.log("jsoupYoutubeLive4yt2140Chapter开始==============================：remark={}", remark);
 
         StopWatch stopWatch = new StopWatch();
         stopWatch.start();
@@ -134,7 +133,7 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
 
         List<CoinYoutubeYt2140Live> coinYoutubeYt2140LiveList = coinYoutubeMapper.findJsoupYt2140LiveListByParams(params);
         if (coinYoutubeYt2140LiveList.isEmpty()) {
-            log.warn("jsoupYoutubeLive4yt2140Chapter 结束：coinYoutubeYt2140LiveList is empty");
+            XxlJobHelper.log("jsoupYoutubeLive4yt2140Chapter 结束：coinYoutubeYt2140LiveList is empty");
             return;
         }
 
@@ -159,12 +158,12 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
                 failCount++;
             } finally {
                 coinYoutubeMapper.insertOrUpdateYt2140Live(coinYoutubeYt2140Live);
-                log.warn("jsoupYoutubeLive4yt2140Chapter update status：originalUrl={},status={}", coinYoutubeYt2140Live.getOriginalUrl(), coinYoutubeYt2140Live.getStatus());
+                XxlJobHelper.log("jsoupYoutubeLive4yt2140Chapter update status：originalUrl={},status={}", coinYoutubeYt2140Live.getOriginalUrl(), coinYoutubeYt2140Live.getStatus());
             }
         }
 
         stopWatch.stop();
-        log.warn("jsoupFulibaPicDetail 结束：totalSize={},successCount={},failCount={},time={}", coinYoutubeYt2140LiveList.size(), successCount, failCount, stopWatch.getTotalTimeMillis());
+        XxlJobHelper.log("jsoupYoutubeLive4yt2140Chapter结束==============================：totalSize={},successCount={},failCount={},time={}", coinYoutubeYt2140LiveList.size(), successCount, failCount, stopWatch.getTotalTimeMillis());
     }
 
     @Override
@@ -172,8 +171,6 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
     public void jsoupYoutubeLive4yt2140ChapterSub(CoinYoutubeYt2140Live coinYoutubeYt2140Live) {
         List<CoinYoutubeYt2140LiveChapter> yt2140LiveChapterList = new ArrayList<>();
         try {
-            //log.warn("jsoupYoutubeLive4yt2140ChapterSub start：originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
-
             String videoDetailsJson = getVideoDetails(coinYoutubeYt2140Live.getOriginalUrl());
             JSONObject result = JSONObject.parseObject(videoDetailsJson);
 
@@ -183,11 +180,11 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
 
             JSONArray chapterArr = result.getJSONArray("chapters");
             if (chapterArr == null || chapterArr.isEmpty()) {
-                if (coinYoutubeYt2140Live.getPublishTime().plusDays(7).isBefore(LocalDate.now())){
-                    log.warn("jsoupYoutubeLive4yt2140ChapterSub chapters is null or empty，no chapters，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
+                if (coinYoutubeYt2140Live.getPublishTime().plusDays(7).isBefore(LocalDate.now())) {
+                    XxlJobHelper.log("jsoupYoutubeLive4yt2140ChapterSub chapters is null or empty，no chapters，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
                     throw new BusinessException(ResultCodeEnum.UNKNOWN_ERROR.getCode(), "chapters is null or empty，no chapters");
-                }else{
-                    log.warn("jsoupYoutubeLive4yt2140ChapterSub chapters is null or empty，not updated，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
+                } else {
+                    XxlJobHelper.log("jsoupYoutubeLive4yt2140ChapterSub chapters is null or empty，not updated，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
                     throw new BusinessException(ResultCodeEnum.UNKNOWN_ERROR.getCode(), "chapters is null or empty，not updated");
                 }
             }
@@ -211,6 +208,8 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
         } catch (Exception e) {
             if (!(e instanceof BusinessException)) {
                 log.error("jsoupYoutubeLive4yt2140ChapterSub exception，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl(), e);
+                XxlJobHelper.log("jsoupYoutubeLive4yt2140ChapterSub exception，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
+                XxlJobHelper.log(e);
             }
             throw new BusinessException(30000, e.getMessage());
         }
@@ -224,7 +223,7 @@ public class CoinYoutubeServiceImpl implements CoinYoutubeService {
         }
 
         // 构建命令：yt-dlp --dump-json 获取视频元数据（JSON格式）
-        String command = "yt-dlp " + proxyStr + " --dump-json " + videoUrl;
+        String command = "/usr/local/bin/yt-dlp " + proxyStr + " --dump-json " + videoUrl;
 
         Process process = Runtime.getRuntime().exec(command);
         BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));

src/main/java/top/lvzhiqiang/service/impl/Crawler4JavdbServiceImpl.java

@@ -72,7 +72,7 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
             if ("dev".equals(env)) {
                 proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 1080));
             } else {
-                proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 10808));
+                proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 10808));
             }
         }
     }
@@ -164,7 +164,7 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
         Document document;
         Elements itembSelects;
         for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
-            XxlJobHelper.log("videoMonitorActors start，symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
+            XxlJobHelper.log("monitorFavorites start，symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
             Map<String, String> unIcodePoolMap = new HashMap<>();
             List<String> insertCodeList = new ArrayList<>();
             try {
@@ -211,6 +211,8 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
 
                 // jsoupIcodePool
                 bgService.jsoupIcodePool("javdb", 1, 2, 2);
+
+                XxlJobHelper.log("monitorFavorites end，symbol={},actorsRemark={},size={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark(), insertCodeList.size());
             } catch (Exception e) {
                 XxlJobHelper.log("videoMonitorActors exception， symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
                 XxlJobHelper.log(e);

src/test/java/top/lvzhiqiang/TestCoin.java

@@ -20,7 +20,10 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.time.LocalDateTime;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * 单元测试类
@@ -107,7 +110,7 @@ public class TestCoin {
 
     @Test
     public void testJsoupYoutubeLive4yt2140Chapter4No() throws IOException, InterruptedException {
-        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(3, "", null);
+        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(3, "", null, "test");
     }
 
     @Test