add:jsoup youtube live v1

src/main/java/top/lvzhiqiang/config/MyCoinJobs.java

@@ -4,7 +4,8 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 import top.lvzhiqiang.service.CoinService;
-import top.lvzhiqiang.service.impl.CoinService2;
+import top.lvzhiqiang.service.CoinService2;
+import top.lvzhiqiang.service.CoinYoutubeService;
 import top.lvzhiqiang.util.DateUtils;
 
 import javax.annotation.Resource;
@@ -25,6 +26,9 @@ public class MyCoinJobs {
     @Resource
     private CoinService2 coinService2;
 
+    @Resource
+    private CoinYoutubeService coinYoutubeService;
+
     private static final String SCHEDULED_ZONE = "Asia/Shanghai";
 
     /**
@@ -77,4 +81,34 @@ public class MyCoinJobs {
         } catch (Exception e) {
         }
     }
+
+    /**
+     * @throws Exception
+     */
+    @Scheduled(cron = "0 0 1 * * ?", zone = SCHEDULED_ZONE)
+    public void jsoupYoutubeLive4yt2140() throws Exception {
+        log.warn("jsoupYoutubeLive4yt2140开始==============================");
+
+        coinYoutubeService.jsoupYoutubeLive4yt2140();
+    }
+
+    /**
+     * @throws Exception
+     */
+    @Scheduled(cron = "0 30 1 * * ?", zone = SCHEDULED_ZONE)
+    public void jsoupYoutubeLive4yt2140Chapter4No() throws Exception {
+        log.warn("jsoupYoutubeLive4yt2140Chapter4No开始==============================");
+
+        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(3, null, null);
+    }
+
+    /**
+     * @throws Exception
+     */
+    @Scheduled(cron = "0 30 2 * * ?", zone = SCHEDULED_ZONE)
+    public void jsoupYoutubeLive4yt2140Chapter4Fail() throws Exception {
+        log.warn("jsoupYoutubeLive4yt2140Chapter4Fail开始==============================");
+
+        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(2, null, null);
+    }
 }

src/main/java/top/lvzhiqiang/entity/CoinYoutubeYt2140Live.java

@@ -0,0 +1,71 @@
+package top.lvzhiqiang.entity;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+
+/**
+ * 币-油管-2140加密社群-直播表
+ *
+ * @author lvzhiqiang
+ * 2025/5/15 10:50
+ */
+@Data
+public class CoinYoutubeYt2140Live implements Serializable {
+
+    /**
+     * 主键
+     */
+    private String id;
+
+    /**
+     * 主URL
+     */
+    private String originalUrl;
+
+    /**
+     * 主标题
+     */
+    private String fullTitle;
+
+    /**
+     * 状态（4:忽略，3：待爬取，1：成功，2：失败）
+     */
+    private Integer status;
+
+    /**
+     * 失败原因
+     */
+    private String failureCause;
+
+    /**
+     * 发布时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd", timezone = "GMT+8")
+    private LocalDate publishTime;
+
+    /**
+     * 时长
+     */
+    private String duration;
+
+    /**
+     * 章节数
+     */
+    private Integer chapterCount;
+
+    /**
+     * 创建时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private LocalDateTime createTime;
+
+    /**
+     * 最后修改时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private LocalDateTime modifyTime;
+}

src/main/java/top/lvzhiqiang/entity/CoinYoutubeYt2140LiveChapter.java

@@ -0,0 +1,55 @@
+package top.lvzhiqiang.entity;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+/**
+ * 币-油管-2140加密社群-直播表
+ *
+ * @author lvzhiqiang
+ * 2025/5/15 10:50
+ */
+@Data
+public class CoinYoutubeYt2140LiveChapter implements Serializable {
+
+    /**
+     * 主键
+     */
+    private Long id;
+
+    /**
+     * 章节标题
+     */
+    private String chapterTitle;
+
+    /**
+     * LIVE id
+     */
+    private String liveId;
+
+    /**
+     * start_time
+     */
+    private Integer startTime;
+
+    /**
+     * end_time
+     */
+    private Integer endTime;
+
+    /**
+     * 备注
+     */
+    private String remark;
+
+    private Integer sort;
+
+    /**
+     * 最后修改时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private LocalDateTime modifyTime;
+}

src/main/java/top/lvzhiqiang/mapper/CoinYoutubeMapper.java

@@ -0,0 +1,54 @@
+package top.lvzhiqiang.mapper;
+
+import org.apache.ibatis.annotations.Insert;
+import org.apache.ibatis.annotations.Select;
+import top.lvzhiqiang.entity.CoinYoutubeYt2140Live;
+import top.lvzhiqiang.entity.CoinYoutubeYt2140LiveChapter;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Coin youtube mapper
+ *
+ * @author lvzhiqiang
+ * 2025/5/14 18:28
+ */
+public interface CoinYoutubeMapper {
+
+    @Select("SELECT * FROM coin_youtube_yt2140_live WHERE publish_time is not null and delete_flag = 1 order by publish_time desc limit 1")
+    CoinYoutubeYt2140Live findLatestYt2140Live();
+
+    @Insert("INSERT ignore INTO coin_youtube_yt2140_live(id, original_url, full_title, status, failure_cause, publish_time, create_time, modify_time) " +
+            "VALUES (#{id}, #{originalUrl}, #{fullTitle}, #{status}, #{failureCause}, #{publishTime}, now(), now())")
+    int insertIgnoreYt2140Live(CoinYoutubeYt2140Live live);
+
+    @Select({"<script>" +
+            "select * from coin_youtube_yt2140_live WHERE delete_flag = 1" +
+            "<if test=\"id != null and id != ''\">" +
+            "   and id = #{id}" +
+            "</if>" +
+            "<if test=\"originalUrl != null and originalUrl != ''\">" +
+            "   and original_url = #{originalUrl}" +
+            "</if>" +
+            "<if test=\"status != null\">" +
+            "   and status = #{status}" +
+            "</if>" +
+            " order by publish_time asc" +
+            "</script>"})
+    List<CoinYoutubeYt2140Live> findJsoupYt2140LiveListByParams(Map<String, Object> params);
+
+    @Insert({"<script>" +
+            "INSERT ignore INTO coin_youtube_yt2140_live_chapter(chapter_title, live_id, start_time, end_time, sort, modify_time) " +
+            " VALUES " +
+            "<foreach collection='list' item='p' index=\"index\" separator=\",\">" +
+            "   (#{p.chapterTitle}, #{p.liveId}, #{p.startTime}, #{p.endTime}, #{p.sort}, now())" +
+            "</foreach>" +
+            "</script>"})
+    int insertIgnoreYt2140LiveChapterList(List<CoinYoutubeYt2140LiveChapter> yt2140LiveChapterList);
+
+    @Insert("INSERT INTO coin_youtube_yt2140_live(id, original_url, full_title, status, failure_cause, publish_time, duration, chapter_count, create_time, modify_time) " +
+            "VALUES (#{id}, #{originalUrl}, #{fullTitle}, #{status}, #{failureCause}, #{publishTime}, #{duration}, #{chapterCount}, now(), now()) " +
+            "ON DUPLICATE KEY UPDATE full_title=values(full_title),status=values(status),failure_cause=values(failure_cause),publish_time=values(publish_time),duration=values(duration),chapter_count=values(chapter_count),modify_time=now()")
+    void insertOrUpdateYt2140Live(CoinYoutubeYt2140Live coinYoutubeYt2140Live);
+}

src/main/java/top/lvzhiqiang/service/impl/CoinService2.java → src/main/java/top/lvzhiqiang/service/CoinService2.java

@@ -1,4 +1,4 @@
-package top.lvzhiqiang.service.impl;
+package top.lvzhiqiang.service;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;

src/main/java/top/lvzhiqiang/service/CoinYoutubeService.java

@@ -0,0 +1,25 @@
+package top.lvzhiqiang.service;
+
+import top.lvzhiqiang.entity.CoinYoutubeYt2140Live;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Coin youtube Service
+ *
+ * @author lvzhiqiang
+ * 2025/5/14 18:28
+ */
+public interface CoinYoutubeService {
+
+    void jsoupYoutubeLive4yt2140() throws IOException, InterruptedException;
+
+    List<String> getChannelLiveStreams(String channelUrl) throws IOException, InterruptedException;
+
+    void jsoupYoutubeLive4yt2140Chapter(Integer status, String originalUrl, String id);
+
+    void jsoupYoutubeLive4yt2140ChapterSub(CoinYoutubeYt2140Live coinYoutubeYt2140Live);
+
+    String getVideoDetails(String videoUrl) throws IOException, InterruptedException;
+}

src/main/java/top/lvzhiqiang/service/impl/CoinService2Impl.java

@@ -12,6 +12,7 @@ import org.springframework.transaction.annotation.Transactional;
 import top.lvzhiqiang.entity.CoinBinanceOrderHistory;
 import top.lvzhiqiang.entity.CoinBinanceSymbol;
 import top.lvzhiqiang.mapper.CoinMapper;
+import top.lvzhiqiang.service.CoinService2;
 import top.lvzhiqiang.util.JsoupUtil;
 
 import javax.annotation.Resource;

src/main/java/top/lvzhiqiang/service/impl/CoinYoutubeServiceImpl.java

@@ -0,0 +1,244 @@
+package top.lvzhiqiang.service.impl;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StopWatch;
+import top.lvzhiqiang.config.InitRunner;
+import top.lvzhiqiang.entity.CoinYoutubeYt2140Live;
+import top.lvzhiqiang.entity.CoinYoutubeYt2140LiveChapter;
+import top.lvzhiqiang.enumeration.ResultCodeEnum;
+import top.lvzhiqiang.exception.BusinessException;
+import top.lvzhiqiang.mapper.CoinYoutubeMapper;
+import top.lvzhiqiang.service.CoinYoutubeService;
+import top.lvzhiqiang.util.DateUtils;
+import top.lvzhiqiang.util.SpringUtils;
+import top.lvzhiqiang.util.StringUtils;
+import top.lvzhiqiang.util.UUIDUtils;
+
+import javax.annotation.Resource;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Coin youtube Service
+ *
+ * @author lvzhiqiang
+ * 2025/5/15 10:50
+ */
+@Service
+@Slf4j
+public class CoinYoutubeServiceImpl implements CoinYoutubeService {
+    @Resource
+    private CoinYoutubeMapper coinYoutubeMapper;
+    @Value("${spring.profiles.active}")
+    private String env;
+
+    @Override
+    @Async("coinTaskExecutor")
+    public void jsoupYoutubeLive4yt2140() throws IOException, InterruptedException {
+        log.warn("jsoupYoutubeLive4yt2140 开始：");
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+
+        CoinYoutubeYt2140Live coinYoutubeYt2140Live = coinYoutubeMapper.findLatestYt2140Live();
+        String latestOriginalUrl;
+        if (coinYoutubeYt2140Live == null) {
+            latestOriginalUrl = "";
+        } else {
+            latestOriginalUrl = coinYoutubeYt2140Live.getOriginalUrl();
+        }
+
+        String youtubeYt2140LiveUrl = InitRunner.dicCodeMap.get("youtube_yt2140_live_url").getCodeValue();
+        List<String> liveStreams = getChannelLiveStreams(youtubeYt2140LiveUrl);
+
+        int findCount = 0;
+        for (String originalUrl : liveStreams) {
+            if (originalUrl.equals(latestOriginalUrl)) {
+                break;
+            }
+
+            CoinYoutubeYt2140Live live = new CoinYoutubeYt2140Live();
+            live.setId(UUIDUtils.getUUID());
+            live.setOriginalUrl(originalUrl);
+            live.setStatus(3);
+            int count = coinYoutubeMapper.insertIgnoreYt2140Live(live);
+            findCount += count;
+            log.warn("jsoupYoutubeLive4yt2140 live success：originalUrl={}，count={}", originalUrl, count);
+        }
+
+        stopWatch.stop();
+        log.warn("jsoupYoutubeLive4yt2140 结束：findCount={}，time={}", findCount, stopWatch.getTotalTimeMillis());
+    }
+
+    @Override
+    public List<String> getChannelLiveStreams(String channelUrl) throws IOException, InterruptedException {
+        List<String> liveUrls = new ArrayList<>();
+
+        String proxyStr = "";
+        if ("dev".equals(env)) {
+            proxyStr = "--proxy socks5://127.0.0.1:1081/";
+        }
+
+        // 构建命令
+        String command = "yt-dlp " + proxyStr + " --flat-playlist --print url " + channelUrl + "/streams";
+
+        Process process = Runtime.getRuntime().exec(command);
+        BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+
+        String line;
+        while ((line = reader.readLine()) != null) {
+            if (line.startsWith("https://www.youtube.com/watch")) {
+                liveUrls.add(line);
+            }
+        }
+
+        int exitCode = process.waitFor();
+        if (exitCode != 0) {
+            throw new RuntimeException("yt-dlp执行失败，退出码: " + exitCode);
+        }
+
+        return liveUrls;
+    }
+
+    @Override
+    //@Async("coinTaskExecutor")
+    public void jsoupYoutubeLive4yt2140Chapter(Integer status, String originalUrl, String id) {
+        log.warn("jsoupYoutubeLive4yt2140Chapter 开始：status={}", status);
+
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+
+        Map<String, Object> params = new HashMap<>();
+        if (StringUtils.isNotEmpty(id)) {
+            params.put("id", id);
+        }
+        if (StringUtils.isNotEmpty(originalUrl)) {
+            params.put("originalUrl", originalUrl);
+        }
+        if (status != null) {
+            params.put("status", status);
+        }
+
+        List<CoinYoutubeYt2140Live> coinYoutubeYt2140LiveList = coinYoutubeMapper.findJsoupYt2140LiveListByParams(params);
+        if (coinYoutubeYt2140LiveList.isEmpty()) {
+            log.warn("jsoupYoutubeLive4yt2140Chapter 结束：coinYoutubeYt2140LiveList is empty");
+            return;
+        }
+
+        int successCount = 0;
+        int failCount = 0;
+        for (CoinYoutubeYt2140Live coinYoutubeYt2140Live : coinYoutubeYt2140LiveList) {
+            try {
+                Thread.sleep(5000L);
+
+                SpringUtils.getBean(CoinYoutubeServiceImpl.class).jsoupYoutubeLive4yt2140ChapterSub(coinYoutubeYt2140Live);
+                coinYoutubeYt2140Live.setStatus(1);
+                successCount++;
+            } catch (Exception e) {
+                coinYoutubeYt2140Live.setFailureCause(e.getMessage().length() > 400 ? e.getMessage().substring(0, 400) : e.getMessage());
+
+                if (e.getMessage().contains("no chapters")) {
+                    coinYoutubeYt2140Live.setStatus(4);
+                } else {
+                    coinYoutubeYt2140Live.setStatus(2);
+                }
+
+                failCount++;
+            } finally {
+                coinYoutubeMapper.insertOrUpdateYt2140Live(coinYoutubeYt2140Live);
+                log.warn("jsoupYoutubeLive4yt2140Chapter update status：originalUrl={},status={}", coinYoutubeYt2140Live.getOriginalUrl(), coinYoutubeYt2140Live.getStatus());
+            }
+        }
+
+        stopWatch.stop();
+        log.warn("jsoupFulibaPicDetail 结束：totalSize={},successCount={},failCount={},time={}", coinYoutubeYt2140LiveList.size(), successCount, failCount, stopWatch.getTotalTimeMillis());
+    }
+
+    @Override
+    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
+    public void jsoupYoutubeLive4yt2140ChapterSub(CoinYoutubeYt2140Live coinYoutubeYt2140Live) {
+        List<CoinYoutubeYt2140LiveChapter> yt2140LiveChapterList = new ArrayList<>();
+        try {
+            //log.warn("jsoupYoutubeLive4yt2140ChapterSub start：originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
+
+            String videoDetailsJson = getVideoDetails(coinYoutubeYt2140Live.getOriginalUrl());
+            JSONObject result = JSONObject.parseObject(videoDetailsJson);
+
+            coinYoutubeYt2140Live.setFullTitle(result.getString("fulltitle"));
+            coinYoutubeYt2140Live.setPublishTime(LocalDate.parse(result.getString("release_date"), DateUtils.dateFormatter_));
+            coinYoutubeYt2140Live.setDuration(result.getString("duration_string"));
+
+            JSONArray chapterArr = result.getJSONArray("chapters");
+            if (chapterArr == null || chapterArr.isEmpty()) {
+                if (coinYoutubeYt2140Live.getPublishTime().plusDays(7).isBefore(LocalDate.now())){
+                    log.warn("jsoupYoutubeLive4yt2140ChapterSub chapters is null or empty，no chapters，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
+                    throw new BusinessException(ResultCodeEnum.UNKNOWN_ERROR.getCode(), "chapters is null or empty，no chapters");
+                }else{
+                    log.warn("jsoupYoutubeLive4yt2140ChapterSub chapters is null or empty，not updated，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl());
+                    throw new BusinessException(ResultCodeEnum.UNKNOWN_ERROR.getCode(), "chapters is null or empty，not updated");
+                }
+            }
+
+            coinYoutubeYt2140Live.setChapterCount(chapterArr.size());
+            for (int i = 0; i < chapterArr.size(); i++) {
+                JSONObject chapter = chapterArr.getJSONObject(i);
+
+                CoinYoutubeYt2140LiveChapter yt2140LiveChapter = new CoinYoutubeYt2140LiveChapter();
+                yt2140LiveChapter.setChapterTitle(chapter.getString("title"));
+                yt2140LiveChapter.setLiveId(coinYoutubeYt2140Live.getId());
+                yt2140LiveChapter.setStartTime(chapter.getInteger("start_time"));
+                yt2140LiveChapter.setEndTime(chapter.getInteger("end_time"));
+                yt2140LiveChapter.setSort(i + 1);
+
+                yt2140LiveChapterList.add(yt2140LiveChapter);
+            }
+
+            coinYoutubeMapper.insertIgnoreYt2140LiveChapterList(yt2140LiveChapterList);
+        } catch (Exception e) {
+            if (!(e instanceof BusinessException)) {
+                log.error("jsoupYoutubeLive4yt2140ChapterSub exception，originalUrl={}", coinYoutubeYt2140Live.getOriginalUrl(), e);
+            }
+            throw new BusinessException(30000, e.getMessage());
+        }
+    }
+
+    @Override
+    public String getVideoDetails(String videoUrl) throws IOException, InterruptedException {
+        String proxyStr = "";
+        if ("dev".equals(env)) {
+            proxyStr = "--proxy socks5://127.0.0.1:1081/";
+        }
+
+        // 构建命令：yt-dlp --dump-json 获取视频元数据（JSON格式）
+        String command = "yt-dlp " + proxyStr + " --dump-json " + videoUrl;
+
+        Process process = Runtime.getRuntime().exec(command);
+        BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));
+
+        StringBuilder jsonOutput = new StringBuilder();
+        String line;
+        while ((line = reader.readLine()) != null) {
+            jsonOutput.append(line);
+        }
+
+        int exitCode = process.waitFor();
+        if (exitCode != 0) {
+            throw new RuntimeException("yt-dlp 执行失败，退出码: " + exitCode);
+        }
+
+        return jsonOutput.toString(); // 返回 JSON 格式的视频详情
+    }
+}

src/test/java/top/lvzhiqiang/TestCoin.java

@@ -10,10 +10,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import top.lvzhiqiang.entity.CoinWatchlist;
 import top.lvzhiqiang.mapper.CoinMapper;
 import top.lvzhiqiang.service.CoinService;
-import top.lvzhiqiang.service.impl.CoinService2;
+import top.lvzhiqiang.service.CoinService2;
+import top.lvzhiqiang.service.CoinYoutubeService;
 import top.lvzhiqiang.util.DateUtils;
 
 import javax.annotation.Resource;
+import java.io.IOException;
 import java.time.LocalDateTime;
 import java.util.*;
 
@@ -40,6 +42,9 @@ public class TestCoin {
     @Resource
     private CoinMapper coinMapper;
 
+    @Resource
+    private CoinYoutubeService coinYoutubeService;
+
     @Test
     public void testSyncData() {
         LocalDateTime now = LocalDateTime.now();
@@ -89,6 +94,16 @@ public class TestCoin {
         coinService.debugTest();
     }
 
+    @Test
+    public void testJsoupYoutubeLive4yt2140() throws IOException, InterruptedException {
+        coinYoutubeService.jsoupYoutubeLive4yt2140();
+    }
+
+    @Test
+    public void testJsoupYoutubeLive4yt2140Chapter4No() throws IOException, InterruptedException {
+        coinYoutubeService.jsoupYoutubeLive4yt2140Chapter(3, "", null);
+    }
+
     public static void main(String[] args) {
         String s = "[{\"id\":\"dogecoin\",\"symbol\":\"doge\",\"name\":\"Dogecoin\",\"image\":\"https://assets.coingecko.com/coins/images/5/large/dogecoin.png?1696501409\",\"current_price\":0.091631,\"market_cap\":13047603006,\"market_cap_rank\":11,\"fully_diluted_valuation\":13047543490,\"total_volume\":460955017,\"high_24h\":0.094371,\"low_24h\":0.090707,\"price_change_24h\":-0.002739353491127444,\"price_change_percentage_24h\":-2.90276,\"market_cap_change_24h\":-405382366.76885605,\"market_cap_change_percentage_24h\":-3.01333,\"circulating_supply\":142498276383.705,\"total_supply\":142497626383.705,\"max_supply\":null,\"ath\":0.731578,\"ath_change_percentage\":-87.46751,\"ath_date\":\"2021-05-08T05:08:23.458Z\",\"atl\":8.69e-05,\"atl_change_percentage\":105401.73855,\"atl_date\":\"2015-05-06T00:00:00.000Z\",\"roi\":null,\"last_updated\":\"2024-01-03T08:51:38.584Z\"}]";
         JSONArray marketData = JSONArray.parseArray(s);