|
|
@@ -1,6 +1,7 @@
|
|
|
package top.lvzhiqiang.service.impl;
|
|
|
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.xxl.job.core.context.XxlJobHelper;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import me.chanjar.weixin.common.error.WxErrorException;
|
|
|
import me.chanjar.weixin.cp.api.impl.WxCpServiceImpl;
|
|
|
@@ -29,7 +30,6 @@ import java.util.List;
|
|
|
import java.util.Map;
|
|
|
import java.util.concurrent.ScheduledExecutorService;
|
|
|
import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
/**
|
|
|
* Crawler Javdb ServiceImpl
|
|
|
@@ -79,127 +79,145 @@ public class Crawler4JavdbServiceImpl implements Crawler4JavdbService {
|
|
|
|
|
|
@Override
|
|
|
public void monitorActors() {
|
|
|
- scheduler.scheduleAtFixedRate(() -> {
|
|
|
- log.warn("monitorActors开始==============================");
|
|
|
- beforeJavbus();
|
|
|
-
|
|
|
- String website = "javdb";
|
|
|
- List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website, 1);
|
|
|
-
|
|
|
- Document document;
|
|
|
- Elements itembSelects;
|
|
|
- int findSize = 0;
|
|
|
- for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
|
|
|
- try {
|
|
|
- List<String> insertCodeList = new ArrayList<>();
|
|
|
- List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol(), 1);
|
|
|
-
|
|
|
- Map<String, String> headerMap = new HashMap<>();
|
|
|
- headerMap.put("referer", videoMonitorActors.getHttpUrl());
|
|
|
- document = JsoupUtil.requestDocument(videoMonitorActors.getHttpUrl(), JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
|
|
|
- itembSelects = document.select("div.movie-list").select("div.item");
|
|
|
-
|
|
|
- for (Element itembSelect : itembSelects) {
|
|
|
- String code = itembSelect.select("a.box").get(0).select("div.video-title").select("strong").text().trim().toUpperCase();
|
|
|
- if (codeList.contains(code)) {
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- String identificationCode = videoMonitorActorsMapper.findAllLogListByCode(code, 1);
|
|
|
- if (StringUtils.isNotEmpty(identificationCode)) {
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- insertCodeList.add(code);
|
|
|
-
|
|
|
- String url = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
- String title = itembSelect.select("a.box").get(0).attr("title");
|
|
|
- String score = itembSelect.select("a.box").get(0).select("div.score").text().replace(" ", "").trim();
|
|
|
- String date = itembSelect.select("a.box").get(0).select("div.meta").text().trim();
|
|
|
- String tags = itembSelect.select("a.box").get(0).select("div.tags").text().trim();
|
|
|
-
|
|
|
- String content = "<div class=\"highlight\">演员:" + videoMonitorActors.getActorsRemark() + "</div>" +
|
|
|
- "<div>识别码:" + code + "</div>" +
|
|
|
- "<div>标题:" + title + "</div>" +
|
|
|
- "<div>发行日期:" + date + "</div>" +
|
|
|
- "<div class=\"gray\">评价:" + score + "</div>" +
|
|
|
- "<div class=\"gray\">标签:" + tags + "</div>";
|
|
|
-
|
|
|
- JSONObject params = new JSONObject();
|
|
|
- params.put("title", "JAVDB演员监控报警");
|
|
|
- params.put("logUrl", url);
|
|
|
- params.put("btnTxt", "影片详情");
|
|
|
- monitorAlarm4APP_TEXT_CARD(content, params);
|
|
|
-
|
|
|
- Thread.sleep(2000);
|
|
|
+ XxlJobHelper.log("monitorActors开始==============================");
|
|
|
+ beforeJavbus();
|
|
|
+
|
|
|
+ String website = "javdb";
|
|
|
+ List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website, 1);
|
|
|
+ XxlJobHelper.log("videoMonitorActorsList,website={},size={}", website, videoMonitorActorsList.size());
|
|
|
+
|
|
|
+ Document document;
|
|
|
+ Elements itembSelects;
|
|
|
+ int findSize = 0;
|
|
|
+ for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors start,symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
|
|
|
+ try {
|
|
|
+ List<String> insertCodeList = new ArrayList<>();
|
|
|
+ List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol(), 1);
|
|
|
+
|
|
|
+ Map<String, String> headerMap = new HashMap<>();
|
|
|
+ headerMap.put("referer", videoMonitorActors.getHttpUrl());
|
|
|
+ document = JsoupUtil.requestDocument(videoMonitorActors.getHttpUrl(), JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
|
|
|
+ itembSelects = document.select("div.movie-list").select("div.item");
|
|
|
+
|
|
|
+ for (Element itembSelect : itembSelects) {
|
|
|
+ String code = itembSelect.select("a.box").get(0).select("div.video-title").select("strong").text().trim().toUpperCase();
|
|
|
+ if (codeList.contains(code)) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors findAllLogListBySymbol code={} continue", code);
|
|
|
+ continue;
|
|
|
}
|
|
|
|
|
|
- if (insertCodeList.size() > 0) {
|
|
|
- findSize += insertCodeList.size();
|
|
|
- videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol(), 1);
|
|
|
+ String identificationCode = videoMonitorActorsMapper.findAllLogListByCode(code, 1);
|
|
|
+ if (StringUtils.isNotEmpty(identificationCode)) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors findAllLogListByCode code={} continue", code);
|
|
|
+ continue;
|
|
|
}
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+
|
|
|
+ insertCodeList.add(code);
|
|
|
+ XxlJobHelper.log("videoMonitorActors insertCodeList code={} add", code);
|
|
|
+
|
|
|
+ String url = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
+ String title = itembSelect.select("a.box").get(0).attr("title");
|
|
|
+ String score = itembSelect.select("a.box").get(0).select("div.score").text().replace(" ", "").trim();
|
|
|
+ String date = itembSelect.select("a.box").get(0).select("div.meta").text().trim();
|
|
|
+ String tags = itembSelect.select("a.box").get(0).select("div.tags").text().trim();
|
|
|
+
|
|
|
+ String content = "<div class=\"highlight\">演员:" + videoMonitorActors.getActorsRemark() + "</div>" +
|
|
|
+ "<div>识别码:" + code + "</div>" +
|
|
|
+ "<div>标题:" + title + "</div>" +
|
|
|
+ "<div>发行日期:" + date + "</div>" +
|
|
|
+ "<div class=\"gray\">评价:" + score + "</div>" +
|
|
|
+ "<div class=\"gray\">标签:" + tags + "</div>";
|
|
|
+
|
|
|
+ JSONObject params = new JSONObject();
|
|
|
+ params.put("title", "JAVDB演员监控报警");
|
|
|
+ params.put("logUrl", url);
|
|
|
+ params.put("btnTxt", "影片详情");
|
|
|
+ monitorAlarm4APP_TEXT_CARD(content, params);
|
|
|
+
|
|
|
+ Thread.sleep(2000);
|
|
|
}
|
|
|
+
|
|
|
+ if (!insertCodeList.isEmpty()) {
|
|
|
+ findSize += insertCodeList.size();
|
|
|
+ videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol(), 1);
|
|
|
+ }
|
|
|
+ XxlJobHelper.log("videoMonitorActors end,symbol={},actorsRemark={},size={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark(), insertCodeList.size());
|
|
|
+ } catch (Exception e) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors exception,symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
|
|
|
+ XxlJobHelper.log(e);
|
|
|
}
|
|
|
- log.warn("monitorActors结束==============================findSize={}", findSize);
|
|
|
- }, 0, 2, TimeUnit.HOURS);
|
|
|
+ }
|
|
|
|
|
|
- scheduler.scheduleAtFixedRate(() -> {
|
|
|
- log.warn("monitorActors4Own开始==============================");
|
|
|
- beforeJavbus();
|
|
|
+ XxlJobHelper.log("monitorActors结束==============================findSize={}", findSize);
|
|
|
+ }
|
|
|
|
|
|
- String website = "javdb";
|
|
|
- List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website, 2);
|
|
|
+ @Override
|
|
|
+ public void monitorFavorites() {
|
|
|
+ XxlJobHelper.log("monitorFavorites开始==============================");
|
|
|
+ beforeJavbus();
|
|
|
+
|
|
|
+ String website = "javdb";
|
|
|
+ List<VideoMonitorActors> videoMonitorActorsList = videoMonitorActorsMapper.findAllListByWebsite(website, 2);
|
|
|
+ XxlJobHelper.log("videoMonitorActorsList,website={},size={}", website, videoMonitorActorsList.size());
|
|
|
+
|
|
|
+ Document document;
|
|
|
+ Elements itembSelects;
|
|
|
+ for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors start,symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
|
|
|
+ Map<String, String> unIcodePoolMap = new HashMap<>();
|
|
|
+ List<String> insertCodeList = new ArrayList<>();
|
|
|
+ try {
|
|
|
+ List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol(), 2);
|
|
|
+ Map<String, String> headerMap = new HashMap<>();
|
|
|
+ headerMap.put("referer", videoMonitorActors.getHttpUrl());
|
|
|
+ document = JsoupUtil.requestDocument(videoMonitorActors.getHttpUrl(), JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
|
|
|
+ itembSelects = document.select("div.movie-list").select("div.item");
|
|
|
+
|
|
|
+ for (Element itembSelect : itembSelects) {
|
|
|
+ String code = itembSelect.select("a.box").get(0).select("div.video-title").select("strong").text().trim().toUpperCase();
|
|
|
+ if (codeList.contains(code)) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors findAllLogListBySymbol code={} break", code);
|
|
|
+ break;
|
|
|
+ }
|
|
|
|
|
|
- Document document;
|
|
|
- Elements itembSelects;
|
|
|
- for (VideoMonitorActors videoMonitorActors : videoMonitorActorsList) {
|
|
|
- Map<String, String> unIcodePoolMap = new HashMap<>();
|
|
|
- List<String> insertCodeList = new ArrayList<>();
|
|
|
- try {
|
|
|
- List<String> codeList = videoMonitorActorsMapper.findAllLogListBySymbol(videoMonitorActors.getSymbol(), 2);
|
|
|
- Map<String, String> headerMap = new HashMap<>();
|
|
|
- headerMap.put("referer", videoMonitorActors.getHttpUrl());
|
|
|
- document = JsoupUtil.requestDocument(videoMonitorActors.getHttpUrl(), JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
|
|
|
- itembSelects = document.select("div.movie-list").select("div.item");
|
|
|
-
|
|
|
- for (Element itembSelect : itembSelects) {
|
|
|
- String code = itembSelect.select("a.box").get(0).select("div.video-title").select("strong").text().trim().toUpperCase();
|
|
|
- if (codeList.contains(code)) {
|
|
|
- break;
|
|
|
- }
|
|
|
-
|
|
|
- Integer exist = icodePoolMapper.existIcodePoolByCode(code);
|
|
|
- if (exist != null) {
|
|
|
- insertCodeList.add(code);
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- String url = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
- unIcodePoolMap.put(code, url);
|
|
|
+ Integer exist = icodePoolMapper.existIcodePoolByCode(code);
|
|
|
+ if (exist != null) {
|
|
|
insertCodeList.add(code);
|
|
|
+ XxlJobHelper.log("videoMonitorActors existIcodePoolByCode code={} continue", code);
|
|
|
|
|
|
- Thread.sleep(2000);
|
|
|
+ continue;
|
|
|
}
|
|
|
|
|
|
- // insert to IcodePool
|
|
|
- if (unIcodePoolMap.size() > 0) {
|
|
|
- unIcodePoolMap.forEach((k, v) -> icodePoolMapper.insert(k, v));
|
|
|
- }
|
|
|
- // insert to video_monitor_actors_log
|
|
|
- if (insertCodeList.size() > 0) {
|
|
|
- videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol(), 2);
|
|
|
- }
|
|
|
- // jsoupIcodePool
|
|
|
- bgService.jsoupIcodePool("javdb", 1, 2, 2);
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+ String url = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
+ unIcodePoolMap.put(code, url);
|
|
|
+ insertCodeList.add(code);
|
|
|
+ XxlJobHelper.log("videoMonitorActors insertCodeList code={} add", code);
|
|
|
+
|
|
|
+ Thread.sleep(2000);
|
|
|
+ }
|
|
|
+
|
|
|
+ // insert to IcodePool
|
|
|
+ if (!unIcodePoolMap.isEmpty()) {
|
|
|
+ unIcodePoolMap.forEach((k, v) -> icodePoolMapper.insert(k, v));
|
|
|
+ }
|
|
|
+ XxlJobHelper.log("monitorFavorites icodePoolMapper.insert size={}", unIcodePoolMap.size());
|
|
|
+
|
|
|
+ // insert to video_monitor_actors_log
|
|
|
+ if (!insertCodeList.isEmpty()) {
|
|
|
+ videoMonitorActorsMapper.insertLogList(insertCodeList, videoMonitorActors.getSymbol(), 2);
|
|
|
}
|
|
|
+ XxlJobHelper.log("monitorFavorites videoMonitorActorsMapper.insertLogList size={}", insertCodeList.size());
|
|
|
+
|
|
|
+ // jsoupIcodePool
|
|
|
+ bgService.jsoupIcodePool("javdb", 1, 2, 2);
|
|
|
+ } catch (Exception e) {
|
|
|
+ XxlJobHelper.log("videoMonitorActors exception, symbol={},actorsRemark={}", videoMonitorActors.getSymbol(), videoMonitorActors.getActorsRemark());
|
|
|
+ XxlJobHelper.log(e);
|
|
|
}
|
|
|
+ }
|
|
|
|
|
|
- log.warn("monitorActors4Own结束==============================");
|
|
|
- }, 0, 3, TimeUnit.HOURS);
|
|
|
+ XxlJobHelper.log("monitorFavorites结束==============================");
|
|
|
}
|
|
|
|
|
|
@Override
|