|
|
@@ -702,6 +702,261 @@ public class BgServiceImpl implements BgService {
|
|
|
log.warn("jsoupVideoInfoInfantry4javdb 统计:total={},success={}", videoInfoInfantryList.size(), j);
|
|
|
}
|
|
|
|
|
|
+ @Async
|
|
|
+ @Override
|
|
|
+ public void jsoupVideoInfoUncensored4javdb(Integer status, String url, String identificationCodeP) {
|
|
|
+ log.warn("jsoupVideoInfoUncensored4javdb 开始:status={}", status);
|
|
|
+
|
|
|
+ // 获取待抓取码列表
|
|
|
+ List<VideoInfoUncensored> videoInfoUncensoredList;
|
|
|
+ if (StringUtils.isEmpty(identificationCodeP)) {
|
|
|
+ videoInfoUncensoredList = videoInfoUncensoredMapper.findByCodeAndType(null, status, null);
|
|
|
+ if (videoInfoUncensoredList.size() == 0) {
|
|
|
+ log.warn("status={}的videoInfoUncensoredList为空", status);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ videoInfoUncensoredList = videoInfoUncensoredMapper.findByCodeAndType(identificationCodeP, status, null);
|
|
|
+ videoInfoUncensoredList.forEach(e -> {
|
|
|
+ e.setJavdbUrl(url);
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ List<DicCode> dicCodeList = dicCodeMapper.findAll();
|
|
|
+ Map<String, String> javdbConstantMap = dicCodeList.stream()
|
|
|
+ .filter(x -> x.getType() != null && 2 == x.getType() && x.getEnv().contains(env))
|
|
|
+ .collect(Collectors.toMap(DicCode::getCodeKey, DicCode::getCodeValue, (key1, key2) -> key1));
|
|
|
+
|
|
|
+ beforeProxy();
|
|
|
+
|
|
|
+ // 获取图片保存路径
|
|
|
+ String uncensoredPath = WebAppConfig.dicCodeList.stream().filter(x -> 1 == x.getType() && env.equals(x.getEnv()) && "apics_path".equals(x.getCodeKey())).findFirst().get().getCodeValue();
|
|
|
+
|
|
|
+ // 遍历
|
|
|
+ Document document;
|
|
|
+ Document javdbSearchDocument;
|
|
|
+ int j = 0;
|
|
|
+ for (int i = 0; i < videoInfoUncensoredList.size(); i++) {
|
|
|
+ long start = System.currentTimeMillis();
|
|
|
+ VideoInfoUncensored videoInfoUncensored = videoInfoUncensoredList.get(i);
|
|
|
+
|
|
|
+ int retryCount = 0;
|
|
|
+ while (retryCount <= 3) {
|
|
|
+ try {
|
|
|
+ Thread.sleep(3000);
|
|
|
+
|
|
|
+ String picTime;
|
|
|
+ Elements itembSelects;
|
|
|
+ boolean ignoreCodeFlag = false;
|
|
|
+ if (StringUtils.isNotEmpty(videoInfoUncensored.getJavdbUrl())) {
|
|
|
+ Map<String, String> headerMap = new HashMap<>();
|
|
|
+ headerMap.put("referer", url);
|
|
|
+ document = JsoupUtil.requestDocument(url, JsoupUtil.HTTP_GET, proxy, null, headerMap, null);
|
|
|
+ } else {
|
|
|
+ String searchUrl = javdbConstantMap.get("javdb").concat("search?q=").concat(videoInfoUncensored.getIdentificationCode()).concat("&f=all");
|
|
|
+ Map<String, String> header3Map = new HashMap<>();
|
|
|
+ header3Map.put("referer", searchUrl);
|
|
|
+
|
|
|
+ javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
|
|
|
+
|
|
|
+ itembSelects = javdbSearchDocument.select("div.movie-list").select("div.item");
|
|
|
+ if (itembSelects.size() == 0) {
|
|
|
+ String newName = videoInfoUncensored.getName();
|
|
|
+ searchUrl = javdbConstantMap.get("javdb").concat("search?q=").concat(newName).concat("&f=all");
|
|
|
+ javdbSearchDocument = JsoupUtil.requestDocument(searchUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
|
|
|
+ itembSelects = javdbSearchDocument.select("div.movie-list").select("div.item");
|
|
|
+
|
|
|
+ ignoreCodeFlag = true;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (itembSelects.size() == 0) {
|
|
|
+ throw new BusinessException(30000, "javdb search result null");
|
|
|
+ }
|
|
|
+
|
|
|
+ String title;
|
|
|
+ String code;
|
|
|
+ String codeUrl = null;
|
|
|
+ for (Element itembSelect : itembSelects) {
|
|
|
+ title = itembSelect.select("a.box").get(0).attr("title");
|
|
|
+ code = itembSelect.select("a.box").get(0).select("div.video-title").select("strong").text();
|
|
|
+ if (code.equalsIgnoreCase(videoInfoUncensored.getIdentificationCode())) {
|
|
|
+ codeUrl = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
+ break;
|
|
|
+ } else if (title.equalsIgnoreCase(videoInfoUncensored.getName()) && ignoreCodeFlag) {
|
|
|
+ codeUrl = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ String newName = videoInfoUncensored.getName().replace("●", "さ");
|
|
|
+ if (title.contains(newName)) {
|
|
|
+ codeUrl = itembSelect.select("a.box").get(0).attr("abs:href");
|
|
|
+ videoInfoUncensored.setName(newName);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (StringUtils.isEmpty(codeUrl)) {
|
|
|
+ throw new BusinessException(30000, "javdb search result mismatch");
|
|
|
+ }
|
|
|
+
|
|
|
+ document = JsoupUtil.requestDocument(codeUrl, JsoupUtil.HTTP_GET, proxy, null, header3Map, null);
|
|
|
+ videoInfoUncensored.setJavdbUrl(codeUrl);
|
|
|
+ }
|
|
|
+
|
|
|
+ picTime = parseDocument4Uncensored4Javdb(document, uncensoredPath, videoInfoUncensored);
|
|
|
+ videoInfoUncensored.setStatus(2);
|
|
|
+ videoInfoUncensoredMapper.updateJsoupInfoById(videoInfoUncensored);
|
|
|
+ videoInfoOtherMapper.insertOrUpdate(videoInfoUncensored.getIdentificationCode(), 2);
|
|
|
+
|
|
|
+ j++;
|
|
|
+
|
|
|
+ log.warn("jsoupVideoInfoUncensored4javdb success:i={},picTime={},time={},identificationCode={}", i, picTime, System.currentTimeMillis() - start, videoInfoUncensored.getIdentificationCode());
|
|
|
+ break;
|
|
|
+ } catch (Exception e) {
|
|
|
+ ++retryCount;
|
|
|
+
|
|
|
+ if (retryCount < 4) {
|
|
|
+ log.error("jsoupVideoInfoUncensored4javdb error重试:i={},retryCount={},time={},identificationCode={},javdbUrl={}", i, retryCount, System.currentTimeMillis() - start, videoInfoUncensored.getIdentificationCode(), videoInfoUncensored.getJavdbUrl(), e);
|
|
|
+ } else if (retryCount == 4) {
|
|
|
+ videoInfoUncensoredMapper.updateStatus(videoInfoUncensored.getIdentificationCode(), 3);
|
|
|
+ log.error("jsoupVideoInfoUncensored4javdb error:i={},time={},identificationCode={},javdbsUrl={}", i, System.currentTimeMillis() - start, videoInfoUncensored.getIdentificationCode(), videoInfoUncensored.getJavdbUrl(), e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ log.warn("jsoupVideoInfoUncensored4javdb 统计:total={},success={}", videoInfoUncensoredList.size(), j);
|
|
|
+ }
|
|
|
+
|
|
|
+ private String parseDocument4Uncensored4Javdb(Document document, String uncensoredPath, VideoInfoUncensored videoInfoUncensored) throws Exception {
|
|
|
+ Elements container = document.select("section.section > div.container");
|
|
|
+ if (container.size() == 0) {
|
|
|
+ throw new BusinessException(30000, "番号无效!");
|
|
|
+ }
|
|
|
+
|
|
|
+ Elements videoDetail = container.select("div.video-detail");
|
|
|
+ // 名称
|
|
|
+ videoInfoUncensored.setName(videoDetail.select("h2.title").select("strong.current-title").text().trim());
|
|
|
+
|
|
|
+ Elements moviePanelInfos = videoDetail.select("nav.movie-panel-info");
|
|
|
+ Element pEle = moviePanelInfos.get(0);
|
|
|
+ // 识别码
|
|
|
+ String iCode = pEle.select("div:contains(番號)").select("span.value").first().text().replace(" ", "").replaceAll("\\s+", "");
|
|
|
+ if (!videoInfoUncensored.getIdentificationCode().equalsIgnoreCase(iCode)) {
|
|
|
+ throw new Exception("番号与站点不一致");
|
|
|
+ }
|
|
|
+ // 发行日期
|
|
|
+ String issueDate = pEle.select("div:contains(日期)").select("span.value").first().text().replace(" ", "").replaceAll("\\s+", "");
|
|
|
+ videoInfoUncensored.setIssueDate(LocalDate.parse(issueDate, DateUtils.dateFormatter));
|
|
|
+ // 长度
|
|
|
+ String length = pEle.select("div:contains(時長)").select("span.value").first().text().replace(" ", "").replaceAll("\\s+", "");
|
|
|
+ videoInfoUncensored.setLength(length);
|
|
|
+ // 导演
|
|
|
+ Elements directorEles = pEle.select("div:contains(導演)").select("span.value");
|
|
|
+ if (directorEles.size() > 0) {
|
|
|
+ videoInfoUncensored.setDirector(directorEles.first().select("a[href]").first().text().replace(" ", "").replaceAll("\\s+", ""));
|
|
|
+ }
|
|
|
+ // 制作商
|
|
|
+ Elements markerEles = pEle.select("div:contains(片商)").select("span.value");
|
|
|
+ if (markerEles.size() > 0) {
|
|
|
+ videoInfoUncensored.setMaker(markerEles.first().select("a[href]").first().text().replace(" ", "").replaceAll("\\s+", ""));
|
|
|
+ }
|
|
|
+ // 发行商
|
|
|
+ Elements issuerEles = pEle.select("div:contains(發行)").select("span.value");
|
|
|
+ if (issuerEles.size() > 0) {
|
|
|
+ videoInfoUncensored.setIssuer(issuerEles.first().select("a[href]").first().text().replace(" ", "").replaceAll("\\s+", ""));
|
|
|
+ }
|
|
|
+ // 类别
|
|
|
+ Elements genresEles = pEle.select("div:contains(類別)").select("span.value");
|
|
|
+ if (genresEles.size() > 0) {
|
|
|
+ StringBuffer sb = new StringBuffer();
|
|
|
+ Elements ahrefEles = genresEles.first().select("a[href]");
|
|
|
+ for (Element ahrefEle : ahrefEles) {
|
|
|
+ sb.append(ahrefEle.text().replace(" ", "").replaceAll("\\s+", "")).append(",");
|
|
|
+ }
|
|
|
+ if (sb.length() > 0) {
|
|
|
+ sb = sb.deleteCharAt(sb.length() - 1);
|
|
|
+ }
|
|
|
+ videoInfoUncensored.setGenres(sb.toString());
|
|
|
+ }
|
|
|
+ // 演员
|
|
|
+ Elements castEles = pEle.select("div:contains(演員)").select("span.value");
|
|
|
+ if (castEles.size() > 0) {
|
|
|
+ StringBuffer sb = new StringBuffer();
|
|
|
+ Elements ahrefEles = castEles.first().select("a[href]");
|
|
|
+ for (Element ahrefEle : ahrefEles) {
|
|
|
+ sb.append(ahrefEle.text().replace(" ", "").replaceAll("\\s+", "")).append(",");
|
|
|
+ }
|
|
|
+ if (sb.length() > 0) {
|
|
|
+ sb = sb.deleteCharAt(sb.length() - 1);
|
|
|
+ }
|
|
|
+ videoInfoUncensored.setCast(sb.toString());
|
|
|
+ }
|
|
|
+
|
|
|
+ // 图片 流出
|
|
|
+ String liuchus = "流出/".concat(videoInfoUncensored.getMainWho()).concat("/").concat(iCode);
|
|
|
+ StringBuffer picTime = new StringBuffer("{cover:");
|
|
|
+
|
|
|
+ // 图片URL bigImage
|
|
|
+ String liuchuCover = liuchus.concat("/cover/");
|
|
|
+
|
|
|
+ String fileName = issueDate.concat(" ").concat(videoInfoUncensored.getIdentificationCode()).concat(" ").concat(StringUtils.escapeJavParam(videoInfoUncensored.getName()));
|
|
|
+ byte[] imgUrlBytes = fileName.getBytes(StandardCharsets.UTF_8);
|
|
|
+ if (imgUrlBytes.length > 251) {
|
|
|
+ byte[] imgUrlDestBytes = new byte[251];
|
|
|
+ System.arraycopy(imgUrlBytes, 0, imgUrlDestBytes, 0, 251);
|
|
|
+ fileName = new String(imgUrlDestBytes, StandardCharsets.UTF_8).replace("�", "");
|
|
|
+ }
|
|
|
+ fileName = fileName.concat(".jpg");
|
|
|
+
|
|
|
+ if (!new File(uncensoredPath.concat(liuchuCover), fileName).exists()) {
|
|
|
+ Elements videoMetaPanel = videoDetail.select("div.column-video-cover");
|
|
|
+ String href = videoMetaPanel.select("a > img").first().attr("src");
|
|
|
+
|
|
|
+ long start = System.currentTimeMillis();
|
|
|
+ Connection.Response response = Jsoup.connect(href).method(Connection.Method.GET).ignoreContentType(true).timeout(50 * 1000).execute();
|
|
|
+
|
|
|
+ saveFile2(response.bodyStream(), uncensoredPath.concat(liuchuCover), fileName);
|
|
|
+ picTime.append(System.currentTimeMillis() - start).append(",");
|
|
|
+ }
|
|
|
+
|
|
|
+ videoInfoUncensored.setImgUrl(iCode.concat("/cover/").concat(fileName));
|
|
|
+
|
|
|
+ // 图片URL img_gf
|
|
|
+ String liuchuImgGF = liuchus.concat("/img_gf/");
|
|
|
+
|
|
|
+ File liuchuImgGFFile = new File(uncensoredPath.concat(liuchuImgGF));
|
|
|
+ if (!liuchuImgGFFile.exists() || liuchuImgGFFile.listFiles().length == 0) {
|
|
|
+ Elements sampleBoxEles = videoDetail.select("div.preview-images").select("a.tile-item");
|
|
|
+ long start2 = System.currentTimeMillis();
|
|
|
+ if (sampleBoxEles.size() > 0) {
|
|
|
+ Connection.Response responseImg;
|
|
|
+
|
|
|
+ for (Element sampleBoxEle : sampleBoxEles) {
|
|
|
+ String sampleBoxHref = sampleBoxEle.attr("href");
|
|
|
+ if (!sampleBoxHref.contains("http")) {
|
|
|
+ sampleBoxHref = sampleBoxEle.attr("abs:href");
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ responseImg = Jsoup.connect(sampleBoxHref).method(Connection.Method.GET).ignoreContentType(true).timeout(50 * 1000).execute();
|
|
|
+ } catch (HttpStatusException e) {
|
|
|
+ sampleBoxHref = sampleBoxEle.select("img").attr("src");
|
|
|
+ if (!sampleBoxHref.contains("http")) {
|
|
|
+ sampleBoxHref = sampleBoxEle.select("img").attr("abs:src");
|
|
|
+ }
|
|
|
+ responseImg = Jsoup.connect(sampleBoxHref).method(Connection.Method.GET).ignoreContentType(true).timeout(50 * 1000).execute();
|
|
|
+ }
|
|
|
+ String sampleBoxFileName = sampleBoxHref.substring(sampleBoxHref.lastIndexOf("/") + 1);
|
|
|
+ saveFile2(responseImg.bodyStream(), uncensoredPath.concat(liuchuImgGF), sampleBoxFileName);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ log.error("jsoupVideoInfoUncensored img_gf null,identificationCode={}", videoInfoUncensored.getIdentificationCode());
|
|
|
+ }
|
|
|
+ picTime.append("img_gf:").append(System.currentTimeMillis() - start2).append("}");
|
|
|
+ }
|
|
|
+
|
|
|
+ return picTime.toString();
|
|
|
+ }
|
|
|
+
|
|
|
private String parseDocument4Infantry4Javdb(Document document, String infantryPath, VideoInfoInfantry videoInfoInfantry) throws Exception {
|
|
|
Elements container = document.select("section.section > div.container");
|
|
|
if (container.size() == 0) {
|
|
|
@@ -1482,7 +1737,7 @@ public class BgServiceImpl implements BgService {
|
|
|
long start = System.currentTimeMillis();
|
|
|
Connection.Response response = Jsoup.connect(href).method(Connection.Method.GET).ignoreContentType(true).timeout(50 * 1000).execute();
|
|
|
|
|
|
- String fileName = issueDate.concat(" ").concat(videoInfoPool.getIdentificationCode()).concat(" ").concat(videoInfoPool.getName().replace("/", "")).replace("?", "?");
|
|
|
+ String fileName = issueDate.concat(" ").concat(videoInfoPool.getIdentificationCode()).concat(" ").concat(StringUtils.escapeJavParam(videoInfoPool.getName()));
|
|
|
byte[] imgUrlBytes = fileName.getBytes(StandardCharsets.UTF_8);
|
|
|
if (imgUrlBytes.length > 251) {
|
|
|
byte[] imgUrlDestBytes = new byte[251];
|