update:jousp抽取数据增量更新v1

src/main/java/top/lvzhiqiang/controller/BgController.java

@@ -155,6 +155,26 @@ public class BgController {
     }
 
     /**
+     * insetOrupdateScoreOrComment
+     *
+     * @author lvzhiqiang
+     * 2022/5/4 9:54
+     */
+    @RequestMapping("/insertOrUpdateScoreOrComment")
+    @ResponseBody
+    public String insertOrUpdateScoreOrComment(String identificationCode, String score, String comment) throws IllegalAccessException {
+        if (StringUtils.isEmpty(identificationCode)) {
+            throw new ParameterException("identificationCode为空!");
+        }
+
+        if (StringUtils.isEmpty(score) && StringUtils.isEmpty(comment)) {
+            throw new ParameterException("score和comment不能都为空!");
+        }
+
+        return bgService.insertOrUpdateScoreOrComment(identificationCode.toUpperCase(), score, comment);
+    }
+
+    /**
      * findDicCode
      *
      * @author lvzhiqiang
@@ -231,24 +251,4 @@ public class BgController {
 
         return bgService.findVideoInfo(identificationCode, type, order, crudT);
     }
-
-    /**
-     * insetOrupdateScoreOrComment
-     *
-     * @author lvzhiqiang
-     * 2022/5/4 9:54
-     */
-    @RequestMapping("/insertOrUpdateScoreOrComment")
-    @ResponseBody
-    public String insertOrUpdateScoreOrComment(String identificationCode, String score, String comment) throws IllegalAccessException {
-        if (StringUtils.isEmpty(identificationCode)) {
-            throw new ParameterException("identificationCode为空!");
-        }
-
-        if (StringUtils.isEmpty(score) && StringUtils.isEmpty(comment)) {
-            throw new ParameterException("score和comment不能都为空!");
-        }
-
-        return bgService.insertOrUpdateScoreOrComment(identificationCode.toUpperCase(), score, comment);
-    }
 }

src/main/java/top/lvzhiqiang/mapper/VideoCastMapper.java

@@ -47,6 +47,17 @@ public interface VideoCastMapper {
     int insert(VideoCast videoCast);
 
     /**
+     * 新增/修改
+     *
+     * @param videoCast
+     */
+    @Insert("INSERT INTO video_cast(name, code, type, sort, create_time, modify_time) " +
+            "VALUES (#{name}, #{code}, #{type}, #{sort}, now(), now()) " +
+            "ON DUPLICATE KEY UPDATE name=values(name),type=values(type),modify_time=now()")
+    @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id")
+    int insertOrUpdate(VideoCast videoCast);
+
+    /**
      * 查询所有
      */
     @Select("SELECT * FROM video_cast WHERE delete_flag = 1 ORDER BY sort")

src/main/java/top/lvzhiqiang/mapper/VideoGenresMapper.java

@@ -47,6 +47,17 @@ public interface VideoGenresMapper {
     int insert(VideoGenres videoGenres);
 
     /**
+     * 新增/修改
+     *
+     * @param videoGenres
+     */
+    @Insert("INSERT INTO video_genres(name, code, sort, create_time, modify_time) " +
+            "VALUES (#{name}, #{code}, #{sort}, now(), now()) " +
+            "ON DUPLICATE KEY UPDATE name=values(name),modify_time=now()")
+    @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id")
+    int insertOrUpdate(VideoGenres videoGenres);
+
+    /**
      * 查询所有
      */
     @Select("SELECT * FROM video_genres WHERE delete_flag = 1 ORDER BY sort")

src/main/java/top/lvzhiqiang/mapper/VideoInfoCastMapper.java

@@ -35,6 +35,21 @@ public interface VideoInfoCastMapper {
     void insertList(Set<VideoInfoCast> videoInfoCastList);
 
     /**
+     * 批量新增/修改
+     *
+     * @param videoInfoCastList
+     */
+    @Insert({"<script>" +
+            "INSERT INTO video_info_cast(identification_code, name , type, create_time) " +
+            "VALUES " +
+            "<foreach collection='collection' item='vic' index=\"index\" separator=\",\">" +
+            "   (#{vic.identificationCode}, #{vic.name}, #{vic.type}, now())" +
+            " </foreach>" +
+            " ON DUPLICATE KEY UPDATE identification_code=values(identification_code),name=values(name),type=values(type),create_time=now()" +
+            "</script>"})
+    void insertOrUpdate(Set<VideoInfoCast> videoInfoCastList);
+
+    /**
      * 新增
      *
      * @param videoInfoCast

src/main/java/top/lvzhiqiang/mapper/VideoInfoGenresMapper.java

@@ -36,6 +36,21 @@ public interface VideoInfoGenresMapper {
     void insertList(Set<VideoInfoGenres> videoInfoGenresList);
 
     /**
+     * 批量新增/修改
+     *
+     * @param videoInfoGenresList
+     */
+    @Insert({"<script>" +
+            "INSERT INTO video_info_genres(identification_code, name , type, create_time) " +
+            "VALUES " +
+            "<foreach collection='collection' item='vig' index=\"index\" separator=\",\">" +
+            "   (#{vig.identificationCode}, #{vig.name}, #{vig.type}, now())" +
+            " </foreach>" +
+            " ON DUPLICATE KEY UPDATE identification_code=values(identification_code),name=values(name),type=values(type),create_time=now()" +
+            "</script>"})
+    int insertOrUpdate(Set<VideoInfoGenres> videoInfoGenresList);
+
+    /**
      * 新增
      *
      * @param videoInfoGenres

src/main/java/top/lvzhiqiang/mapper/VideoInfoInfantryMapper.java

@@ -56,8 +56,8 @@ public interface VideoInfoInfantryMapper {
     /**
      * 查询所有识别码
      */
-    @Select("select distinct identification_code from video_info_infantry")
-    List<String> findAllIcode();
+    @Select("select distinct identification_code from video_info_infantry where type = #{infantryType}")
+    List<String> findAllIcode(String infantryType);
 
     /**
      * 根据条件查询

src/main/java/top/lvzhiqiang/service/impl/BgServiceImpl.java

@@ -392,10 +392,10 @@ public class BgServiceImpl implements BgService {
                     ++retryCount;
 
                     if (retryCount < 4) {
-                        log.error("jsoupVideoInfo error重试：i={}，retryCount={}，time={}，identificationCode={}", i, retryCount, System.currentTimeMillis() - start, identificationCode, e);
+                        log.error("jsoupVideoInfo error重试：i={}，retryCount={}，time={}，identificationCode={}，javbusUrl={}", i, retryCount, System.currentTimeMillis() - start, identificationCode, javbusUrl, e);
                     } else if (retryCount == 4) {
                         videoInfoMapper.updateStatus(identificationCode, 3);
-                        log.error("jsoupVideoInfo error：i={}，time={}，identificationCode={}", i, System.currentTimeMillis() - start, identificationCode, e);
+                        log.error("jsoupVideoInfo error：i={}，time={}，identificationCode={}，javbusUrl={}", i, System.currentTimeMillis() - start, identificationCode, javbusUrl, e);
                     }
                 }
             }
@@ -427,9 +427,20 @@ public class BgServiceImpl implements BgService {
 
         if (type == 1) {
             JavAllInfo javAllInfo = new JavAllInfo();
-            getAllFilePaths(picPath, javAllInfo);
 
-            saveInfo(javAllInfo);
+            // 删除所有
+            if (isDel == 1) {
+                videoGenresMapper.deleteAll();
+                videoInfoMapper.deleteAll();
+                videoCastMapper.deleteAll();
+                videoInfoGenresMapper.deleteAll();
+                videoInfoCastMapper.deleteAll();
+            }
+
+            List<String> allIcode = videoInfoMapper.findAllIcode();
+            getAllFilePaths(picPath, javAllInfo, allIcode);
+
+            saveInfo(javAllInfo, isDel);
         } else if (type == 3) {
             JavAllInfo4Uncensored javAllInfo4Uncensored = new JavAllInfo4Uncensored();
             getAllFilePaths4Uncensored(picPath, javAllInfo4Uncensored);
@@ -440,7 +451,7 @@ public class BgServiceImpl implements BgService {
             picPath = picPath.concat(infantryType);
             getAllFilePaths4Infantry(picPath, javAllInfo4Infantry, infantryType);
 
-            saveInfo4Infantry(javAllInfo4Infantry, isDel);
+            saveInfo4Infantry(javAllInfo4Infantry, isDel, infantryType);
         }
 
         long endTime = System.currentTimeMillis();
@@ -846,7 +857,7 @@ public class BgServiceImpl implements BgService {
     }
 
     // 递归获取某目录下的所有子目录以及子文件
-    private void getAllFilePaths(String filePath, JavAllInfo javAllInfo) {
+    private void getAllFilePaths(String filePath, JavAllInfo javAllInfo, List<String> allIcode) {
         File[] files = new File(filePath).listFiles();
         if (files == null) {
             return;
@@ -861,7 +872,7 @@ public class BgServiceImpl implements BgService {
         for (File file : files) {
             if (file.isDirectory()) {
                 // 文件夹
-                getAllFilePaths(file.getAbsolutePath(), javAllInfo);
+                getAllFilePaths(file.getAbsolutePath(), javAllInfo, allIcode);
             } else {
                 String fileName = file.getName();
                 if (fileName.endsWith(".jpg") || (fileName.endsWith(".lnk") && fileName.contains(".jpg"))) {
@@ -872,6 +883,10 @@ public class BgServiceImpl implements BgService {
                     try {
                         boolean isMain = false;
                         if (fileName.endsWith(".jpg")) {
+                            if (allIcode.contains(nameArr[0])) {
+                                continue;
+                            }
+
                             isMain = true;
                             // 获取正片信息
                             VideoInfo videoInfo = new VideoInfo();
@@ -932,6 +947,10 @@ public class BgServiceImpl implements BgService {
                     }
                 } else if (!fileName.endsWith(".jpg") && !fileName.endsWith(".lnk")) {
                     String[] nameArr = fileName.substring(0, fileName.lastIndexOf(".")).split("\\s+");
+                    if (allIcode.contains(nameArr[1])) {
+                        continue;
+                    }
+
                     String parentName = file.getParentFile().getName();
 
                     javAllInfo.getVideoUrlMap().put(nameArr[1], parentName.concat("/").concat(fileName));
@@ -1060,21 +1079,14 @@ public class BgServiceImpl implements BgService {
 
     // 保存所有文件
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
-    public void saveInfo(JavAllInfo javAllInfo) {
-        // 删除所有
-        videoGenresMapper.deleteAll();
-        videoInfoMapper.deleteAll();
-        videoCastMapper.deleteAll();
-        videoGenresMapper.deleteAll();
-        videoInfoCastMapper.deleteAll();
-
+    public void saveInfo(JavAllInfo javAllInfo, Integer isDel) {
         // 保存分类
         Set<String> videoGenresSet = javAllInfo.getVideoGenresSet();
         //List<VideoGenres> videoGenresList = new ArrayList<>();
         for (String s : videoGenresSet) {
             VideoGenres videoGenres = new VideoGenres();
             videoGenres.setName(s);
-            videoGenresMapper.insert(videoGenres);
+            videoGenresMapper.insertOrUpdate(videoGenres);
             System.out.println(videoGenres);
             //videoGenresList.add(videoGenres);
         }
@@ -1087,7 +1099,7 @@ public class BgServiceImpl implements BgService {
             VideoCast videoCast = new VideoCast();
             videoCast.setName(entry.getKey());
             videoCast.setType(Integer.parseInt(entry.getValue()));
-            videoCastMapper.insert(videoCast);
+            videoCastMapper.insertOrUpdate(videoCast);
             System.out.println(videoCast);
             //videoCastList.add(videoCast);
         }
@@ -1114,10 +1126,10 @@ public class BgServiceImpl implements BgService {
 
         // 保存影片类别关联信息
         Set<VideoInfoGenres> videoInfoGenresSet = javAllInfo.getVideoInfoGenresSet();
-        videoInfoGenresMapper.insertList(videoInfoGenresSet);
+        videoInfoGenresMapper.insertOrUpdate(videoInfoGenresSet);
         // 保存影片类别关联信息
         Set<VideoInfoCast> videoInfoCastSet = javAllInfo.getVideoInfoCastSet();
-        videoInfoCastMapper.insertList(videoInfoCastSet);
+        videoInfoCastMapper.insertOrUpdate(videoInfoCastSet);
     }
 
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
@@ -1127,31 +1139,53 @@ public class BgServiceImpl implements BgService {
             videoInfoUncensoredMapper.deleteAll();
         }
 
+        List<String> allIcode = videoInfoUncensoredMapper.findAllIcode();
 
         // 保存影片信息
         List<VideoInfoUncensored> videoInfoUncensoredList = javAllInfo4Uncensored.getVideoInfoUncensoredList();
+        int videoInfoUncensoredListBefore = videoInfoUncensoredList.size();
         Map<String, String> videoUrlMap = javAllInfo4Uncensored.getVideoUrlMap();
+        Iterator<VideoInfoUncensored> iterator = videoInfoUncensoredList.iterator();
+        while (iterator.hasNext()) {
+            if (allIcode.contains(iterator.next().getIdentificationCode())) {
+                iterator.remove();
+            }
+        }
         videoInfoUncensoredList.parallelStream().forEach(e -> {
             e.setVideoUrl(videoUrlMap.get(e.getIdentificationCode()));
         });
+        System.out.println("videoInfoUncensoredListBefore:" + videoInfoUncensoredListBefore + "，videoInfoUncensoredListAfter:" + videoInfoUncensoredList.size());
 
-        int videoInfoCount = videoInfoUncensoredMapper.insertList(videoInfoUncensoredList);
+        int videoInfoCount = 0;
+        if (videoInfoUncensoredList.size() > 0) {
+            videoInfoCount = videoInfoUncensoredMapper.insertList(videoInfoUncensoredList);
+        }
         System.out.println("videoInfoCount:" + videoInfoCount);
     }
 
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
-    public void saveInfo4Infantry(JavAllInfo4Infantry javAllInfo4Infantry, Integer isDel) {
+    public void saveInfo4Infantry(JavAllInfo4Infantry javAllInfo4Infantry, Integer isDel, String infantryType) {
         // 删除所有
         if (isDel == 1) {
             videoInfoInfantryMapper.deleteAll();
         }
 
+        List<String> allIcode = videoInfoInfantryMapper.findAllIcode(infantryType);
+
         // 保存影片信息
         List<VideoInfoInfantry> videoInfoInfantryList = javAllInfo4Infantry.getVideoInfoInfantryList();
+        int videoInfoInfantryListBefore = videoInfoInfantryList.size();
         Map<String, String> videoUrlMap = javAllInfo4Infantry.getVideoUrlMap();
+        Iterator<VideoInfoInfantry> iterator = videoInfoInfantryList.iterator();
+        while (iterator.hasNext()) {
+            if (allIcode.contains(iterator.next().getIdentificationCode())) {
+                iterator.remove();
+            }
+        }
         videoInfoInfantryList.parallelStream().forEach(e -> {
             e.setVideoUrl(videoUrlMap.get(e.getIdentificationCode()));
         });
+        System.out.println("videoInfoInfantryListBefore:" + videoInfoInfantryListBefore + "，videoInfoInfantryListAfter:" + videoInfoInfantryList.size());
 
         int videoInfoCount = 0;
         if (videoInfoInfantryList.size() > 0) {

src/test/java/top/lvzhiqiang/Test6.java

@@ -5,12 +5,20 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import top.lvzhiqiang.entity.VideoCast;
+import top.lvzhiqiang.entity.VideoGenres;
+import top.lvzhiqiang.entity.VideoInfoGenres;
 import top.lvzhiqiang.entity.VideoInfoPool;
+import top.lvzhiqiang.mapper.VideoCastMapper;
+import top.lvzhiqiang.mapper.VideoGenresMapper;
+import top.lvzhiqiang.mapper.VideoInfoGenresMapper;
 import top.lvzhiqiang.mapper.VideoInfoPoolMapper;
 
 import javax.annotation.Resource;
 import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * 单元测试类
@@ -30,6 +38,15 @@ public class Test6 {
     @Resource
     private VideoInfoPoolMapper videoInfoPoolMapper;
 
+    @Resource
+    private VideoGenresMapper videoGenresMapper;
+
+    @Resource
+    private VideoCastMapper videoCastMapper;
+
+    @Resource
+    private VideoInfoGenresMapper videoInfoGenresMapper;
+
     @Test
     public void testGetBusinessDeptSummary() {
         List<VideoInfoPool> videoInfoPoolList = videoInfoPoolMapper.findAll();
@@ -45,4 +62,36 @@ public class Test6 {
             }
         }
     }
+
+    @Test
+    public void testInsertOrUpdateVideoGenres() {
+        VideoGenres videoGenres = new VideoGenres();
+        videoGenres.setName("教学1");
+        videoGenresMapper.insertOrUpdate(videoGenres);
+        System.out.println(videoGenres);
+    }
+
+    @Test
+    public void testInsertOrUpdateVideoCast() {
+        VideoCast videoCast = new VideoCast();
+        videoCast.setName("JULIA3");
+        videoCast.setType(3);
+        videoCastMapper.insertOrUpdate(videoCast);
+        System.out.println(videoCast);
+    }
+
+    @Test
+    public void testInsertOrUpdateVideoInfoGenres() {
+        Set<VideoInfoGenres> videoInfoGenresSet = new HashSet<>();
+
+        VideoInfoGenres videoInfoGenres1 = new VideoInfoGenres();
+        videoInfoGenres1.setIdentificationCode("ABP-959");
+        videoInfoGenres1.setName("LoveFeet");
+        videoInfoGenres1.setType(2);
+        videoInfoGenresSet.add(videoInfoGenres1);
+
+        videoInfoGenresMapper.insertOrUpdate(videoInfoGenresSet);
+
+        System.out.println(videoInfoGenresSet);
+    }
 }