add:syncCoinmarketcapCMap v1

pom.xml

@@ -94,7 +94,7 @@
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>fastjson</artifactId>
-            <version>1.2.54</version>
+            <version>2.0.44</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
         <dependency>

src/main/java/top/lvzhiqiang/config/InitRunner.java

@@ -44,6 +44,7 @@ public class InitRunner implements ApplicationRunner {
 
     public static Proxy proxy = null;
     public static Map<String, JSONObject> publicParamsMap = null;
+    public static Map<String, DicCode> dicCodeMap = null;
 
     @Override
     public void run(ApplicationArguments args) {
@@ -58,7 +59,7 @@ public class InitRunner implements ApplicationRunner {
             }
         }
 
-        Map<String, DicCode> dicCodeMap = dicCodeMapper.findAllMap();
+        dicCodeMap = dicCodeMapper.findAllMap();
         FtpUtil.setParams(dicCodeMap.get("ftp_host").getCodeValue(), dicCodeMap.get("ftp_port").getCodeValue(),
                 dicCodeMap.get("ftp_username").getCodeValue(), dicCodeMap.get("ftp_password").getCodeValue(),
                 dicCodeMap.get("ftp_basepath").getCodeValue(), dicCodeMap.get("ftp_baseurl").getCodeValue());

src/main/java/top/lvzhiqiang/config/MyCoinJobs.java

@@ -42,4 +42,14 @@ public class MyCoinJobs {
     public void syncData4TraderList() {
         coinService.syncData4TraderList();
     }
+
+    /**
+     * 每天6：00 jsoup coinmarketcap map
+     */
+    @Scheduled(cron = "0 0 6 * * ?", zone = SCHEDULED_ZONE)
+    public void syncCoinmarketcapCMap() {
+        log.warn("jsoupCoinmarketcapCMap开始==============================");
+
+        coinService.syncCoinmarketcapCMap();
+    }
 }

src/main/java/top/lvzhiqiang/entity/CoinCmcMap.java

@@ -0,0 +1,80 @@
+package top.lvzhiqiang.entity;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+/**
+ * CoinCmcMap
+ *
+ * @author lvzhiqiang
+ * 2023/12/31 18:52
+ */
+@Data
+public class CoinCmcMap implements Serializable {
+
+    /**
+     * 主键
+     */
+    private Long id;
+
+    /**
+     *
+     */
+    private Long cmcId;
+
+    /**
+     *
+     */
+    private Long cmcRank;
+
+    /**
+     *
+     */
+    private String name;
+
+    /**
+     *
+     */
+    private String symbol;
+
+    /**
+     *
+     */
+    private String slug;
+
+    /**
+     *
+     */
+    private Integer isActive;
+
+    /**
+     *
+     */
+    private String status;
+
+    /**
+     *
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
+    private LocalDateTime firstHistoricalData;
+
+    /**
+     *
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
+    private LocalDateTime lastHistoricalData;
+
+    /**
+     *
+     */
+    private String platform;
+
+    /**
+     *
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
+    private LocalDateTime modifyTime;
+}

src/main/java/top/lvzhiqiang/mapper/CoinMapper.java

@@ -122,4 +122,17 @@ public interface CoinMapper {
             " order by ${sortField} ${sort}" +
             "</script>"})
     List<FileImage> findImageList(Map<String, Object> params);
+
+    @Insert({"<script>" +
+            "INSERT INTO coin_cmc_map(cmc_id,cmc_rank,name,symbol,slug,is_active,status," +
+            "first_historical_data,last_historical_data,platform,modify_time)" +
+            " VALUES " +
+            "<foreach collection='list' item='mt' index=\"index\" separator=\",\">" +
+            "   (#{mt.cmcId},#{mt.cmcRank},#{mt.name},#{mt.symbol},#{mt.slug},#{mt.isActive},#{mt.status}," +
+            "#{mt.firstHistoricalData},#{mt.lastHistoricalData},#{mt.platform},now())" +
+            " </foreach>" +
+            " ON DUPLICATE KEY UPDATE cmc_rank=values(cmc_rank),name=values(name),symbol=values(symbol),slug=values(slug)," +
+            "is_active=values(is_active),status=values(status),first_historical_data=values(first_historical_data),last_historical_data=values(last_historical_data),modify_time=now()" +
+            "</script>"})
+    void insertCmcMapList(List<CoinCmcMap> cmcMapList);
 }

src/main/java/top/lvzhiqiang/service/CoinService.java

@@ -44,4 +44,6 @@ public interface CoinService {
     String watchlistDetail(String symbol, String operationType);
 
     Object watchlistUpdate(String symbol, String remark);
+
+    void syncCoinmarketcapCMap();
 }

src/main/java/top/lvzhiqiang/service/impl/CoinServiceImpl.java

@@ -43,6 +43,7 @@ import java.util.*;
 import java.util.concurrent.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Coin ServiceImpl
@@ -193,6 +194,88 @@ public class CoinServiceImpl implements CoinService {
         log.warn("syncData4TraderList 结束：time={},totalNum={}", stopWatch.getTotalTimeSeconds(), totalNum);
     }
 
+
+    @Override
+    public void syncCoinmarketcapCMap() {
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+
+        String coinmarketcapApikey = InitRunner.dicCodeMap.get("coinmarketcap_apikey").getCodeValue();
+        String coinmarketcapIdmapUrl = InitRunner.dicCodeMap.get("coinmarketcap_idmap_url").getCodeValue();
+        String coinmarketcapIdmapParams4listingStatus = InitRunner.dicCodeMap.get("coinmarketcap_idmap_params_listing_status").getCodeValue();
+        String coinmarketcapIdmapParams4aux = InitRunner.dicCodeMap.get("coinmarketcap_idmap_params_aux").getCodeValue();
+        Map<String, String> headerMap = new HashMap<>();
+        headerMap.put("Accept", "application/json");
+        headerMap.put("Accept-Encoding", "deflate, gzip");
+        headerMap.put("X-CMC_PRO_API_KEY", coinmarketcapApikey);
+
+        String[] listingStatusArr = coinmarketcapIdmapParams4listingStatus.split(",");
+        Map<String, String> paramMap = new LinkedHashMap<>();
+        int MAX_NUMBER = 1000;
+        int MAX_NUMBER2 = 5000;
+        Long totalNum = 0L;
+        for (String listingStatus : listingStatusArr) {
+            paramMap.put("listing_status", listingStatus);
+            paramMap.put("aux", coinmarketcapIdmapParams4aux);
+
+            try {
+                int j = 0;
+                Long totalNum2 = 0L;
+                for (; ; ) {
+                    Thread.sleep(3000L);
+
+                    j++;
+
+                    paramMap.put("start", String.valueOf((j - 1) * MAX_NUMBER2 + 1));
+                    paramMap.put("limit", String.valueOf(MAX_NUMBER2));
+
+                    Connection.Response response = JsoupUtil.requestBody(coinmarketcapIdmapUrl, JsoupUtil.HTTP_GET, InitRunner.proxy, headerMap, paramMap);
+                    JSONObject result = JSONObject.parseObject(response.body());
+                    JSONArray dataJA = result.getJSONArray("data");
+
+                    List<CoinCmcMap> cmcMapList = new ArrayList<>();
+                    CoinCmcMap coinCmcMap;
+                    for (int i = 0; i < dataJA.size(); i++) {
+                        JSONObject dataJO = dataJA.getJSONObject(i);
+
+                        coinCmcMap = new CoinCmcMap();
+                        coinCmcMap.setCmcId(dataJO.getLong("id"));
+                        coinCmcMap.setCmcRank(dataJO.getLong("rank"));
+                        coinCmcMap.setName(dataJO.getString("name"));
+                        coinCmcMap.setSymbol(dataJO.getString("symbol"));
+                        coinCmcMap.setSlug(dataJO.getString("slug"));
+                        coinCmcMap.setIsActive(dataJO.getInteger("is_active"));
+                        coinCmcMap.setStatus(dataJO.getString("status"));
+                        coinCmcMap.setFirstHistoricalData(DateUtils.stringutcToLocalDateTime(dataJO.getString("first_historical_data")));
+                        coinCmcMap.setLastHistoricalData(DateUtils.stringutcToLocalDateTime(dataJO.getString("last_historical_data")));
+                        coinCmcMap.setPlatform(dataJO.getString("platform"));
+
+                        cmcMapList.add(coinCmcMap);
+                    }
+
+                    // 新增或者更新
+                    Stream.iterate(0, n -> n + 1).limit((cmcMapList.size() + MAX_NUMBER - 1) / MAX_NUMBER)
+                            .forEach(i -> {
+                                List<CoinCmcMap> list = cmcMapList.stream().skip((long) i * MAX_NUMBER).limit(MAX_NUMBER).collect(Collectors.toList());
+                                coinMapper.insertCmcMapList(list);
+                            });
+                    totalNum += cmcMapList.size();
+                    totalNum2 += cmcMapList.size();
+
+                    if (dataJA.size() < MAX_NUMBER2) {
+                        break;
+                    }
+                }
+
+                log.warn("syncCoinmarketcapCMap {} success，totalNum={}", listingStatus, totalNum2);
+            } catch (Exception e) {
+                log.error("syncCoinmarketcapCMap {} error", listingStatus, e);
+            }
+        }
+
+        log.warn("syncCoinmarketcapCMap 结束：time={},totalNum={}", stopWatch.getTotalTimeSeconds(), totalNum);
+    }
+
     @Override
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
     public void syncData4TraderListSub(JSONArray dataList) {
@@ -719,29 +802,53 @@ public class CoinServiceImpl implements CoinService {
 
                     JSONObject marketData = jsonObject.getJSONObject("market_data");
                     // 总市值排名
-                    Integer totalMarketRanking = marketData.getInteger("market_cap_rank");
-                    coinWatchlist.setTotalMarketRanking(totalMarketRanking);
+                    if (marketData.containsKey("market_cap_rank") && null != marketData.get("market_cap_rank")) {
+                        Integer totalMarketRanking = marketData.getInteger("market_cap_rank");
+                        coinWatchlist.setTotalMarketRanking(totalMarketRanking);
+                    }
+
                     // 总市值
-                    BigDecimal totalMarketValue = marketData.getJSONObject("market_cap").getBigDecimal("usd");
-                    coinWatchlist.setTotalMarketValue(totalMarketValue);
+                    if (marketData.containsKey("market_cap") && null != marketData.get("market_cap")) {
+                        BigDecimal totalMarketValue = marketData.getJSONObject("market_cap").getBigDecimal("usd");
+                        coinWatchlist.setTotalMarketValue(totalMarketValue);
+                    }
+
                     // 市场价格
-                    String markPrice = marketData.getJSONObject("current_price").getString("usd");
-                    coinWatchlist.setMarkPrice(markPrice);
+                    if (marketData.containsKey("current_price") && null != marketData.get("current_price")) {
+                        String markPrice = marketData.getJSONObject("current_price").getString("usd");
+                        coinWatchlist.setMarkPrice(markPrice);
+                    }
+
                     // 历史最高价格
-                    String highestHistoricalPrice = marketData.getJSONObject("ath").getString("usd");
-                    coinWatchlist.setHighestHistoricalPrice(highestHistoricalPrice);
+                    if (marketData.containsKey("ath") && null != marketData.get("ath")) {
+                        String highestHistoricalPrice = marketData.getJSONObject("ath").getString("usd");
+                        coinWatchlist.setHighestHistoricalPrice(highestHistoricalPrice);
+                    }
+
                     // 历史最高日期
-                    LocalDate highestHistoricalDate = LocalDate.parse(marketData.getJSONObject("ath_date").getString("usd"), DateUtils.utcTimeFormatter);
-                    coinWatchlist.setHighestHistoricalDate(highestHistoricalDate);
+                    if (marketData.containsKey("ath_date") && null != marketData.get("ath_date")) {
+                        LocalDate highestHistoricalDate = LocalDate.parse(marketData.getJSONObject("ath_date").getString("usd"), DateUtils.utcTimeFormatter);
+                        coinWatchlist.setHighestHistoricalDate(highestHistoricalDate);
+                    }
+
                     // 历史最低价格
-                    String lowestHistoricalPrice = marketData.getJSONObject("atl").getString("usd");
-                    coinWatchlist.setLowestHistoricalPrice(lowestHistoricalPrice);
+                    if (marketData.containsKey("atl") && null != marketData.get("atl")) {
+                        String lowestHistoricalPrice = marketData.getJSONObject("atl").getString("usd");
+                        coinWatchlist.setLowestHistoricalPrice(lowestHistoricalPrice);
+                    }
+
                     // 历史最低日期
-                    LocalDate lowestHistoricalDate = LocalDate.parse(marketData.getJSONObject("atl_date").getString("usd"), DateUtils.utcTimeFormatter);
-                    coinWatchlist.setLowestHistoricalDate(lowestHistoricalDate);
+                    if (marketData.containsKey("atl_date") && null != marketData.get("atl_date")) {
+                        LocalDate lowestHistoricalDate = LocalDate.parse(marketData.getJSONObject("atl_date").getString("usd"), DateUtils.utcTimeFormatter);
+                        coinWatchlist.setLowestHistoricalDate(lowestHistoricalDate);
+                    }
+
                     // 涨幅倍数
-                    BigDecimal increaseMultiple = new BigDecimal(highestHistoricalPrice).divide(new BigDecimal(lowestHistoricalPrice), 0, RoundingMode.HALF_UP);
-                    coinWatchlist.setIncreaseMultiple(increaseMultiple.intValue());
+                    if (StringUtils.isNotEmpty(coinWatchlist.getHighestHistoricalPrice()) && StringUtils.isNotEmpty(coinWatchlist.getLowestHistoricalPrice())) {
+                        BigDecimal increaseMultiple = new BigDecimal(coinWatchlist.getHighestHistoricalPrice()).divide(new BigDecimal(coinWatchlist.getLowestHistoricalPrice()), 0, RoundingMode.HALF_UP);
+                        coinWatchlist.setIncreaseMultiple(increaseMultiple.intValue());
+                    }
+
                     // 发行日期
                     // 发行天数
                     if (coinWatchlist.getIssuingDate() != null) {

src/main/java/top/lvzhiqiang/util/DateUtils.java

@@ -389,6 +389,13 @@ public class DateUtils {
         return ZonedDateTime.now(ZoneOffset.UTC).format(utcTimeFormatter);
     }
 
+    public static LocalDateTime stringutcToLocalDateTime(String time) {
+        if (StringUtils.isEmpty(time)) {
+            return null;
+        }
+        return ZonedDateTime.parse(time).withZoneSameInstant(ZoneId.of("Asia/Shanghai")).toLocalDateTime();
+    }
+
     public static void main(String[] args) {
         //System.out.println(getFewDateStrsByMonth(getToday(), 7));
         String sss = "2023-12-18T16:30:00+09:00";
@@ -396,5 +403,11 @@ public class DateUtils {
         System.out.println(parse);
 
         System.out.println(getUTCTimeStr());
+
+        String ss = "2013-11-05T13:30:11.000Z";
+        System.out.println(stringutcToLocalDateTime(ss));
+
+        LocalDateTime localDateTime = OffsetDateTime.parse(ss, DateUtils.utcTimeFormatter).toLocalDateTime();
+        System.out.println(localDateTime);
     }
 }