update:coingecko jsoup 兼容数据不全优化v1

src/main/java/top/lvzhiqiang/entity/CoinWatchlist.java

@@ -22,6 +22,8 @@ public class CoinWatchlist implements Serializable {
      */
     private Long id;
 
+    private Long cmcId;
+
     /**
      * 名称
      */

src/main/java/top/lvzhiqiang/mapper/CoinMapper.java

@@ -97,7 +97,7 @@ public interface CoinMapper {
             "</script>"})
     List<CoinWatchlist> findWatchlistList(Map<String, Object> params);
 
-    @Update("update coin_watchlist set total_market_ranking=#{totalMarketRanking},total_market_value=#{totalMarketValue}," +
+    @Update("update coin_watchlist set cmc_id=#{cmcId},total_market_ranking=#{totalMarketRanking},total_market_value=#{totalMarketValue}," +
             "mark_price=#{markPrice},highest_historical_price=#{highestHistoricalPrice},lowest_historical_price=#{lowestHistoricalPrice}," +
             "highest_historical_date=#{highestHistoricalDate},lowest_historical_date=#{lowestHistoricalDate},increase_multiple=#{increaseMultiple}," +
             "issuing_days=#{issuingDays},modify_time=now() where id = #{id}")

src/main/java/top/lvzhiqiang/service/impl/CoinServiceImpl.java

@@ -41,6 +41,7 @@ import java.time.LocalDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -206,7 +207,7 @@ public class CoinServiceImpl implements CoinService {
         String coinmarketcapIdmapParams4aux = InitRunner.dicCodeMap.get("coinmarketcap_idmap_params_aux").getCodeValue();
         Map<String, String> headerMap = new HashMap<>();
         headerMap.put("Accept", "application/json");
-        headerMap.put("Accept-Encoding", "deflate, gzip");
+        headerMap.put("Accept-Encoding", "deflate,gzip");
         headerMap.put("X-CMC_PRO_API_KEY", coinmarketcapApikey);
 
         String[] listingStatusArr = coinmarketcapIdmapParams4listingStatus.split(",");
@@ -789,6 +790,7 @@ public class CoinServiceImpl implements CoinService {
             params.put("sort", "desc");
             List<CoinWatchlist> watchlistList = coinMapper.findWatchlistList(params);
 
+            Map<Long, CoinWatchlist> watchlistMap4CmC = new HashMap<>();
             for (CoinWatchlist coinWatchlist : watchlistList) {
                 try {
                     Thread.sleep(5000L);
@@ -805,12 +807,19 @@ public class CoinServiceImpl implements CoinService {
                     if (marketData.containsKey("market_cap_rank") && null != marketData.get("market_cap_rank")) {
                         Integer totalMarketRanking = marketData.getInteger("market_cap_rank");
                         coinWatchlist.setTotalMarketRanking(totalMarketRanking);
+                    } else {
+                        watchlistMap4CmC.put(coinWatchlist.getCmcId(), coinWatchlist);
                     }
 
                     // 总市值
                     if (marketData.containsKey("market_cap") && null != marketData.get("market_cap")) {
                         BigDecimal totalMarketValue = marketData.getJSONObject("market_cap").getBigDecimal("usd");
                         coinWatchlist.setTotalMarketValue(totalMarketValue);
+                        if (totalMarketValue.compareTo(BigDecimal.ZERO) == 0) {
+                            watchlistMap4CmC.put(coinWatchlist.getCmcId(), coinWatchlist);
+                        }
+                    } else {
+                        watchlistMap4CmC.put(coinWatchlist.getCmcId(), coinWatchlist);
                     }
 
                     // 市场价格
@@ -861,6 +870,10 @@ public class CoinServiceImpl implements CoinService {
                     log.error("jsoup CoinWatchlist error，coinWatchlist={}", coinWatchlist, ex);
                 }
             }
+
+            if (watchlistMap4CmC.size() > 0) {
+                parseWatchlistMap4CmC(watchlistMap4CmC);
+            }
         }, 0, 1, TimeUnit.HOURS);
 
         // Upbit交易所监控报警
@@ -910,6 +923,37 @@ public class CoinServiceImpl implements CoinService {
         return null;
     }
 
+    public void parseWatchlistMap4CmC(Map<Long, CoinWatchlist> watchlistMap4CmC) {
+        String coinmarketcapApikey = InitRunner.dicCodeMap.get("coinmarketcap_apikey").getCodeValue();
+        String coinmarketcapQuotesLatestUrl = InitRunner.dicCodeMap.get("coinmarketcap_quotes_latest_url").getCodeValue();
+        Map<String, String> headerMap = new HashMap<>();
+        headerMap.put("Accept", "application/json");
+        headerMap.put("Accept-Encoding", "deflate,gzip");
+        headerMap.put("X-CMC_PRO_API_KEY", coinmarketcapApikey);
+
+        Map<String, String> paramMap = new LinkedHashMap<>();
+        paramMap.put("id", StringUtils.join(watchlistMap4CmC.keySet(), ","));
+        AtomicInteger i = new AtomicInteger();
+        try {
+            Connection.Response response = JsoupUtil.requestBody(coinmarketcapQuotesLatestUrl, JsoupUtil.HTTP_GET, InitRunner.proxy, headerMap, paramMap);
+            JSONObject result = JSONObject.parseObject(response.body());
+            JSONObject dataJO = result.getJSONObject("data");
+
+            watchlistMap4CmC.forEach((key, value) -> {
+                i.getAndIncrement();
+                if (dataJO.containsKey(key)) {
+                    JSONObject jsonObject = dataJO.getJSONObject(String.valueOf(key));
+                    value.setTotalMarketRanking(jsonObject.getInteger("cmc_rank"));
+                    value.setTotalMarketValue(jsonObject.getBigDecimal("self_reported_market_cap").setScale(2, RoundingMode.HALF_UP));
+
+                    coinMapper.updateCoinWatchlist(value);
+                }
+            });
+        } catch (Exception e) {
+            log.error("parseWatchlistMap4CmC error，size={}，i={}", watchlistMap4CmC.size(), i.get(), e);
+        }
+    }
+
     @Override
     @Async("coinTaskExecutor")
     public void monitorAlarm4APP_TEXT_CARD(String content, JSONObject params, WxCpService wxCpServiceFinal) {

src/main/resources/static/coin.html

@@ -232,9 +232,9 @@
             <input type="text" style="width: 100px;padding-top: 3px;" id="apis-quiet-div-cmcmap-pages" disabled="disabled" value="999999">
             <input type="text" style="width: 100px;padding-top: 3px;" id="apis-quiet-div-cmcmap-keyword" placeholder="关键词">
             <select id="apis-quiet-div-cmcmap-filterField" style="height: 24px;">
-                <option value="active">active</option>
-                <option value="inactive">inactive</option>
-                <option value="untracked">untracked</option>
+                <option value="active">活动</option>
+                <option value="inactive">非活动</option>
+                <option value="untracked">未跟踪</option>
                 <option value="">--</option>
             </select>
             <select id="apis-quiet-div-cmcmap-sortField" style="height: 24px;">

src/main/resources/static/js/my-coin.js

@@ -98,6 +98,12 @@ function initOther() {
     $(".apis-move-div-button3").click(function () {
         ownClearInterval();
     });
+
+    $("#apis-quiet-div-cmcmap-keyword").keydown(function (e) {
+        if (e.keyCode == 13) {
+            $(".apis-quiet-div-button2").click();
+        }
+    });
 }
 
 function ownClearInterval() {

src/test/java/top/lvzhiqiang/TestCoin.java

@@ -12,10 +12,12 @@ import top.lvzhiqiang.mapper.CoinMapper;
 import top.lvzhiqiang.service.CoinService;
 import top.lvzhiqiang.util.DateUtils;
 import top.lvzhiqiang.util.JsoupUtil;
+import top.lvzhiqiang.util.StringUtils;
 
 import javax.annotation.Resource;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -76,6 +78,7 @@ public class TestCoin {
         params.put("sort", "desc");
         List<CoinWatchlist> watchlistList = coinMapper.findWatchlistList(params);
 
+        Proxy proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 1080));
         for (CoinWatchlist coinWatchlist : watchlistList) {
             try {
                 Thread.sleep(5000L);
@@ -84,34 +87,58 @@ public class TestCoin {
             }
 
             try {
-                Connection.Response response = JsoupUtil.requestBody(coinWatchlist.getCollectUrl(), JsoupUtil.HTTP_GET, Proxy.NO_PROXY, new HashMap<>(), new HashMap<>());
+                Connection.Response response = JsoupUtil.requestBody(coinWatchlist.getCollectUrl(), JsoupUtil.HTTP_GET, proxy, new HashMap<>(), new HashMap<>());
                 JSONObject jsonObject = JSONObject.parseObject(response.body());
 
                 JSONObject marketData = jsonObject.getJSONObject("market_data");
                 // 总市值排名
-                Integer totalMarketRanking = marketData.getInteger("market_cap_rank");
-                coinWatchlist.setTotalMarketRanking(totalMarketRanking);
+                if (marketData.containsKey("market_cap_rank") && null != marketData.get("market_cap_rank")) {
+                    Integer totalMarketRanking = marketData.getInteger("market_cap_rank");
+                    coinWatchlist.setTotalMarketRanking(totalMarketRanking);
+                }
+
                 // 总市值
-                BigDecimal totalMarketValue = marketData.getJSONObject("market_cap").getBigDecimal("usd");
-                coinWatchlist.setTotalMarketValue(totalMarketValue);
+                if (marketData.containsKey("market_cap") && null != marketData.get("market_cap")) {
+                    BigDecimal totalMarketValue = marketData.getJSONObject("market_cap").getBigDecimal("usd");
+                    coinWatchlist.setTotalMarketValue(totalMarketValue);
+                }
+
                 // 市场价格
-                String markPrice = marketData.getJSONObject("current_price").getString("usd");
-                coinWatchlist.setMarkPrice(markPrice);
+                if (marketData.containsKey("current_price") && null != marketData.get("current_price")) {
+                    String markPrice = marketData.getJSONObject("current_price").getString("usd");
+                    coinWatchlist.setMarkPrice(markPrice);
+                }
+
                 // 历史最高价格
-                String highestHistoricalPrice = marketData.getJSONObject("ath").getString("usd");
-                coinWatchlist.setHighestHistoricalPrice(highestHistoricalPrice);
+                if (marketData.containsKey("ath") && null != marketData.get("ath")) {
+                    String highestHistoricalPrice = marketData.getJSONObject("ath").getString("usd");
+                    coinWatchlist.setHighestHistoricalPrice(highestHistoricalPrice);
+                }
+
                 // 历史最高日期
-                LocalDate highestHistoricalDate = LocalDate.parse(marketData.getJSONObject("ath_date").getString("usd"), DateUtils.utcTimeFormatter);
-                coinWatchlist.setHighestHistoricalDate(highestHistoricalDate);
+                if (marketData.containsKey("ath_date") && null != marketData.get("ath_date")) {
+                    LocalDate highestHistoricalDate = LocalDate.parse(marketData.getJSONObject("ath_date").getString("usd"), DateUtils.utcTimeFormatter);
+                    coinWatchlist.setHighestHistoricalDate(highestHistoricalDate);
+                }
+
                 // 历史最低价格
-                String lowestHistoricalPrice = marketData.getJSONObject("atl").getString("usd");
-                coinWatchlist.setLowestHistoricalPrice(lowestHistoricalPrice);
+                if (marketData.containsKey("atl") && null != marketData.get("atl")) {
+                    String lowestHistoricalPrice = marketData.getJSONObject("atl").getString("usd");
+                    coinWatchlist.setLowestHistoricalPrice(lowestHistoricalPrice);
+                }
+
                 // 历史最低日期
-                LocalDate lowestHistoricalDate = LocalDate.parse(marketData.getJSONObject("atl_date").getString("usd"), DateUtils.utcTimeFormatter);
-                coinWatchlist.setLowestHistoricalDate(lowestHistoricalDate);
+                if (marketData.containsKey("atl_date") && null != marketData.get("atl_date")) {
+                    LocalDate lowestHistoricalDate = LocalDate.parse(marketData.getJSONObject("atl_date").getString("usd"), DateUtils.utcTimeFormatter);
+                    coinWatchlist.setLowestHistoricalDate(lowestHistoricalDate);
+                }
+
                 // 涨幅倍数
-                BigDecimal increaseMultiple = new BigDecimal(highestHistoricalPrice).divide(new BigDecimal(lowestHistoricalPrice), 0, RoundingMode.HALF_UP);
-                coinWatchlist.setIncreaseMultiple(increaseMultiple.intValue());
+                if (StringUtils.isNotEmpty(coinWatchlist.getHighestHistoricalPrice()) && StringUtils.isNotEmpty(coinWatchlist.getLowestHistoricalPrice())) {
+                    BigDecimal increaseMultiple = new BigDecimal(coinWatchlist.getHighestHistoricalPrice()).divide(new BigDecimal(coinWatchlist.getLowestHistoricalPrice()), 0, RoundingMode.HALF_UP);
+                    coinWatchlist.setIncreaseMultiple(increaseMultiple.intValue());
+                }
+
                 // 发行日期
                 // 发行天数
                 if (coinWatchlist.getIssuingDate() != null) {