From 04db7c6dc5fdc267bc01414f5fc300620f235eb1 Mon Sep 17 00:00:00 2001 From: huangweixiong Date: Wed, 3 Aug 2022 14:24:35 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E7=BD=91=E5=85=B3=E9=87=8D=E5=90=AF?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=A4=B1=E6=95=88=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gateway/dao/ApiCountHistoryDao.java | 9 ++ .../gateway/entity/ApiCountHistoryEntity.java | 41 +++++++++ .../gateway/service/MonitorServiceV2.java | 83 +++++++++++++++++-- .../io/renren/GatewaySyncServiceTest.java | 33 ++++++++ 4 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 renren-admin/src/main/java/io/renren/modules/gateway/dao/ApiCountHistoryDao.java create mode 100644 renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java create mode 100644 renren-admin/src/test/java/io/renren/GatewaySyncServiceTest.java diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/dao/ApiCountHistoryDao.java b/renren-admin/src/main/java/io/renren/modules/gateway/dao/ApiCountHistoryDao.java new file mode 100644 index 00000000..554c6d01 --- /dev/null +++ b/renren-admin/src/main/java/io/renren/modules/gateway/dao/ApiCountHistoryDao.java @@ -0,0 +1,9 @@ +package io.renren.modules.gateway.dao; + +import io.renren.common.dao.BaseDao; +import io.renren.modules.gateway.entity.ApiCountHistoryEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ApiCountHistoryDao extends BaseDao { +} diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java b/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java new file mode 100644 index 00000000..cf2aad7f --- /dev/null +++ b/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java @@ -0,0 +1,41 @@ +package io.renren.modules.gateway.entity; + +import com.baomidou.mybatisplus.annotation.TableName; +import io.renren.common.entity.BaseEntity; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.io.Serializable; + + + +/** + * 数据同步算法 + * 数据库需要存两个值 1: 上一次来监控系统拉取的 "历史调用总数" A，每次拉取都覆盖 + * 2. 第二个值：拉取"历史调用总数"A1后，跟存储的"上一次来监控系统拉取的 历史调用总数"A比较， + * 如果新拉取的(A1)较小(代表了网关重启了)，先把"上一次拉取的 历史调用总数"A存储起来， + * 每次出现这种情况都存储一份，记做 B1，B2, ... Bn， 存储完之后使用A1的值覆盖A + * 最终界面展示"历史调用总数"时，值的算法：B1 + B2 +... + Bn + A + * + * 实体类A对应currentCount，B对应historyCount + */ +@Data +@EqualsAndHashCode(callSuper = false) +@TableName(value = "t_api_count_history", autoResultMap = true) +public class ApiCountHistoryEntity extends BaseEntity implements Serializable { + + /** + * 数据版本递增 + */ + private Long version; + + /** + * 当前数据量 + */ + private Long currentCount; + + /** + * 历史数据量 + */ + private Long historyCount; +} diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java b/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java index 88265323..a9df81a3 100644 --- a/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java +++ b/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java @@ -1,16 +1,17 @@ package io.renren.modules.gateway.service; import cn.hutool.core.date.DateUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import io.renren.modules.gateway.dao.ApiCountHistoryDao; +import io.renren.modules.gateway.entity.ApiCountHistoryEntity; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Service public class MonitorServiceV2 { @@ -21,7 +22,10 @@ public class MonitorServiceV2 { @Autowired private RestTemplate restTemplate; - void fetchCallCount(){ + @Autowired + private ApiCountHistoryDao apiCountHistoryDao; + + public void fetchCallCount(){ Long currentTime = System.currentTimeMillis() / 1000; String query = "sum(apigateway_http_status)"; String url = gatewayDomain + "/juapi/metrics/api/v1/query?query={query}" + "&time=" + currentTime.toString(); @@ -32,7 +36,76 @@ public class MonitorServiceV2 { if (data != null) { List result = (List) data.get("result"); if (result.size() == 1) { + HashMap map = result.get(0); + List value = (List) map.get("value"); + if (value.size() == 2) { + String count = (String) value.get(1); + long currentCount = Long.parseLong(count); + handleDataCompare(currentCount); + } } } } + + /** + ** 数据同步算法 + * * 数据库需要存两个值 1: 上一次来监控系统拉取的 "历史调用总数" A，每次拉取都覆盖 + * * 2. 第二个值：拉取"历史调用总数"A1后，跟存储的"上一次来监控系统拉取的 历史调用总数"A比较， + * * 如果新拉取的(A1)较小(代表了网关重启了)，先把"上一次拉取的 历史调用总数"A存储起来， + * * 每次出现这种情况都存储一份，记做 B1，B2, ... Bn， 存储完之后使用A1的值覆盖A + * * 最终界面展示"历史调用总数"时，值的算法：B1 + B2 +... + Bn + A + * + * @param currentCount 网关那边当前的总数 + */ + public void handleDataCompare(Long currentCount) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.orderByDesc(ApiCountHistoryEntity::getVersion) + .last("LIMIT 1"); + ApiCountHistoryEntity apiCountHistoryEntity = apiCountHistoryDao.selectOne(queryWrapper); + if (apiCountHistoryEntity == null){ + apiCountHistoryEntity = new ApiCountHistoryEntity(); + apiCountHistoryEntity.setCurrentCount(currentCount); + apiCountHistoryEntity.setHistoryCount(0L); + apiCountHistoryEntity.setVersion(0L); + apiCountHistoryDao.insert(apiCountHistoryEntity); + }else if (currentCount < apiCountHistoryEntity.getCurrentCount()){ + //保存旧值 + apiCountHistoryEntity.setHistoryCount(apiCountHistoryEntity.getCurrentCount()); + apiCountHistoryEntity.setCreateDate(new Date()); + apiCountHistoryDao.updateById(apiCountHistoryEntity); + //新增记录 + ApiCountHistoryEntity newHistoryEntity = new ApiCountHistoryEntity(); + newHistoryEntity.setCurrentCount(currentCount); + newHistoryEntity.setHistoryCount(0L); + newHistoryEntity.setVersion(apiCountHistoryEntity.getVersion() + 1); + apiCountHistoryDao.insert(newHistoryEntity); + }else if (currentCount > apiCountHistoryEntity.getCurrentCount()){//相等时没有必要更新 + apiCountHistoryEntity.setCurrentCount(currentCount); + apiCountHistoryDao.updateById(apiCountHistoryEntity); + } + } + + /** + * 查询历史总数 + * @return + */ + public Long getCallCount(){ + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.orderByDesc(ApiCountHistoryEntity::getVersion) + .last("LIMIT 1"); + ApiCountHistoryEntity apiCountHistoryEntity = apiCountHistoryDao.selectOne(queryWrapper); + if (apiCountHistoryEntity != null) { + Long currentCount = apiCountHistoryEntity.getCurrentCount(); + QueryWrapper hisQueryWrapper = new QueryWrapper<>(); + hisQueryWrapper.select("sum(history_count) as history_count"); + ApiCountHistoryEntity historyEntity = apiCountHistoryDao.selectOne(hisQueryWrapper); + if (historyEntity != null) { + return historyEntity.getHistoryCount() + currentCount; + } + } + + return 0L; + } + + } diff --git a/renren-admin/src/test/java/io/renren/GatewaySyncServiceTest.java b/renren-admin/src/test/java/io/renren/GatewaySyncServiceTest.java new file mode 100644 index 00000000..49719946 --- /dev/null +++ b/renren-admin/src/test/java/io/renren/GatewaySyncServiceTest.java @@ -0,0 +1,33 @@ +package io.renren; + +import cn.hutool.core.lang.UUID; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import io.renren.modules.gateway.service.MonitorServiceV2; +import io.renren.modules.processForm.service.ApiGatewayService; +import io.renren.modules.resource.dao.ResourceDao; +import io.renren.modules.resource.entity.ResourceEntity; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public class GatewaySyncServiceTest { + @Autowired + private MonitorServiceV2 monitorServiceV2; + + @Test + public void syncCount() { + + Integer[] dataSet = {10, 11, 50, 30, 44, 2,44,9}; + for (Integer count : dataSet) { + monitorServiceV2.handleDataCompare(Long.valueOf(count)); + } + } + +} \ No newline at end of file From 8e69ca0526d86e6e9e6d0b7ebf62af2a5e6dab62 Mon Sep 17 00:00:00 2001 From: huangweixiong Date: Wed, 3 Aug 2022 18:04:47 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E7=BD=91=E5=85=B3=E6=80=BB=E6=95=B0?= =?UTF-8?q?=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/db/V4.6__gateway_add_table.sql | 10 ++++++++ .../gateway/controller/MonitorController.java | 11 +++++++++ .../gateway/entity/ApiCountHistoryEntity.java | 16 +++++++++++-- .../gateway/service/MonitorServiceV2.java | 23 ++++++++++++++++++- .../src/main/resources/application.yml | 1 + 5 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 config/db/V4.6__gateway_add_table.sql diff --git a/config/db/V4.6__gateway_add_table.sql b/config/db/V4.6__gateway_add_table.sql new file mode 100644 index 00000000..036be543 --- /dev/null +++ b/config/db/V4.6__gateway_add_table.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS `t_api_count_history`; +CREATE TABLE `t_api_count_history` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `version` bigint COMMENT '数据版本号，从0递增', + `current_count`bigint COMMENT '当前总数，只有最新一条有效', + `history_count`bigint COMMENT '保存历史重启的数量', + `create_time` datetime NULL DEFAULT NULL, + `update_time` datetime NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB COMMENT = '网关历史调用总数'; diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java b/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java index 1a5dac6c..5ed9c1d0 100644 --- a/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java +++ b/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java @@ -5,6 +5,7 @@ import cn.hutool.core.net.URLEncoder; import cn.hutool.core.util.URLUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import io.renren.modules.gateway.service.MonitorServiceV2; import io.renren.modules.monitor.entity.Result; import io.renren.modules.resource.dao.AttrDao; import io.renren.modules.resource.dao.ResourceDao; @@ -62,6 +63,9 @@ public class MonitorController { @Autowired private SysDeptDao sysDeptDao; + @Autowired + private MonitorServiceV2 monitorServiceV2; + // @RequestMapping("/metrics/**") void forward(HttpServletRequest request, HttpServletResponse response) throws IOException { //类似nginx匹配前缀 @@ -326,4 +330,11 @@ public class MonitorController { return Result.success(selectCount); } + @GetMapping("/gateway-monitor/getCallCount") + @ApiOperation("查询总api调用总量") + public Result getCallCount(){ + Long callCount = monitorServiceV2.getCallCount(); + return Result.success(callCount); + } + } diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java b/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java index cf2aad7f..8a1a8fe9 100644 --- a/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java +++ b/renren-admin/src/main/java/io/renren/modules/gateway/entity/ApiCountHistoryEntity.java @@ -1,12 +1,15 @@ package io.renren.modules.gateway.entity; +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import io.renren.common.entity.BaseEntity; import lombok.Data; import lombok.EqualsAndHashCode; import java.io.Serializable; - +import java.util.Date; /** @@ -22,7 +25,11 @@ import java.io.Serializable; @Data @EqualsAndHashCode(callSuper = false) @TableName(value = "t_api_count_history", autoResultMap = true) -public class ApiCountHistoryEntity extends BaseEntity implements Serializable { +public class ApiCountHistoryEntity implements Serializable { + + @TableId + private Long id; + /** * 数据版本递增 @@ -38,4 +45,9 @@ public class ApiCountHistoryEntity extends BaseEntity implements Serializable { * 历史数据量 */ private Long historyCount; + + @TableField(fill = FieldFill.INSERT) + private Date createTime; + + private Date updateTime; } diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java b/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java index a9df81a3..ed437491 100644 --- a/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java +++ b/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java @@ -8,10 +8,15 @@ import io.renren.modules.gateway.entity.ApiCountHistoryEntity; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import javax.annotation.PostConstruct; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; @Service public class MonitorServiceV2 { @@ -19,12 +24,28 @@ public class MonitorServiceV2 { @Value("${hisense.gateway.url}") private String gatewayDomain; + @Value("${hisense.gateway.sync-enabled}") + private Boolean enableSync; + @Autowired private RestTemplate restTemplate; @Autowired private ApiCountHistoryDao apiCountHistoryDao; + @PostConstruct + public void init(){ + if (enableSync) { + ScheduledExecutorService service = Executors.newScheduledThreadPool(1); + service.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + fetchCallCount(); + } + }, 0, 60, TimeUnit.SECONDS); + } + } + public void fetchCallCount(){ Long currentTime = System.currentTimeMillis() / 1000; String query = "sum(apigateway_http_status)"; @@ -71,7 +92,7 @@ public class MonitorServiceV2 { }else if (currentCount < apiCountHistoryEntity.getCurrentCount()){ //保存旧值 apiCountHistoryEntity.setHistoryCount(apiCountHistoryEntity.getCurrentCount()); - apiCountHistoryEntity.setCreateDate(new Date()); + apiCountHistoryEntity.setUpdateTime(new Date()); apiCountHistoryDao.updateById(apiCountHistoryEntity); //新增记录 ApiCountHistoryEntity newHistoryEntity = new ApiCountHistoryEntity(); diff --git a/renren-admin/src/main/resources/application.yml b/renren-admin/src/main/resources/application.yml index c365e361..b8ce003a 100644 --- a/renren-admin/src/main/resources/application.yml +++ b/renren-admin/src/main/resources/application.yml @@ -14,6 +14,7 @@ census: # 海信网关 hisense: gateway: + sync-enabled: false url: http://devtest-security-app.hismarttv.com:8080 # Tomcat server: