diff --git a/renren-admin/src/main/java/io/renren/common/config/RestTemplateConfig.java b/renren-admin/src/main/java/io/renren/common/config/RestTemplateConfig.java index c68a171f..34ffd370 100644 --- a/renren-admin/src/main/java/io/renren/common/config/RestTemplateConfig.java +++ b/renren-admin/src/main/java/io/renren/common/config/RestTemplateConfig.java @@ -28,8 +28,8 @@ public class RestTemplateConfig { @Bean public ClientHttpRequestFactory simpleClientHttpRequestFactory() { SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); - factory.setReadTimeout(2000);//单位为ms - factory.setConnectTimeout(500);//单位为ms + factory.setReadTimeout(30 * 000);//单位为ms + factory.setConnectTimeout(5 * 1000);//单位为ms factory.setOutputStreaming(false); // SocketAddress address = new InetSocketAddress("127.0.0.1", 8888); diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java b/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java index df3f1689..7b5a0c65 100644 --- a/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java +++ b/renren-admin/src/main/java/io/renren/modules/gateway/controller/MonitorController.java @@ -1,8 +1,20 @@ package io.renren.modules.gateway.controller; +import cn.hutool.core.net.URLEncoder; +import cn.hutool.core.util.URLUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import io.renren.modules.monitor.entity.Result; +import io.renren.modules.resource.dao.AttrDao; +import io.renren.modules.resource.dao.ResourceDao; +import io.renren.modules.resource.entity.AttrEntity; +import io.renren.modules.resource.entity.ResourceEntity; +import io.renren.modules.sys.dao.SysDeptDao; +import io.renren.modules.sys.entity.SysDeptEntity; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import lombok.extern.log4j.Log4j; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -27,7 +39,7 @@ import java.net.URL; import java.net.URLConnection; import java.util.*; -@Controller +@RestController @Api(tags = "网关统计") @Log4j2 public class MonitorController { @@ -41,6 +53,15 @@ public class MonitorController { @Value("${server.servlet.context-path}") private String context_path; + @Autowired + private ResourceDao resourceDao; + + @Autowired + private AttrDao attrDao; + + @Autowired + private SysDeptDao sysDeptDao; + // @RequestMapping("/metrics/**") void forward(HttpServletRequest request, HttpServletResponse response) throws IOException { //类似nginx匹配前缀 @@ -136,4 +157,173 @@ public class MonitorController { e.printStackTrace(); } } + + @GetMapping("/gateway-monitor/queryGroupByAbility") + @ApiOperation("统计数据按能力归集") + public Result queryGroupByAbility(String query, String time){ + String url = gatewayDomain + "/juapi/metrics/api/v1/query?query={query}" + "&time=" + time; + ResponseEntity entity = restTemplate.getForEntity(url, HashMap.class, query); + HashMap body = entity.getBody(); + HashMap data = (HashMap) body.get("data"); + if (data != null){ + List result = (List) data.get("result"); + ArrayList results = new ArrayList<>(); + ArrayList abilityIds = new ArrayList<>(); + for (HashMap hashMap : result) { + Map metric = (Map) hashMap.get("metric"); + if (metric != null && metric.get("groupInfo") != null){ + List value = (List) hashMap.get("value"); + if (value.size() == 2){ + metric.put("count", value.get(1)); + try{ + Long groupInfo = Long.valueOf((String) metric.get("groupInfo")); + abilityIds.add(groupInfo); + metric.put("groupInfo", groupInfo); + results.add(metric); + }catch (Exception e) { + //忽略 + } + + } + } + } + + if (results.size() > 0) { + LambdaQueryWrapper queryWrapper = new QueryWrapper().lambda(); + queryWrapper + .select(ResourceEntity::getId, + ResourceEntity::getName, + ResourceEntity::getApiMethodType, + ResourceEntity::getType, + ResourceEntity::getApiUrl) + .in(ResourceEntity::getId, abilityIds); + List entities = resourceDao.selectList(queryWrapper); + for (Map map : results) { + Long groupInfo = (Long) map.get("groupInfo"); + for (ResourceEntity resourceEntity : entities) { + if (groupInfo != null && groupInfo.equals(resourceEntity.getId())) { + map.put("name", resourceEntity.getName()); + map.put("ApiMethodType", resourceEntity.getApiMethodType()); + map.put("type", resourceEntity.getType()); + map.put("apiUrl", resourceEntity.getApiUrl()); + LambdaQueryWrapper attrQueryWrapper = new LambdaQueryWrapper<>(); + attrQueryWrapper.select(AttrEntity::getAttrType,AttrEntity::getAttrValue) + .eq(AttrEntity::getDataResourceId,groupInfo) + .eq(AttrEntity::getAttrType,"服务商名") + .eq(AttrEntity::getDelFlag, 0); + AttrEntity attrEntity = attrDao.selectOne(attrQueryWrapper); + if (attrEntity.getAttrValue() != null) { + map.put("privider", attrEntity.getAttrValue()); + } + entities.remove(resourceEntity); + break; + } + } + } + } + + return Result.success(results); + } + + return Result.success(Collections.emptyList()); + } + + @GetMapping("/gateway-monitor/queryGroupByDepartment") + @ApiOperation("统计数据按部门归集") + public Result queryGroupByDepartment(String query, String time){ + String url = gatewayDomain + "/juapi/metrics/api/v1/query?query={query}" + "&time=" + time; + ResponseEntity entity = restTemplate.getForEntity(url, HashMap.class, query); + HashMap body = entity.getBody(); + HashMap data = (HashMap) body.get("data"); + if (data != null){ + List result = (List) data.get("result"); + ArrayList results = new ArrayList<>(); + ArrayList abilityIds = new ArrayList<>(); + for (HashMap hashMap : result) { + Map metric = (Map) hashMap.get("metric"); + if (metric != null && metric.get("deptInfo") != null){ + List value = (List) hashMap.get("value"); + if (value.size() == 2){ + metric.put("count", value.get(1)); + try { + Long groupInfo = Long.valueOf((String) metric.get("deptInfo")); + abilityIds.add(groupInfo); + metric.put("deptInfo", groupInfo); + results.add(metric); + }catch (Exception e) { + //忽略 + } + + } + } + } + + if (results.size() > 0) { + LambdaQueryWrapper queryWrapper = new QueryWrapper().lambda(); + queryWrapper.select(SysDeptEntity::getName,SysDeptEntity::getId) + .in(SysDeptEntity::getId, abilityIds); + List entities = sysDeptDao.selectList(queryWrapper); + for (Map map : results) { + Long deptInfo = (Long) map.get("deptInfo"); + for (SysDeptEntity sysDeptEntity : entities) { + if (deptInfo != null && deptInfo.equals(sysDeptEntity.getId())) { + map.put("name", sysDeptEntity.getName()); + entities.remove(sysDeptEntity); + break; + } + } + } + } + + return Result.success(results); + } + + return Result.success(Collections.emptyList()); + } + + @GetMapping("/gateway-monitor/queryGroupByDeptInRange") + @ApiOperation("统计数据按部门显示趋势") + public Result queryGroupByDeptInRange(String query, String start, String end, String step){ + String url = gatewayDomain + "/juapi/metrics/api/v1/query_range?query={query}" + + "&start=" + start + + "&end=" + end + + "&step=" + step; + + ResponseEntity entity = restTemplate.getForEntity(url, HashMap.class, query); + HashMap body = entity.getBody(); + HashMap data = (HashMap) body.get("data"); + if (data != null){ + List result = (List) data.get("result"); + ArrayList results = new ArrayList<>(result.size()); + for (HashMap hashMap : result) { + Map metric = (Map) hashMap.get("metric"); + if (metric != null && metric.get("deptInfo") != null){ + try{ + Long filterId = Long.valueOf((String) metric.get("deptInfo")); + SysDeptEntity sysDeptEntity = sysDeptDao.selectById(filterId); + if (sysDeptEntity != null && StringUtils.isNotBlank(sysDeptEntity.getName())){ + metric.put("name", sysDeptEntity.getName()); + results.add(hashMap); + } + }catch (Exception e){ + log.warn("数据异常忽略", e); + } + + } + } + return Result.success(results); + } + + return Result.success(Collections.emptyList()); + } + + @GetMapping("/gateway-monitor/queryGroupCount") + @ApiOperation("查询总api数量") + public Result queryGroupCount(){ + LambdaQueryWrapper queryWrapper = new QueryWrapper().lambda(); + queryWrapper.isNotNull(ResourceEntity::getGroupId).eq(ResourceEntity::getDelFlag, 0); + Integer selectCount = resourceDao.selectCount(queryWrapper); + return Result.success(selectCount); + } + } diff --git a/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java b/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java new file mode 100644 index 00000000..88265323 --- /dev/null +++ b/renren-admin/src/main/java/io/renren/modules/gateway/service/MonitorServiceV2.java @@ -0,0 +1,38 @@ +package io.renren.modules.gateway.service; + +import cn.hutool.core.date.DateUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class MonitorServiceV2 { + + @Value("${hisense.gateway.url}") + private String gatewayDomain; + + @Autowired + private RestTemplate restTemplate; + + void fetchCallCount(){ + Long currentTime = System.currentTimeMillis() / 1000; + String query = "sum(apigateway_http_status)"; + String url = gatewayDomain + "/juapi/metrics/api/v1/query?query={query}" + "&time=" + currentTime.toString(); + + ResponseEntity entity = restTemplate.getForEntity(url, HashMap.class, query); + HashMap body = entity.getBody(); + HashMap data = (HashMap) body.get("data"); + if (data != null) { + List result = (List) data.get("result"); + if (result.size() == 1) { + } + } + } +} diff --git a/renren-admin/src/main/java/io/renren/modules/processForm/listener/v2/CorrectionListenerV2.java b/renren-admin/src/main/java/io/renren/modules/processForm/listener/v2/CorrectionListenerV2.java index 47feab22..f28984c0 100644 --- a/renren-admin/src/main/java/io/renren/modules/processForm/listener/v2/CorrectionListenerV2.java +++ b/renren-admin/src/main/java/io/renren/modules/processForm/listener/v2/CorrectionListenerV2.java @@ -298,15 +298,19 @@ public class CorrectionListenerV2 implements TaskListener, ExecutionListener, Ac StringBuilder allMsg = new StringBuilder(); allMsg.append("您的能力申请已通过,访问令牌如下："); allMsg.append('\n'); + + boolean hasData = false; for (TAbilityApplicationDTO abilityApplicationDTO : dtoList) { ResourceEntity resourceEntity = resourceService.selectById(abilityApplicationDTO.getResourceId()); if (resourceEntity == null) { continue; } //没有groupid当做没有接口，直接跳过 - if (null == resourceEntity.getGroupId()) + if (resourceEntity == null || resourceEntity.getGroupId() == null) continue; + hasData = true; + String code = UUID.randomUUID().toString(); apiGatewayService.subscribeCode(String.valueOf(abilityApplicationDTO.getId()), code); String apiPrefix = "/juapi/" + abilityApplicationDTO.getResourceId(); @@ -315,6 +319,8 @@ public class CorrectionListenerV2 implements TaskListener, ExecutionListener, Ac allMsg.append('\n'); } + //一条也没有跳过 + if (!hasData) return; TaskService taskService = ProcessEngines.getDefaultProcessEngine().getTaskService(); diff --git a/renren-admin/src/main/java/io/renren/modules/processForm/service/ApiGatewayService.java b/renren-admin/src/main/java/io/renren/modules/processForm/service/ApiGatewayService.java index a62e962c..b174ed59 100644 --- a/renren-admin/src/main/java/io/renren/modules/processForm/service/ApiGatewayService.java +++ b/renren-admin/src/main/java/io/renren/modules/processForm/service/ApiGatewayService.java @@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.google.common.collect.Sets; import io.renren.modules.processForm.dao.TAbilityApplicationDao; import io.renren.modules.processForm.entity.TAbilityApplicationEntity; import io.renren.modules.resource.dao.ResourceDao; @@ -20,8 +21,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -65,8 +65,13 @@ public class ApiGatewayService { if (resourceEntity.getApiMethodType() != null) { methods = resourceEntity.getApiMethodType().toUpperCase(); } - if (StringUtils.isBlank(apiUrl) || StringUtils.isBlank(methods)){ - String msg = String.format("注册api参数为空，跳过 apiUrl:%s, methods:%s, resourceId:%s", apiUrl, methods, resourceId); + Long deptId = resourceEntity.getDeptId(); + + HashSet supportMethod = Sets.newHashSet("POST", "GET"); + + if (StringUtils.isBlank(apiUrl) || deptId == null || deptId == 0 || StringUtils.isBlank(methods) || !supportMethod.contains(methods)){ + String msg = String.format("注册api参数为空，跳过 apiUrl:%s, deptId:%ld methods:%s, resourceId:%s", apiUrl, deptId, methods, resourceId); + //重要参数没有当成不需要注册 log.info(msg); return; @@ -78,7 +83,7 @@ public class ApiGatewayService { if (StringUtils.isBlank(uris)) { uris = "/"; } - String apiPrefix = "/juapi/" + resourceId; + String apiPrefix = "/juapi/" + deptId + "/" + resourceId; HashMap groupEntity = new HashMap(); groupEntity.put("id", resourceId); groupEntity.put("name", resourceEntity.getName()); @@ -107,6 +112,7 @@ public class ApiGatewayService { routeEntity.put("group", id); routeEntity.put("methods", methods); routeEntity.put("uris", apiPrefix + uris); + routeEntity.put("enableMetric", true); ResponseEntity routeResEntity = restTemplate.postForEntity(routeUrl, routeEntity, HashMap.class); if (routeResEntity.getStatusCode() != HttpStatus.OK || !responseEntity.hasBody()){ //失败则删除group @@ -122,6 +128,34 @@ public class ApiGatewayService { } } + public void deleteGroup(String groupId) { + String groupUrl = gatewayUrl + "/apiops/api/groups"; + restTemplate.delete(groupUrl + "/" + groupId); + } + + public void resetApiGroup(String groupId){ + String apiQueryUrl = gatewayUrl + "/apiops/api/routers?group=" + groupId; + ResponseEntity forEntity = restTemplate.getForEntity(apiQueryUrl, HashMap.class); + HashMap body = forEntity.getBody(); + Boolean isEmpty = (Boolean) body.get("empty"); + if (!isEmpty) { + List content = (List) body.get("content"); + for (Map map : content) { + String id = (String) map.get("id"); + if (StringUtils.isNotBlank(id)) { + String apiDelUrl = gatewayUrl + "/apiops/api/routers/" + id; + try{ + restTemplate.delete(apiDelUrl); + }catch (Exception e){ + + } + + } + + } + } + } + /** * 将code关联到group，api这希望code由我们来生成 * 关联流程：创建消费者 -> 订阅接口传入code关联消费者与group @@ -145,6 +179,7 @@ public class ApiGatewayService { HashMap consumerEntity = new HashMap(); consumerEntity.put("id", formId); consumerEntity.put("name", resourceEntity.getName() + "-concumer"); + consumerEntity.put("code", code); String consumerUrl = gatewayUrl + "/apiops/api/consumers"; HashMap consumerResponse = restTemplate.postForEntity(consumerUrl, consumerEntity, HashMap.class).getBody(); @@ -157,7 +192,7 @@ public class ApiGatewayService { subscribeEntity.put("consumerId", formId); subscribeEntity.put("routerId", groupId); subscribeEntity.put("routerType","group"); - subscribeEntity.put("code", code); + String subscribeUrl = gatewayUrl + "/apiops/api/subscribers"; HashMap body = restTemplate.postForEntity(subscribeUrl, subscribeEntity, HashMap.class).getBody(); diff --git a/renren-admin/src/main/java/io/renren/modules/resource/dataResource/domain/TsingtaoDataResourceService.java b/renren-admin/src/main/java/io/renren/modules/resource/dataResource/domain/TsingtaoDataResourceService.java index 3b543c6d..ec8287aa 100644 --- a/renren-admin/src/main/java/io/renren/modules/resource/dataResource/domain/TsingtaoDataResourceService.java +++ b/renren-admin/src/main/java/io/renren/modules/resource/dataResource/domain/TsingtaoDataResourceService.java @@ -78,6 +78,7 @@ public class TsingtaoDataResourceService extends AbstractDataResourceService { return result; } catch (Exception e) { + logger.error("资源数据调用失败", e); e.printStackTrace(); return null; } diff --git a/renren-admin/src/main/java/io/renren/modules/security/config/ShiroConfig.java b/renren-admin/src/main/java/io/renren/modules/security/config/ShiroConfig.java index a41f72f3..ea1f4306 100644 --- a/renren-admin/src/main/java/io/renren/modules/security/config/ShiroConfig.java +++ b/renren-admin/src/main/java/io/renren/modules/security/config/ShiroConfig.java @@ -99,6 +99,8 @@ public class ShiroConfig { filterMap.put("/upload/**", "anon"); filterMap.put("/census/center/v3/**", "oauth2"); filterMap.put("/census/center/**", "anon"); // 全局各类统计 包含 /census/center/v2 + filterMap.put("/metrics/**", "anon"); + filterMap.put("/gateway-monitor/**", "anon"); filterMap.put("/**", "oauth2"); shiroFilter.setFilterChainDefinitionMap(filterMap); diff --git a/renren-admin/src/main/resources/application-dev.yml b/renren-admin/src/main/resources/application-dev.yml index e6fd0067..30e09a8f 100644 --- a/renren-admin/src/main/resources/application-dev.yml +++ b/renren-admin/src/main/resources/application-dev.yml @@ -6,7 +6,7 @@ spring: #MySQL driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://192.168.124.236:3306/share_platform?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&useSSL=false + url: jdbc:mysql://192.168.124.236:3306/share_platform?useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&useSSL=false username: root password: Hisense2019 #Hisense2019 diff --git a/renren-admin/src/test/java/io/renren/ApiGatewayServiceTest.java b/renren-admin/src/test/java/io/renren/ApiGatewayServiceTest.java index 84ec33b7..cb98e6f8 100644 --- a/renren-admin/src/test/java/io/renren/ApiGatewayServiceTest.java +++ b/renren-admin/src/test/java/io/renren/ApiGatewayServiceTest.java @@ -13,6 +13,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import java.util.List; @@ -40,20 +41,32 @@ public class ApiGatewayServiceTest { .like(ResourceEntity::getApiUrl,"http%"); List resourceEntities = resourceDao.selectList(select); resourceEntities.forEach(item -> { - apiGatewayService.registerApi2Gateway(String.valueOf(item.getId())); + String id = String.valueOf(item.getId()); + apiGatewayService.resetApiGroup(id); + apiGatewayService.deleteGroup(id); + apiGatewayService.registerApi2Gateway(id); }); } @Test public void registerAPI(){ - apiGatewayService.registerApi2Gateway("1522550194544123907"); + String id = "1522550194544123907"; + apiGatewayService.resetApiGroup(id); + try{ + apiGatewayService.deleteGroup(id); + }catch (Exception e){ + e.printStackTrace(); + } + + apiGatewayService.registerApi2Gateway(id); } @Test public void registerCode2Group() { String code = UUID.randomUUID().toString(); - apiGatewayService.subscribeCode("1523913824099762177", code); + apiGatewayService.subscribeCode("1522756586483789825", code); + System.out.println(code); } } \ No newline at end of file