From cc21e26ab6e499d7226887f8dcd12a1909b5e6e6 Mon Sep 17 00:00:00 2001 From: wangliwen Date: Mon, 13 Jun 2022 15:11:18 +0800 Subject: [PATCH] =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/ResourceController.java | 150 +------------- .../resource/service/ResourceService.java | 7 +- .../service/impl/ResourceServiceImpl.java | 192 ++++++++++++++++++ .../resource/task/KnowledgeBaseTask.java | 29 +++ 4 files changed, 230 insertions(+), 148 deletions(-) create mode 100644 renren-admin/src/main/java/io/renren/modules/resource/task/KnowledgeBaseTask.java diff --git a/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java b/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java index ccc09718..7d9b4cbe 100644 --- a/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java +++ b/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java @@ -1,7 +1,6 @@ package io.renren.modules.resource.controller; import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import io.renren.common.annotation.LogOperation; import io.renren.common.constant.Constant; @@ -11,13 +10,11 @@ import io.renren.common.validator.ValidatorUtils; import io.renren.common.validator.group.AddGroup; import io.renren.common.validator.group.DefaultGroup; import io.renren.modules.resource.dto.ResourceDTO; -import io.renren.modules.resource.entity.AttrEntity; import io.renren.modules.resource.service.ResourceService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiOperation; -import okhttp3.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,19 +24,12 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.*; import org.springframework.web.client.RestTemplate; import springfox.documentation.annotations.ApiIgnore; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; +import java.util.HashMap; +import java.util.Map; /** * 资源表 @@ -279,141 +269,7 @@ public class ResourceController { @GetMapping("knowledgeBase") @ApiOperation("对接知识库数据") public Result knowledgeBase() { - final int pageSize = 100; - final long timestamp = LocalDateTime.now().toInstant(ZoneOffset.ofHours(8)).toEpochMilli(); - final OkHttpClient client = new OkHttpClient(); - Arrays.stream(catalogIds).map(index -> { - logger.info("处理：" + index); - CopyOnWriteArrayList task = new CopyOnWriteArrayList<>(); - AtomicBoolean end = new AtomicBoolean(true); - AtomicInteger pageIndex = new AtomicInteger(1); - AtomicInteger maxPage = new AtomicInteger(100); // 防止死循环 - do { - int page = pageIndex.getAndIncrement(); - logger.info("处理：" + index + " 分页{}", page); - task.add(CompletableFuture.supplyAsync(() -> { - try { - logger.info("分页任务处理：" + index + " 分页{} 时间 -->" + timestamp, page); - JSONObject bizContentParam = new JSONObject(); - bizContentParam.put("appkey", appKey); - bizContentParam.put("catalogId", index); - bizContentParam.put("pageIndex", page); - bizContentParam.put("pageSize", pageSize); - String bizContent_ = bizContentParam.toJSONString(); - logger.info("biz_content参数：{}", bizContent_); - // 通过FormBody对象构建Builder来添加表单参数 - FormBody.Builder signFormBody = new FormBody.Builder() - .add("app_id", appId) - .add("interface_id", methodId) - .add("version", version) - .add("timestamp", String.valueOf(timestamp)) - .add("origin", origin) - .add("charset", charset) - .add("biz_content", bizContent_); - logger.info(index + "分页{}对接知识库数据请求参数：" + signFormBody.build().contentType().toString(), page); - - Request signRequest = new Request.Builder().url(sign).post(signFormBody.build()).build(); - Response signResponse = - client.newCall(signRequest).execute(); - String signResult = signResponse.body().string(); - logger.info("{}分页signResult数据：" + signResult, page); - JSONObject signJsonObject = JSON.parseObject(signResult); - if (!signJsonObject.containsKey("data")) { - logger.info("获取sign异常：" + signResult); - end.set(false); - throw new RuntimeException("获取sign异常"); - } - if (signJsonObject.get("data") == null) { - logger.info("获取sign异常：" + signResult); - end.set(false); - throw new RuntimeException("获取sign异常"); - } - Map signData = (Map) signJsonObject.get("data"); - String signString = signData.get("sign").toString(); - - signFormBody.add("sign", signString); - Request gatewayRequest = new Request.Builder().url(gateway).post(signFormBody.build()).build(); - Response gatewayResponse = client.newCall(gatewayRequest).execute(); - String gatewayResult = gatewayResponse.body().string(); - logger.info("{}分页数据：" + gatewayResult, page); - JSONObject gatewayJsonObject = JSON.parseObject(gatewayResult); - JSONObject gatewayData = JSON.parseObject(gatewayJsonObject.get("data").toString()); - JSONArray infos = gatewayData.getJSONObject("data").getJSONArray("infos"); - int total = gatewayData.getJSONObject("data").getIntValue("total"); - if (maxPage.get() != (total / pageSize)) { - maxPage.set(total / pageSize); - } - if (!infos.isEmpty()) { // 不为空则数据分页还没传输结束 - logger.info("--继续分页--"); - end.set(true); // TODO - } else { - end.set(false); - } - return infos; - } catch (Exception e) { - logger.info("获取异常：", e); - end.set(false); - return new JSONArray(); - } - }).thenAcceptAsync(list -> { - logger.info("知识库数据量:{}", list.size()); - if (list.size() < 1) { - end.set(false); - } - list.parallelStream().forEach(resource -> { - Map map = (Map) resource; - ResourceDTO dto = new ResourceDTO(); - dto.setName(map.get("title").toString()); - dto.setType("知识库"); - dto.setVisits(0L); - //所属部门暂时设为青岛市政府办公厅 - dto.setDeptId(1517116100113850370L); - dto.setNote1(map.get("uuid").toString()); - dto.setDelFlag(0); - ArrayList infoList = new ArrayList<>(); - map.forEach((key, value) -> { - switch (key) { - case "title": - dto.setName(value.toString()); - break; - case "url": - dto.setLink(value.toString()); - break; - case "createtime": - Date createDate = new Date(Long.parseLong(value.toString())); - dto.setCreateDate(createDate); - break; - default: - AttrEntity attrEntity = new AttrEntity(); - attrEntity.setDelFlag(0); - attrEntity.setAttrType(key); - attrEntity.setAttrValue(value.toString()); - infoList.add(attrEntity); - break; - } - }); - AttrEntity attrEntity = new AttrEntity(); - attrEntity.setDelFlag(0); - attrEntity.setAttrType("文件类型"); - if ("f49561afc7204f008c4bb3cd821eb6ba".equals(index)) { - attrEntity.setAttrValue("政府公报"); - } else { - attrEntity.setAttrValue("政策解读"); - } - infoList.add(attrEntity); - dto.setInfoList(infoList); - resourceService.insertWithAttrs(dto); - logger.info("插入：" + dto.getName()); - }); - })); - try { - Thread.sleep(500L); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } while (end.get() || pageIndex.get() < maxPage.get()); - return task; - }).collect(Collectors.toList()); + resourceService.KnowledgeBase(); return new Result().ok("任务开启成功！"); } diff --git a/renren-admin/src/main/java/io/renren/modules/resource/service/ResourceService.java b/renren-admin/src/main/java/io/renren/modules/resource/service/ResourceService.java index 343b33dd..6a38dba1 100644 --- a/renren-admin/src/main/java/io/renren/modules/resource/service/ResourceService.java +++ b/renren-admin/src/main/java/io/renren/modules/resource/service/ResourceService.java @@ -12,7 +12,7 @@ import java.util.Map; /** * 资源表 * - * @author dg + * @author dg * @since 1.0 2022-04-13 */ public interface ResourceService extends CrudService { @@ -74,4 +74,9 @@ public interface ResourceService extends CrudService implements ResourceService { + private static Logger logger = LoggerFactory.getLogger(ResourceServiceImpl.class); + @Value("${zsk.appid}") + private String appId; + + @Value("${zsk.appkey}") + private String appKey; + + @Value("${zsk.url.sign}") + private String sign; + + @Value("${zsk.url.gateway}") + private String gateway; + + @Value("${zsk.methodId}") + private String methodId; + + @Value("${zsk.param.charset}") + private String charset; + + @Value("${zsk.param.origin}") + private String origin; + + @Value("${zsk.param.version}") + private String version; + + @Value("${zsk.catalogIds}") + private String[] catalogIds; @Autowired private ResourceDao resourceDao; @@ -62,6 +103,9 @@ public class ResourceServiceImpl extends CrudServiceImpl getWrapper(Map params) { QueryWrapper wrapper = new QueryWrapper<>(); @@ -533,4 +577,152 @@ public class ResourceServiceImpl extends CrudServiceImpl knowledgeUUID = + jdbcTemplate.queryForList("SELECT note1 FROM tb_data_resource WHERE type ='知识库';", String.class) + .stream().distinct().collect(Collectors.toList()); + final int pageSize = 100; + final long timestamp = LocalDateTime.now().toInstant(ZoneOffset.ofHours(8)).toEpochMilli(); + final OkHttpClient client = new OkHttpClient(); + Arrays.stream(catalogIds).map(index -> { + logger.info("处理：" + index); + CopyOnWriteArrayList task = new CopyOnWriteArrayList<>(); + AtomicBoolean end = new AtomicBoolean(true); + AtomicInteger pageIndex = new AtomicInteger(1); + AtomicInteger maxPage = new AtomicInteger(100); // 防止死循环 + do { + int page = pageIndex.getAndIncrement(); + logger.info("处理：" + index + " 分页{}", page); + task.add(CompletableFuture.supplyAsync(() -> { + try { + logger.info("分页任务处理：" + index + " 分页{} 时间 -->" + timestamp, page); + JSONObject bizContentParam = new JSONObject(); + bizContentParam.put("appkey", appKey); + bizContentParam.put("catalogId", index); + bizContentParam.put("pageIndex", page); + bizContentParam.put("pageSize", pageSize); + String bizContent_ = bizContentParam.toJSONString(); + logger.info("biz_content参数：{}", bizContent_); + // 通过FormBody对象构建Builder来添加表单参数 + FormBody.Builder signFormBody = new FormBody.Builder() + .add("app_id", appId) + .add("interface_id", methodId) + .add("version", version) + .add("timestamp", String.valueOf(timestamp)) + .add("origin", origin) + .add("charset", charset) + .add("biz_content", bizContent_); + logger.info(index + "分页{}对接知识库数据请求参数：" + signFormBody.build().contentType().toString(), page); + + Request signRequest = new Request.Builder().url(sign).post(signFormBody.build()).build(); + Response signResponse = + client.newCall(signRequest).execute(); + String signResult = signResponse.body().string(); + logger.info("{}分页signResult数据：" + signResult, page); + JSONObject signJsonObject = JSON.parseObject(signResult); + if (!signJsonObject.containsKey("data")) { + logger.info("获取sign异常：" + signResult); + end.set(false); + throw new RuntimeException("获取sign异常"); + } + if (signJsonObject.get("data") == null) { + logger.info("获取sign异常：" + signResult); + end.set(false); + throw new RuntimeException("获取sign异常"); + } + Map signData = (Map) signJsonObject.get("data"); + String signString = signData.get("sign").toString(); + + signFormBody.add("sign", signString); + Request gatewayRequest = new Request.Builder().url(gateway).post(signFormBody.build()).build(); + Response gatewayResponse = client.newCall(gatewayRequest).execute(); + String gatewayResult = gatewayResponse.body().string(); + logger.info("{}分页数据：" + gatewayResult, page); + JSONObject gatewayJsonObject = JSON.parseObject(gatewayResult); + JSONObject gatewayData = JSON.parseObject(gatewayJsonObject.get("data").toString()); + JSONArray infos = gatewayData.getJSONObject("data").getJSONArray("infos"); + int total = gatewayData.getJSONObject("data").getIntValue("total"); + if (maxPage.get() != (total / pageSize)) { + maxPage.set(total / pageSize); + } + if (!infos.isEmpty()) { // 不为空则数据分页还没传输结束 + logger.info("--继续分页--"); + end.set(true); // TODO + } else { + end.set(false); + } + return infos; + } catch (Exception e) { + logger.info("获取异常：", e); + end.set(false); + return new JSONArray(); + } + }).thenAcceptAsync(list -> { + logger.info("知识库数据量:{}", list.size()); + if (list.size() < 1) { + end.set(false); + } + list.parallelStream().filter(resource -> { + Map map = (Map) resource; + return !knowledgeUUID.contains(map.get("uuid").toString()); + }).forEach(resource -> { + Map map = (Map) resource; + ResourceDTO dto = new ResourceDTO(); + dto.setName(map.get("title").toString()); + dto.setType("知识库"); + dto.setVisits(0L); + //所属部门暂时设为青岛市政府办公厅 + dto.setDeptId(1517116100113850370L); + dto.setNote1(map.get("uuid").toString()); + dto.setDelFlag(0); + ArrayList infoList = new ArrayList<>(); + map.forEach((key, value) -> { + switch (key) { + case "title": + dto.setName(value.toString()); + break; + case "url": + dto.setLink(value.toString()); + break; + case "createtime": + Date createDate = new Date(Long.parseLong(value.toString())); + dto.setCreateDate(createDate); + break; + default: + AttrEntity attrEntity = new AttrEntity(); + attrEntity.setDelFlag(0); + attrEntity.setAttrType(key); + attrEntity.setAttrValue(value.toString()); + infoList.add(attrEntity); + break; + } + }); + AttrEntity attrEntity = new AttrEntity(); + attrEntity.setDelFlag(0); + attrEntity.setAttrType("文件类型"); + if ("f49561afc7204f008c4bb3cd821eb6ba".equals(index)) { + attrEntity.setAttrValue("政府公报"); + } else { + attrEntity.setAttrValue("政策解读"); + } + infoList.add(attrEntity); + dto.setInfoList(infoList); + this.insertWithAttrs(dto); + logger.info("插入：" + dto.getName()); + }); + })); + try { + Thread.sleep(500L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } while (end.get() || pageIndex.get() < maxPage.get()); + return task; + }).collect(Collectors.toList()); + } + } \ No newline at end of file diff --git a/renren-admin/src/main/java/io/renren/modules/resource/task/KnowledgeBaseTask.java b/renren-admin/src/main/java/io/renren/modules/resource/task/KnowledgeBaseTask.java new file mode 100644 index 00000000..d5272648 --- /dev/null +++ b/renren-admin/src/main/java/io/renren/modules/resource/task/KnowledgeBaseTask.java @@ -0,0 +1,29 @@ +package io.renren.modules.resource.task; + +import io.renren.modules.job.task.ITask; +import io.renren.modules.resource.service.ResourceService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 知识库定时任务 + */ +@Component("knowledgeBaseTask") +public class KnowledgeBaseTask implements ITask { + private static Logger logger = LoggerFactory.getLogger(KnowledgeBaseTask.class); + @Autowired + private ResourceService resourceService; + + /** + * 执行定时任务接口 + * + * @param params 参数，多参数使用JSON数据 + */ + @Override + public void run(String params) { + logger.info("执行同步知识库任务"); + resourceService.KnowledgeBase(); + } +}