diff --git a/renren-admin/pom.xml b/renren-admin/pom.xml
index 5a032808..0848337f 100644
--- a/renren-admin/pom.xml
+++ b/renren-admin/pom.xml
@@ -38,7 +38,7 @@
1.5.7
linux-x86_64
5.0
- 4.9.3
+ 3.14.9
1.8
@@ -143,6 +143,12 @@
io.minio
minio
${minio.version}
+
+
+ okio
+ com.squareup.okio
+
+
org.springframework.boot
diff --git a/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java b/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java
index c5fc4d12..ccc09718 100644
--- a/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java
+++ b/renren-admin/src/main/java/io/renren/modules/resource/controller/ResourceController.java
@@ -17,10 +17,7 @@ import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
-import okhttp3.FormBody;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.Response;
+import okhttp3.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -30,12 +27,16 @@ import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
+import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import springfox.documentation.annotations.ApiIgnore;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
import java.util.*;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -278,54 +279,52 @@ public class ResourceController {
@GetMapping("knowledgeBase")
@ApiOperation("对接知识库数据")
public Result knowledgeBase() {
-
-// final MultiValueMap paramMap = new LinkedMultiValueMap<>();
-// paramMap.add("app_id", appId);
-// paramMap.add("interface_id", methodId);
-// paramMap.add("version", version);
-// paramMap.add("charset", charset);
-// paramMap.add("timestamp", String.valueOf(timestamp));
-// paramMap.add("origin", origin);
- int pageSize = 100;
- AtomicInteger pageIndex = new AtomicInteger(1);
+ final int pageSize = 100;
+ final long timestamp = LocalDateTime.now().toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
+ final OkHttpClient client = new OkHttpClient();
Arrays.stream(catalogIds).map(index -> {
- List task = new ArrayList<>();
- AtomicBoolean end = new AtomicBoolean(false);
+ logger.info("处理：" + index);
+ CopyOnWriteArrayList task = new CopyOnWriteArrayList<>();
+ AtomicBoolean end = new AtomicBoolean(true);
+ AtomicInteger pageIndex = new AtomicInteger(1);
+ AtomicInteger maxPage = new AtomicInteger(100); // 防止死循环
do {
int page = pageIndex.getAndIncrement();
+ logger.info("处理：" + index + " 分页{}", page);
task.add(CompletableFuture.supplyAsync(() -> {
- long timestamp = new Date().getTime();
- final JSONObject bizContentParam = new JSONObject();
- OkHttpClient client = new OkHttpClient();
- bizContentParam.put("appkey", appKey);
- bizContentParam.put("catalogId", index);
- bizContentParam.put("pageIndex", page);
- bizContentParam.put("pageSize", pageSize);
- String bizContent_ = bizContentParam.toJSONString();
- // 通过FormBody对象构建Builder来添加表单参数
- final FormBody.Builder signFormBody = new FormBody.Builder()
- .add("app_id", appId)
- .add("interface_id", methodId)
- .add("version", version)
- .add("timestamp", String.valueOf(timestamp))
- .add("origin", origin)
- .add("biz_content", bizContent_);
- logger.error(index + "分页{}对接知识库数据请求参数：" + signFormBody.build().contentType().toString(), page);
-
- Request signRequest = new Request.Builder().url(sign).post(signFormBody.build()).build();
try {
+ logger.info("分页任务处理：" + index + " 分页{} 时间 -->" + timestamp, page);
+ JSONObject bizContentParam = new JSONObject();
+ bizContentParam.put("appkey", appKey);
+ bizContentParam.put("catalogId", index);
+ bizContentParam.put("pageIndex", page);
+ bizContentParam.put("pageSize", pageSize);
+ String bizContent_ = bizContentParam.toJSONString();
+ logger.info("biz_content参数：{}", bizContent_);
+ // 通过FormBody对象构建Builder来添加表单参数
+ FormBody.Builder signFormBody = new FormBody.Builder()
+ .add("app_id", appId)
+ .add("interface_id", methodId)
+ .add("version", version)
+ .add("timestamp", String.valueOf(timestamp))
+ .add("origin", origin)
+ .add("charset", charset)
+ .add("biz_content", bizContent_);
+ logger.info(index + "分页{}对接知识库数据请求参数：" + signFormBody.build().contentType().toString(), page);
+
+ Request signRequest = new Request.Builder().url(sign).post(signFormBody.build()).build();
Response signResponse =
client.newCall(signRequest).execute();
String signResult = signResponse.body().string();
- logger.error("{}分页signResult数据：" + signResult, page);
+ logger.info("{}分页signResult数据：" + signResult, page);
JSONObject signJsonObject = JSON.parseObject(signResult);
if (!signJsonObject.containsKey("data")) {
- logger.error("获取sign异常：" + signResult);
+ logger.info("获取sign异常：" + signResult);
end.set(false);
throw new RuntimeException("获取sign异常");
}
if (signJsonObject.get("data") == null) {
- logger.error("获取sign异常：" + signResult);
+ logger.info("获取sign异常：" + signResult);
end.set(false);
throw new RuntimeException("获取sign异常");
}
@@ -333,26 +332,34 @@ public class ResourceController {
String signString = signData.get("sign").toString();
signFormBody.add("sign", signString);
- Request gatewayRequest = new Request.Builder().url(sign).post(signFormBody.build()).build();
+ Request gatewayRequest = new Request.Builder().url(gateway).post(signFormBody.build()).build();
Response gatewayResponse = client.newCall(gatewayRequest).execute();
String gatewayResult = gatewayResponse.body().string();
- logger.error("{}分页数据：" + gatewayResult, page);
+ logger.info("{}分页数据：" + gatewayResult, page);
JSONObject gatewayJsonObject = JSON.parseObject(gatewayResult);
JSONObject gatewayData = JSON.parseObject(gatewayJsonObject.get("data").toString());
JSONArray infos = gatewayData.getJSONObject("data").getJSONArray("infos");
+ int total = gatewayData.getJSONObject("data").getIntValue("total");
+ if (maxPage.get() != (total / pageSize)) {
+ maxPage.set(total / pageSize);
+ }
if (!infos.isEmpty()) { // 不为空则数据分页还没传输结束
- end.set(true);
+ logger.info("--继续分页--");
+ end.set(true); // TODO
} else {
end.set(false);
}
return infos;
} catch (Exception e) {
- logger.error("获取异常：", e);
+ logger.info("获取异常：", e);
end.set(false);
return new JSONArray();
}
}).thenAcceptAsync(list -> {
logger.info("知识库数据量:{}", list.size());
+ if (list.size() < 1) {
+ end.set(false);
+ }
list.parallelStream().forEach(resource -> {
Map map = (Map) resource;
ResourceDTO dto = new ResourceDTO();
@@ -361,6 +368,7 @@ public class ResourceController {
dto.setVisits(0L);
//所属部门暂时设为青岛市政府办公厅
dto.setDeptId(1517116100113850370L);
+ dto.setNote1(map.get("uuid").toString());
dto.setDelFlag(0);
ArrayList infoList = new ArrayList<>();
map.forEach((key, value) -> {
@@ -394,80 +402,19 @@ public class ResourceController {
}
infoList.add(attrEntity);
dto.setInfoList(infoList);
- logger.info("插入：" + dto.getName());
resourceService.insertWithAttrs(dto);
+ logger.info("插入：" + dto.getName());
});
}));
- } while (end.get());
+ try {
+ Thread.sleep(500L);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ } while (end.get() || pageIndex.get() < maxPage.get());
return task;
}).collect(Collectors.toList());
return new Result().ok("任务开启成功！");
-// CompletableFuture.allOf(completableFutures.stream().flatMap(i -> i.stream()).toArray(new CompletableFuture[completableFutures.size()])).join();
-// for (String catalogId : catalogIds) {
-// bizContent = "{\"appkey\":\"" + appKey + "\",\n" +
-// "\"catalogId\":\"" + catalogId + "\",\n" +
-// "\"pageIndex\":1,\n" +
-// "\"pageSize\":1000}";
-// paramMap.add("biz_content", bizContent);
-// try {
-// String signResult = restTemplate.postForObject(sign, paramMap, String.class);
-// JSONObject signJsonObject = JSON.parseObject(signResult);
-// Map signData = (Map) signJsonObject.get("data");
-// String signString = signData.get("sign").toString();
-// paramMap.add("sign", signString);
-// String gatewayResult = restTemplate.postForObject(gateway, paramMap, String.class);
-// JSONObject gatewayJsonObject = JSON.parseObject(gatewayResult);
-// JSONObject gatewayData = JSON.parseObject(gatewayJsonObject.get("data").toString());
-// JSONArray infos = gatewayData.getJSONObject("data").getJSONArray("infos");
-// infos.forEach(item -> {
-// Map map = (Map) item;
-// ResourceDTO dto = new ResourceDTO();
-// dto.setName(map.get("title").toString());
-// dto.setType("知识库");
-// dto.setVisits(0L);
-// //所属部门暂时设为青岛市政府办公厅
-// dto.setDeptId(1517116100113850370L);
-// dto.setDelFlag(0);
-// ArrayList infoList = new ArrayList<>();
-// map.forEach((key, value) -> {
-// switch (key) {
-// case "title":
-// dto.setName(value.toString());
-// break;
-// case "url":
-// dto.setLink(value.toString());
-// break;
-// case "createtime":
-// Date createDate = new Date(Long.parseLong(value.toString()));
-// dto.setCreateDate(createDate);
-// break;
-// default:
-// AttrEntity attrEntity = new AttrEntity();
-// attrEntity.setDelFlag(0);
-// attrEntity.setAttrType(key);
-// attrEntity.setAttrValue(value.toString());
-// infoList.add(attrEntity);
-// break;
-// }
-// });
-// AttrEntity attrEntity = new AttrEntity();
-// attrEntity.setDelFlag(0);
-// attrEntity.setAttrType("文件类型");
-// if ("f49561afc7204f008c4bb3cd821eb6ba".equals(catalogId)) {
-// attrEntity.setAttrValue("政府公报");
-// } else {
-// attrEntity.setAttrValue("政策解读");
-// }
-// infoList.add(attrEntity);
-// dto.setInfoList(infoList);
-// resourceService.insertWithAttrs(dto);
-// });
-// } catch (Exception e) {
-// e.printStackTrace();
-// }
-// paramMap.remove("sign");
-// paramMap.remove("biz_content");
-// }
}