wangliwen
/
langchain4j-aideepin
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

知识库增加统计数据

This commit is contained in:
moyangzhan 2024-05-21 19:09:30 +08:00
parent f4fa8d026f
commit d243e43684
16 changed files with 134 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -65,7 +65,7 @@ vue3+typescript+pnpm
* 创建数据库aideepin
* 执行docs/create.sql
* 填充各模型的配置(至少设置一个)
* 填充各AI平台的配置(至少设置一个)
openai的secretKey

6
adi-common/src/main/java/com/moyz/adi/common/cosntant/RedisKeyConstant.java
View File

@ -91,4 +91,10 @@ public class RedisKeyConstant {
* 值: 用户id
*/
public static final String qa_item_create_limit = "aq:item:create:{0}";
/**
* 重新统计知识库信号
* 值:知识库uuid
*/
public static final String KB_STATISTIC_RECALCULATE_SIGNAL = "kb:statistic:recalculate:signal";
}

2
adi-common/src/main/java/com/moyz/adi/common/dto/KbInfoResp.java
View File

@ -11,4 +11,6 @@ public class KbInfoResp {
private Integer starCount;
private String ownerUuid;
private String ownerName;
private Integer itemCount;
private Integer embeddingCount;
}

1
adi-common/src/main/java/com/moyz/adi/common/dto/KbItemDto.java
View File

@ -1,6 +1,5 @@
package com.moyz.adi.common.dto;
import com.baomidou.mybatisplus.annotation.TableField;
import lombok.Data;
import java.time.LocalDateTime;

9
adi-common/src/main/java/com/moyz/adi/common/dto/KbStatDto.java Normal file
View File

@ -0,0 +1,9 @@
package com.moyz.adi.common.dto;
import lombok.Data;
@Data
public class KbStatDto {
int itemCount;
int embeddingCount;
}

8
adi-common/src/main/java/com/moyz/adi/common/entity/KnowledgeBase.java
View File

@ -30,6 +30,14 @@ public class KnowledgeBase extends BaseEntity {
@TableField("star_count")
private Integer starCount;
@Schema(title = "知识点数量")
@TableField("item_count")
private Integer itemCount;
@Schema(title = "向量数")
@TableField("embedding_count")
private Integer embeddingCount;
@Schema(title = "所属人uuid")
@TableField("owner_uuid")
private String ownerUuid;

2
adi-common/src/main/java/com/moyz/adi/common/mapper/KnowledgeBaseEmbeddingMapper.java
View File

@ -14,4 +14,6 @@ public interface KnowledgeBaseEmbeddingMapper extends BaseMapper<KnowledgeBaseEm
Page<KnowledgeBaseEmbedding> selectByItemUuid(Page<KnowledgeBaseEmbedding> page, @Param("kbItemUuid") String uuid);
boolean deleteByItemUuid(@Param("kbItemUuid") String uuid);
Integer countByKbUuid(@Param("kbUuid") String kbUuid);
}

2
adi-common/src/main/java/com/moyz/adi/common/mapper/KnowledgeBaseItemMapper.java
View File

@ -13,4 +13,6 @@ public interface KnowledgeBaseItemMapper extends BaseMapper<KnowledgeBaseItem> {
@InterceptorIgnore(tenantLine = "true")
Page<KbItemDto> searchByKb(Page<KbItemDto> page, @Param("kbUuid") String kbUuid, @Param("keyword") String keyword);
KnowledgeBaseItem getByUuid(String uuid);
}

7
adi-common/src/main/java/com/moyz/adi/common/mapper/KnowledgeBaseMapper.java
View File

@ -25,4 +25,11 @@ public interface KnowledgeBaseMapper extends BaseMapper<KnowledgeBase> {
* @return
*/
Page<KnowledgeBase> searchByUser(Page<KnowledgeBase> page, @Param("ownerId") long ownerId, @Param("keyword") String keyword, @Param("includeOthersPublic") Boolean includeOthersPublic);
/**
* 更新统计数据
*
* @param uuid
*/
void updateStatByUuid(@Param("uuid") String uuid);
}

4
adi-common/src/main/java/com/moyz/adi/common/service/KnowledgeBaseEmbeddingService.java
View File

@ -32,4 +32,8 @@ public class KnowledgeBaseEmbeddingService extends ServiceImpl<KnowledgeBaseEmbe
public boolean deleteByItemUuid(String kbItemUuid) {
return baseMapper.deleteByItemUuid(kbItemUuid);
}
public Integer countByKbUuid(String kbUuid){
return baseMapper.countByKbUuid(kbUuid);
}
}

18
adi-common/src/main/java/com/moyz/adi/common/service/KnowledgeBaseItemService.java
View File

@ -61,6 +61,9 @@ public class KnowledgeBaseItemService extends ServiceImpl<KnowledgeBaseItemMappe
item.setId(itemEditReq.getId());
baseMapper.updateById(item);
}
knowledgeBaseService.updateStatistic(itemEditReq.getKbUuid());
return ChainWrappers.lambdaQueryChain(baseMapper)
.eq(KnowledgeBaseItem::getUuid, uuid)
.one();
@ -115,6 +118,8 @@ public class KnowledgeBaseItemService extends ServiceImpl<KnowledgeBaseItemMappe
.eq(KnowledgeBaseItem::getId, kbItem.getId())
.set(KnowledgeBaseItem::getIsEmbedded, true)
.update();
knowledgeBaseService.updateStatistic(kbItem.getKbUuid());
return true;
}
@ -130,9 +135,22 @@ public class KnowledgeBaseItemService extends ServiceImpl<KnowledgeBaseItemMappe
return false;
}
knowledgeBaseEmbeddingService.deleteByItemUuid(uuid);
KnowledgeBaseItem item = baseMapper.getByUuid(uuid);
if (null != item) {
knowledgeBaseService.updateStatistic(item.getKbUuid());
}
return true;
}
public int countByKbUuid(String kbUuid) {
return ChainWrappers.lambdaQueryChain(baseMapper)
.eq(KnowledgeBaseItem::getKbUuid, kbUuid)
.eq(KnowledgeBaseItem::getIsDeleted, false)
.count()
.intValue();
}
private boolean checkPrivilege(String uuid) {
if (StringUtils.isBlank(uuid)) {
throw new BaseException(A_PARAMS_ERROR);

31
adi-common/src/main/java/com/moyz/adi/common/service/KnowledgeBaseService.java
View File

@ -34,17 +34,21 @@ import org.apache.commons.lang3.tuple.Pair;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.text.MessageFormat;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.*;
import static com.moyz.adi.common.cosntant.AdiConstant.POI_DOC_TYPES;
import static com.moyz.adi.common.cosntant.AdiConstant.SysConfigKey.QUOTA_BY_QA_ASK_DAILY;
import static com.moyz.adi.common.cosntant.RedisKeyConstant.KB_STATISTIC_RECALCULATE_SIGNAL;
import static com.moyz.adi.common.enums.ErrorEnum.*;
import static dev.langchain4j.data.document.loader.FileSystemDocumentLoader.loadDocument;
@ -80,6 +84,9 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
@Resource
private UserDayCostService userDayCostService;
@Resource
private ThreadPoolTaskExecutor mainExecutor;
public KnowledgeBase saveOrUpdate(KbEditReq kbEditReq) {
String uuid = kbEditReq.getUuid();
KnowledgeBase knowledgeBase = new KnowledgeBase();
@ -180,6 +187,8 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
.eq(KnowledgeBaseItem::getId, knowledgeBaseItem.getId())
.set(KnowledgeBaseItem::getIsEmbedded, true)
.update();
updateStatistic(knowledgeBase.getUuid());
}
return adiFile;
} catch (Exception e) {
@ -308,6 +317,7 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
throw new BaseException(A_QA_ASK_LIMIT);
}
stringRedisTemplate.opsForValue().increment(key);
stringRedisTemplate.expire(key, Duration.ofDays(1));
}
@ -340,6 +350,27 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
.oneOpt().orElseThrow(() -> new BaseException(A_DATA_NOT_FOUND));
}
/**
* Set update knowledge base stat signal
*
* @param kbUuid
*/
public void updateStatistic(String kbUuid) {
stringRedisTemplate.opsForSet().add(KB_STATISTIC_RECALCULATE_SIGNAL, kbUuid);
}
/**
* Update knowledge base stat
*/
@Scheduled(fixedDelay = 60 * 1000)
public void asyncUpdateStatistic() {
Set<String> kbUuidList = stringRedisTemplate.opsForSet().members(KB_STATISTIC_RECALCULATE_SIGNAL);
for (String kbUuid : kbUuidList) {
baseMapper.updateStatByUuid(kbUuid);
stringRedisTemplate.opsForSet().remove(KB_STATISTIC_RECALCULATE_SIGNAL, kbUuid);
}
}
private void checkPrivilege(Long kbId, String kbUuid) {
if (null == kbId && StringUtils.isBlank(kbUuid)) {
throw new BaseException(A_PARAMS_ERROR);

9
adi-common/src/main/resources/mapper/KnowledgeBaseEmbeddingMapper.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.moyz.adi.common.mapper.KnowledgeBaseEmbeddingMapper">
<select id="countByKbUuid" resultType="java.lang.Integer">
select count(1)
from adi_knowledge_base_embedding
where metadata ->> 'kb_uuid' = #{kbUuid}
</select>
</mapper>

5
adi-common/src/main/resources/mapper/KnowledgeBaseItemMapper.xml
View File

@ -14,4 +14,9 @@
order by create_time desc
</select>
<select id="getByUuid" resultType="com.moyz.adi.common.entity.KnowledgeBaseItem">
select *
from adi_knowledge_base_item a
where uuid = #{uuid}
</select>
</mapper>

12
adi-common/src/main/resources/mapper/KnowledgeBaseMapper.xml
View File

@ -29,4 +29,16 @@
order by star_count,update_time desc
</select>
<update id="updateStatByUuid">
update adi_knowledge_base
set item_count = (select count(1)
from adi_knowledge_base_item
where kb_uuid = #{uuid}
and is_deleted = false),
embedding_count = (select count(1)
from adi_knowledge_base_embedding
where metadata ->> 'kb_uuid' = #{uuid})
where uuid = #{uuid}
</update>
</mapper>

8
docs/create.sql
View File

@ -441,10 +441,12 @@ create table adi_knowledge_base
title varchar(250) default ''::character varying not null,
remark text default ''::character varying not null,
is_public boolean default false not null,
star_count int default 0 not null,
owner_id bigint default 0 not null,
owner_uuid varchar(32) default ''::character varying not null,
owner_name varchar(45) default ''::character varying not null,
star_count int default 0 not null,
item_count int default 0 not null,
embedding_count int default 0 not null,
create_time timestamp default CURRENT_TIMESTAMP not null,
update_time timestamp default CURRENT_TIMESTAMP not null,
is_deleted boolean default false not null
@ -460,6 +462,10 @@ comment on column adi_knowledge_base.is_public is '是否公开';
comment on column adi_knowledge_base.star_count is '点赞数';
comment on column adi_knowledge_base.item_count is '知识点数量';
comment on column adi_knowledge_base.embedding_count is '向量数';
comment on column adi_knowledge_base.owner_id is '所属人id';
comment on column adi_knowledge_base.owner_uuid is '所属人uuid';