知识库增加统计数据

README.md

@@ -65,7 +65,7 @@ vue3+typescript+pnpm
 
 * 创建数据库aideepin
 * 执行docs/create.sql
-* 填充各模型的配置(至少设置一个)
+* 填充各AI平台的配置(至少设置一个)
 
 openai的secretKey
 

adi-common/src/main/java/com/moyz/adi/common/cosntant/RedisKeyConstant.java

@@ -91,4 +91,10 @@ public class RedisKeyConstant {
      * 值: 用户id
      */
     public static final String qa_item_create_limit = "aq:item:create:{0}";
+
+    /**
+     * 重新统计知识库信号
+     * 值:知识库uuid
+     */
+    public static final String KB_STATISTIC_RECALCULATE_SIGNAL = "kb:statistic:recalculate:signal";
 }

adi-common/src/main/java/com/moyz/adi/common/dto/KbInfoResp.java

@@ -11,4 +11,6 @@ public class KbInfoResp {
     private Integer starCount;
     private String ownerUuid;
     private String ownerName;
+    private Integer itemCount;
+    private Integer embeddingCount;
 }

adi-common/src/main/java/com/moyz/adi/common/dto/KbItemDto.java

@@ -1,6 +1,5 @@
 package com.moyz.adi.common.dto;
 
-import com.baomidou.mybatisplus.annotation.TableField;
 import lombok.Data;
 
 import java.time.LocalDateTime;

adi-common/src/main/java/com/moyz/adi/common/dto/KbStatDto.java

@@ -0,0 +1,9 @@
+package com.moyz.adi.common.dto;
+
+import lombok.Data;
+
+@Data
+public class KbStatDto {
+    int itemCount;
+    int embeddingCount;
+}

adi-common/src/main/java/com/moyz/adi/common/entity/KnowledgeBase.java

@@ -30,6 +30,14 @@ public class KnowledgeBase extends BaseEntity {
     @TableField("star_count")
     private Integer starCount;
 
+    @Schema(title = "知识点数量")
+    @TableField("item_count")
+    private Integer itemCount;
+
+    @Schema(title = "向量数")
+    @TableField("embedding_count")
+    private Integer embeddingCount;
+
     @Schema(title = "所属人uuid")
     @TableField("owner_uuid")
     private String ownerUuid;

adi-common/src/main/java/com/moyz/adi/common/mapper/KnowledgeBaseEmbeddingMapper.java

@@ -14,4 +14,6 @@ public interface KnowledgeBaseEmbeddingMapper extends BaseMapper<KnowledgeBaseEm
     Page<KnowledgeBaseEmbedding> selectByItemUuid(Page<KnowledgeBaseEmbedding> page, @Param("kbItemUuid") String uuid);
 
     boolean deleteByItemUuid(@Param("kbItemUuid") String uuid);
+
+    Integer countByKbUuid(@Param("kbUuid") String kbUuid);
 }

adi-common/src/main/java/com/moyz/adi/common/mapper/KnowledgeBaseItemMapper.java

@@ -13,4 +13,6 @@ public interface KnowledgeBaseItemMapper extends BaseMapper<KnowledgeBaseItem> {
 
     @InterceptorIgnore(tenantLine = "true")
     Page<KbItemDto> searchByKb(Page<KbItemDto> page, @Param("kbUuid") String kbUuid, @Param("keyword") String keyword);
+
+    KnowledgeBaseItem getByUuid(String uuid);
 }

adi-common/src/main/java/com/moyz/adi/common/mapper/KnowledgeBaseMapper.java

@@ -25,4 +25,11 @@ public interface KnowledgeBaseMapper extends BaseMapper<KnowledgeBase> {
      * @return
      */
     Page<KnowledgeBase> searchByUser(Page<KnowledgeBase> page, @Param("ownerId") long ownerId, @Param("keyword") String keyword, @Param("includeOthersPublic") Boolean includeOthersPublic);
+
+    /**
+     * 更新统计数据
+     *
+     * @param uuid
+     */
+    void updateStatByUuid(@Param("uuid") String uuid);
 }

adi-common/src/main/java/com/moyz/adi/common/service/KnowledgeBaseEmbeddingService.java

@@ -32,4 +32,8 @@ public class KnowledgeBaseEmbeddingService extends ServiceImpl<KnowledgeBaseEmbe
     public boolean deleteByItemUuid(String kbItemUuid) {
         return baseMapper.deleteByItemUuid(kbItemUuid);
     }
+
+    public Integer countByKbUuid(String kbUuid){
+        return baseMapper.countByKbUuid(kbUuid);
+    }
 }

adi-common/src/main/java/com/moyz/adi/common/service/KnowledgeBaseItemService.java

@@ -61,6 +61,9 @@ public class KnowledgeBaseItemService extends ServiceImpl<KnowledgeBaseItemMappe
             item.setId(itemEditReq.getId());
             baseMapper.updateById(item);
         }
+
+        knowledgeBaseService.updateStatistic(itemEditReq.getKbUuid());
+
         return ChainWrappers.lambdaQueryChain(baseMapper)
                 .eq(KnowledgeBaseItem::getUuid, uuid)
                 .one();
@@ -115,6 +118,8 @@ public class KnowledgeBaseItemService extends ServiceImpl<KnowledgeBaseItemMappe
                 .eq(KnowledgeBaseItem::getId, kbItem.getId())
                 .set(KnowledgeBaseItem::getIsEmbedded, true)
                 .update();
+
+        knowledgeBaseService.updateStatistic(kbItem.getKbUuid());
         return true;
     }
 
@@ -130,9 +135,22 @@ public class KnowledgeBaseItemService extends ServiceImpl<KnowledgeBaseItemMappe
             return false;
         }
         knowledgeBaseEmbeddingService.deleteByItemUuid(uuid);
+
+        KnowledgeBaseItem item = baseMapper.getByUuid(uuid);
+        if (null != item) {
+            knowledgeBaseService.updateStatistic(item.getKbUuid());
+        }
         return true;
     }
 
+    public int countByKbUuid(String kbUuid) {
+        return ChainWrappers.lambdaQueryChain(baseMapper)
+                .eq(KnowledgeBaseItem::getKbUuid, kbUuid)
+                .eq(KnowledgeBaseItem::getIsDeleted, false)
+                .count()
+                .intValue();
+    }
+
     private boolean checkPrivilege(String uuid) {
         if (StringUtils.isBlank(uuid)) {
             throw new BaseException(A_PARAMS_ERROR);

adi-common/src/main/java/com/moyz/adi/common/service/KnowledgeBaseService.java

@@ -34,17 +34,21 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
 
 import java.text.MessageFormat;
+import java.time.Duration;
 import java.time.LocalDateTime;
 import java.util.*;
 
 import static com.moyz.adi.common.cosntant.AdiConstant.POI_DOC_TYPES;
 import static com.moyz.adi.common.cosntant.AdiConstant.SysConfigKey.QUOTA_BY_QA_ASK_DAILY;
+import static com.moyz.adi.common.cosntant.RedisKeyConstant.KB_STATISTIC_RECALCULATE_SIGNAL;
 import static com.moyz.adi.common.enums.ErrorEnum.*;
 import static dev.langchain4j.data.document.loader.FileSystemDocumentLoader.loadDocument;
 
@@ -80,6 +84,9 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
     @Resource
     private UserDayCostService userDayCostService;
 
+    @Resource
+    private ThreadPoolTaskExecutor mainExecutor;
+
     public KnowledgeBase saveOrUpdate(KbEditReq kbEditReq) {
         String uuid = kbEditReq.getUuid();
         KnowledgeBase knowledgeBase = new KnowledgeBase();
@@ -180,6 +187,8 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
                         .eq(KnowledgeBaseItem::getId, knowledgeBaseItem.getId())
                         .set(KnowledgeBaseItem::getIsEmbedded, true)
                         .update();
+
+                updateStatistic(knowledgeBase.getUuid());
             }
             return adiFile;
         } catch (Exception e) {
@@ -308,6 +317,7 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
             throw new BaseException(A_QA_ASK_LIMIT);
         }
         stringRedisTemplate.opsForValue().increment(key);
+        stringRedisTemplate.expire(key, Duration.ofDays(1));
     }
 
 
@@ -340,6 +350,27 @@ public class KnowledgeBaseService extends ServiceImpl<KnowledgeBaseMapper, Knowl
                 .oneOpt().orElseThrow(() -> new BaseException(A_DATA_NOT_FOUND));
     }
 
+    /**
+     * Set update knowledge base stat signal
+     *
+     * @param kbUuid
+     */
+    public void updateStatistic(String kbUuid) {
+        stringRedisTemplate.opsForSet().add(KB_STATISTIC_RECALCULATE_SIGNAL, kbUuid);
+    }
+
+    /**
+     * Update knowledge base stat
+     */
+    @Scheduled(fixedDelay = 60 * 1000)
+    public void asyncUpdateStatistic() {
+        Set<String> kbUuidList = stringRedisTemplate.opsForSet().members(KB_STATISTIC_RECALCULATE_SIGNAL);
+        for (String kbUuid : kbUuidList) {
+            baseMapper.updateStatByUuid(kbUuid);
+            stringRedisTemplate.opsForSet().remove(KB_STATISTIC_RECALCULATE_SIGNAL, kbUuid);
+        }
+    }
+
     private void checkPrivilege(Long kbId, String kbUuid) {
         if (null == kbId && StringUtils.isBlank(kbUuid)) {
             throw new BaseException(A_PARAMS_ERROR);

adi-common/src/main/resources/mapper/KnowledgeBaseEmbeddingMapper.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.moyz.adi.common.mapper.KnowledgeBaseEmbeddingMapper">
+    <select id="countByKbUuid" resultType="java.lang.Integer">
+        select count(1)
+        from adi_knowledge_base_embedding
+        where metadata ->> 'kb_uuid' = #{kbUuid}
+    </select>
+</mapper>

adi-common/src/main/resources/mapper/KnowledgeBaseItemMapper.xml

@@ -14,4 +14,9 @@
         order by create_time desc
     </select>
 
+    <select id="getByUuid" resultType="com.moyz.adi.common.entity.KnowledgeBaseItem">
+        select *
+        from adi_knowledge_base_item a
+        where uuid = #{uuid}
+    </select>
 </mapper>

adi-common/src/main/resources/mapper/KnowledgeBaseMapper.xml

@@ -29,4 +29,16 @@
         order by star_count,update_time desc
     </select>
 
+    <update id="updateStatByUuid">
+        update adi_knowledge_base
+        set item_count      = (select count(1)
+                               from adi_knowledge_base_item
+                               where kb_uuid = #{uuid}
+                                 and is_deleted = false),
+            embedding_count = (select count(1)
+                               from adi_knowledge_base_embedding
+                               where metadata ->> 'kb_uuid' = #{uuid})
+        where uuid = #{uuid}
+    </update>
+
 </mapper>

docs/create.sql

@@ -436,18 +436,20 @@ VALUES ('quota_by_qa_item_monthly', '100');
 
 create table adi_knowledge_base
 (
-    id          bigserial primary key,
+    id              bigserial primary key,
-    uuid        varchar(32)  default ''::character varying not null,
+    uuid            varchar(32)  default ''::character varying not null,
-    title       varchar(250) default ''::character varying not null,
+    title           varchar(250) default ''::character varying not null,
-    remark      text         default ''::character varying not null,
+    remark          text         default ''::character varying not null,
-    is_public   boolean      default false                 not null,
+    is_public       boolean      default false                 not null,
-    star_count  int          default 0                     not null,
+    owner_id        bigint       default 0                     not null,
-    owner_id    bigint       default 0                     not null,
+    owner_uuid      varchar(32)  default ''::character varying not null,
-    owner_uuid  varchar(32)  default ''::character varying not null,
+    owner_name      varchar(45)  default ''::character varying not null,
-    owner_name  varchar(45)  default ''::character varying not null,
+    star_count      int          default 0                     not null,
-    create_time timestamp    default CURRENT_TIMESTAMP     not null,
+    item_count      int          default 0                     not null,
-    update_time timestamp    default CURRENT_TIMESTAMP     not null,
+    embedding_count int          default 0                     not null,
-    is_deleted  boolean      default false                 not null
+    create_time     timestamp    default CURRENT_TIMESTAMP     not null,
+    update_time     timestamp    default CURRENT_TIMESTAMP     not null,
+    is_deleted      boolean      default false                 not null
 );
 
 comment on table adi_knowledge_base is '知识库';
@@ -460,6 +462,10 @@ comment on column adi_knowledge_base.is_public is '是否公开';
 
 comment on column adi_knowledge_base.star_count is '点赞数';
 
+comment on column adi_knowledge_base.item_count is '知识点数量';
+
+comment on column adi_knowledge_base.embedding_count is '向量数';
+
 comment on column adi_knowledge_base.owner_id is '所属人id';
 
 comment on column adi_knowledge_base.owner_uuid is '所属人uuid';