diff --git a/Processor/ProcessC2CMessage.go b/Processor/ProcessC2CMessage.go index a8aeccea..bfe31ab5 100644 --- a/Processor/ProcessC2CMessage.go +++ b/Processor/ProcessC2CMessage.go @@ -68,7 +68,7 @@ func (p *Processors) ProcessC2CMessage(data *dto.WSC2CMessageData) error { //1,idmap还原真实userid, //发信息使用的是userid - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { log.Fatalf("Error storing ID: %v", err) } @@ -205,7 +205,7 @@ func (p *Processors) ProcessC2CMessage(data *dto.WSC2CMessageData) error { //框架内指令 p.HandleFrameworkCommand(messageText, data, "group_private") //映射str的messageID到int - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { mylog.Printf("Error storing ID: %v", err) return nil diff --git a/Processor/ProcessChannelDirectMessage.go b/Processor/ProcessChannelDirectMessage.go index 50a099e3..5ada9175 100644 --- a/Processor/ProcessChannelDirectMessage.go +++ b/Processor/ProcessChannelDirectMessage.go @@ -95,7 +95,7 @@ func (p *Processors) ProcessChannelDirectMessage(data *dto.WSDirectMessageData) //3,通过idmap用channelid获取guildid, //发信息使用的是guildid //todo 优化数据库读写次数 - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { log.Fatalf("Error storing ID: %v", err) } @@ -344,7 +344,7 @@ func (p *Processors) ProcessChannelDirectMessage(data *dto.WSDirectMessageData) //userid := int(userid64) //映射str的messageID到int - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { mylog.Printf("Error storing ID: %v", err) return nil diff --git a/Processor/ProcessGroupMessage.go b/Processor/ProcessGroupMessage.go index c5c16ea3..11195ec9 100644 --- a/Processor/ProcessGroupMessage.go +++ b/Processor/ProcessGroupMessage.go @@ -78,7 +78,7 @@ func (p *Processors) ProcessGroupMessage(data *dto.WSGroupATMessageData) error { //框架内指令 p.HandleFrameworkCommand(messageText, data, "group") //映射str的messageID到int - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { mylog.Printf("Error storing ID: %v", err) return nil diff --git a/Processor/ProcessGuildATMessage.go b/Processor/ProcessGuildATMessage.go index bb2cf227..d9b8f1fc 100644 --- a/Processor/ProcessGuildATMessage.go +++ b/Processor/ProcessGuildATMessage.go @@ -195,7 +195,7 @@ func (p *Processors) ProcessGuildATMessage(data *dto.WSATMessageData) error { echostr := fmt.Sprintf("%s_%d_%d", AppIDString, s, currentTimeMillis) //映射str的messageID到int - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { mylog.Printf("Error storing ID: %v", err) return nil diff --git a/Processor/ProcessGuildNormalMessage.go b/Processor/ProcessGuildNormalMessage.go index d7d39589..fd01d02d 100644 --- a/Processor/ProcessGuildNormalMessage.go +++ b/Processor/ProcessGuildNormalMessage.go @@ -191,7 +191,7 @@ func (p *Processors) ProcessGuildNormalMessage(data *dto.WSMessageData) error { // 构造echostr，包括AppID，原始的s变量和当前时间戳 echostr := fmt.Sprintf("%s_%d_%d", AppIDString, s, currentTimeMillis) //映射str的messageID到int - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { mylog.Printf("Error storing ID: %v", err) return nil diff --git a/Processor/ProcessThreadMessage.go b/Processor/ProcessThreadMessage.go index 1d9d596f..29533d04 100644 --- a/Processor/ProcessThreadMessage.go +++ b/Processor/ProcessThreadMessage.go @@ -312,7 +312,7 @@ func (p *Processors) ProcessThreadMessage(data *dto.WSThreadData) error { // 构造echostr，包括AppID，原始的s变量和当前时间戳 echostr := fmt.Sprintf("%s_%d_%d", AppIDString, s, currentTimeMillis) //映射str的messageID到int - messageID64, err := idmap.StoreIDv2(data.ID) + messageID64, err := idmap.StoreCachev2(data.ID) if err != nil { mylog.Printf("Error storing ID: %v", err) return nil diff --git a/handlers/delete_msg.go b/handlers/delete_msg.go index f7a87295..f2830094 100644 --- a/handlers/delete_msg.go +++ b/handlers/delete_msg.go @@ -18,7 +18,7 @@ func init() { func DeleteMsg(client callapi.Client, api openapi.OpenAPI, apiv2 openapi.OpenAPI, message callapi.ActionMessage) (string, error) { //还原msgid - RealMsgID, err := idmap.RetrieveRowByIDv2(message.Params.MessageID.(string)) + RealMsgID, err := idmap.RetrieveRowByCachev2(message.Params.MessageID.(string)) if err != nil { mylog.Printf("error retrieving real RChannelID: %v", err) } diff --git a/handlers/message_parser.go b/handlers/message_parser.go index a3562074..9948a873 100644 --- a/handlers/message_parser.go +++ b/handlers/message_parser.go @@ -146,7 +146,7 @@ func SendResponse(client callapi.Client, err error, message *callapi.ActionMessa // 设置响应值 response := ServerResponse{} if resp != nil { - messageID64, mapErr = idmap.StoreIDv2(resp.Message.ID) + messageID64, mapErr = idmap.StoreCachev2(resp.Message.ID) if mapErr != nil { mylog.Printf("Error storing ID: %v", mapErr) return "", nil @@ -246,7 +246,7 @@ func SendGuildResponse(client callapi.Client, err error, message *callapi.Action // 设置响应值 response := ServerResponse{} if resp != nil { - messageID64, mapErr = idmap.StoreIDv2(resp.ID) + messageID64, mapErr = idmap.StoreCachev2(resp.ID) if mapErr != nil { mylog.Printf("Error storing ID: %v", mapErr) return "", nil @@ -304,7 +304,7 @@ func SendC2CResponse(client callapi.Client, err error, message *callapi.ActionMe // 设置响应值 response := ServerResponse{} if resp != nil { - messageID64, mapErr = idmap.StoreIDv2(resp.Message.ID) + messageID64, mapErr = idmap.StoreCachev2(resp.Message.ID) if mapErr != nil { mylog.Printf("Error storing ID: %v", mapErr) return "", nil @@ -361,7 +361,7 @@ func SendGuildPrivateResponse(client callapi.Client, err error, message *callapi // 设置响应值 response := ServerResponse{} if resp != nil { - messageID64, mapErr = idmap.StoreIDv2(resp.ID) + messageID64, mapErr = idmap.StoreCachev2(resp.ID) if mapErr != nil { mylog.Printf("Error storing ID: %v", mapErr) return "", nil diff --git a/handlers/send_group_msg_raw.go b/handlers/send_group_msg_raw.go index 714edc57..bd9c0fb6 100644 --- a/handlers/send_group_msg_raw.go +++ b/handlers/send_group_msg_raw.go @@ -160,7 +160,7 @@ func HandleSendGroupMsgRaw(client callapi.Client, api openapi.OpenAPI, apiv2 ope if isNumeric(messageID) && messageID != "0" { // 当messageID是字符串形式的数字时，执行转换 - RealMsgID, err := idmap.RetrieveRowByIDv2(messageID) + RealMsgID, err := idmap.RetrieveRowByCachev2(messageID) if err != nil { mylog.Printf("error retrieving real MessageID: %v", err) } else { diff --git a/idmap/service.go b/idmap/service.go index d6029185..2265e194 100644 --- a/idmap/service.go +++ b/idmap/service.go @@ -31,11 +31,12 @@ var ( ) const ( - DBName = "idmap.db" - BucketName = "ids" - ConfigBucket = "config" - UserInfoBucket = "UserInfo" - CounterKey = "currentRow" + DBName = "idmap.db" + BucketName = "ids" + CacheBucketName = "cache" + ConfigBucket = "config" + UserInfoBucket = "UserInfo" + CounterKey = "currentRow" ) var db *bbolt.DB @@ -64,6 +65,10 @@ func InitializeDB() { if _, err := tx.CreateBucketIfNotExists([]byte(ConfigBucket)); err != nil { return err } + // 创建储存缓存的Bucket + if _, err := tx.CreateBucketIfNotExists([]byte(CacheBucketName)); err != nil { + return err + } return nil }) @@ -72,13 +77,13 @@ func InitializeDB() { } } -func ClearBucket(bucketName string) { +func DeleteBucket(bucketName string) { // 清空指定的bucket err := db.Update(func(tx *bbolt.Tx) error { // 获取指定的bucket bucket := tx.Bucket([]byte(bucketName)) if bucket == nil { - mylog.Printf("ids表不存在.") + mylog.Printf(bucketName + "表不存在.") return nil // 如果bucket不存在，直接返回nil } @@ -95,17 +100,61 @@ func ClearBucket(bucketName string) { if err != nil { log.Fatalf("Error clearing bucket %s: %v", bucketName, err) } else { - mylog.Printf("ids清理成功.") - err := Compaction("idmap.db", "idmap_compacted.db") - if err != nil { - log.Fatalf("Failed to compact database: %v", err) - } else { - log.Println("Database compaction successful.") - // 可选：替换旧数据库文件 - // os.Remove("idmap.db") - // os.Rename("idmap_compacted.db", "idmap.db") - log.Println("请手动备份原始idmap.db(可选)并将idmap_compacted.db改名为idmap.db") + mylog.Printf(bucketName + "清理成功.请手动运行-compaction") + } +} + +func CleanBucket(bucketName string) { + var deleteCount int + + err := db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketName)) + if b == nil { + return fmt.Errorf("bucket %s not found", bucketName) + } + + // 遍历并检查id的长度 + c := b.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + id := string(v) + if len(id) != 32 { + if err := c.Delete(); err != nil { + return err + } + deleteCount++ + } + } + + // 遍历并检查reverseKey的长度 + for k, _ := c.First(); k != nil; k, _ = c.Next() { + if strings.HasPrefix(string(k), "row-") { + id := string(b.Get(k)) + if len(id) != 32 { + if err := b.Delete(k); err != nil { + return err + } + deleteCount++ + } + } } + + return nil + }) + + if err != nil { + log.Fatalf("Failed to clean bucket %s: %v", bucketName, err) + } + + log.Printf("Cleaned %d entries from bucket %s.", deleteCount, bucketName) +} + +func CompactionIdmap() { + err := Compaction("idmap.db", "idmap_compacted.db") + if err != nil { + log.Fatalf("Failed to compact database: %v", err) + } else { + log.Println("Database compaction successful.") + log.Println("请手动备份原始idmap.db(可选)并将idmap_compacted.db改名为idmap.db") } } @@ -273,6 +322,69 @@ func StoreID(id string) (int64, error) { return newRow, err } +// 根据a储存b +func StoreCache(id string) (int64, error) { + var newRow int64 + + err := db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(CacheBucketName)) + + // 检查ID是否已经存在 + existingRowBytes := b.Get([]byte(id)) + if existingRowBytes != nil { + newRow = int64(binary.BigEndian.Uint64(existingRowBytes)) + return nil + } + //写入虚拟值 + if !config.GetHashIDValue() { + // 如果ID不存在，则为它分配一个新的行号 数字递增 + currentRowBytes := b.Get([]byte(CounterKey)) + if currentRowBytes == nil { + newRow = 1 + } else { + currentRow := binary.BigEndian.Uint64(currentRowBytes) + newRow = int64(currentRow) + 1 + } + } else { + // 生成新的行号 + var err error + maxDigits := 18 // int64的位数上限-1 + for digits := 9; digits <= maxDigits; digits++ { + newRow, err = GenerateRowID(id, digits) + if err != nil { + return err + } + // 检查新生成的行号是否重复 + rowKey := fmt.Sprintf("row-%d", newRow) + if b.Get([]byte(rowKey)) == nil { + // 找到了一个唯一的行号，可以跳出循环 + break + } + // 如果到达了最大尝试次数还没有找到唯一的行号，则返回错误 + if digits == maxDigits { + return fmt.Errorf("unable to find a unique row ID after %d attempts", maxDigits-8) + } + } + } + + rowBytes := make([]byte, 8) + binary.BigEndian.PutUint64(rowBytes, uint64(newRow)) + //写入递增值 + if !config.GetHashIDValue() { + b.Put([]byte(CounterKey), rowBytes) + } + //真实对应虚拟 用来直接判断是否存在,并快速返回 + b.Put([]byte(id), rowBytes) + + reverseKey := fmt.Sprintf("row-%d", newRow) + b.Put([]byte(reverseKey), []byte(id)) + + return nil + }) + + return newRow, err +} + func SimplifiedStoreID(id string) (int64, error) { var newRow int64 @@ -440,6 +552,48 @@ func StoreIDv2(id string) (int64, error) { return StoreID(id) } +// StoreCachev2 根据a储存b +func StoreCachev2(id string) (int64, error) { + if config.GetLotusValue() && !config.GetLotusWithoutIdmaps() { + // 使用网络请求方式 + serverDir := config.GetServer_dir() + portValue := config.GetPortValue() + + // 根据portValue确定协议 + protocol := "http" + if portValue == "443" { + protocol = "https" + } + + // 构建请求URL + url := fmt.Sprintf("%s://%s:%s/getid?type=16&id=%s", protocol, serverDir, portValue, id) + resp, err := http.Get(url) + if err != nil { + return 0, fmt.Errorf("failed to send request: %v", err) + } + defer resp.Body.Close() + + // 解析响应 + var response map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return 0, fmt.Errorf("failed to decode response: %v", err) + } + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("error response from server: %s", response["error"]) + } + + rowValue, ok := response["row"].(float64) + if !ok { + return 0, fmt.Errorf("invalid response format") + } + + return int64(rowValue), nil + } + + // 如果lotus为假,就保持原来的store的方法 + return StoreCache(id) +} + // 群号 然后 用户号 func StoreIDv2Pro(id string, subid string) (int64, int64, error) { if config.GetLotusValue() && !config.GetLotusWithoutIdmaps() { @@ -506,6 +660,25 @@ func RetrieveRowByID(rowid string) (string, error) { return id, err } +// 根据b得到a +func RetrieveRowByCache(rowid string) (string, error) { + var id string + err := db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(CacheBucketName)) + + // 根据行号检索ID + idBytes := b.Get([]byte("row-" + rowid)) + if idBytes == nil { + return ErrKeyNotFound + } + id = string(idBytes) + + return nil + }) + + return id, err +} + // 群号 然后 用户号 func RetrieveRowByIDv2Pro(newRowID string, newSubRowID string) (string, string, error) { if config.GetLotusValue() && !config.GetLotusWithoutIdmaps() { @@ -623,6 +796,48 @@ func RetrieveRowByIDv2(rowid string) (string, error) { return RetrieveRowByID(rowid) } +// RetrieveRowByCachev2 根据b得到a +func RetrieveRowByCachev2(rowid string) (string, error) { + // 根据portValue确定协议 + protocol := "http" + portValue := config.GetPortValue() + if portValue == "443" { + protocol = "https" + } + + if config.GetLotusValue() && !config.GetLotusWithoutIdmaps() { + // 使用网络请求方式 + serverDir := config.GetServer_dir() + + // 构建请求URL + url := fmt.Sprintf("%s://%s:%s/getid?type=17&id=%s", protocol, serverDir, portValue, rowid) + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("failed to send request: %v", err) + } + defer resp.Body.Close() + + // 解析响应 + var response map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return "", fmt.Errorf("failed to decode response: %v", err) + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("error response from server: %s", response["error"]) + } + + idValue, ok := response["id"].(string) + if !ok { + return "", fmt.Errorf("invalid response format") + } + + return idValue, nil + } + + // 如果lotus为假,就保持原来的RetrieveRowByIDv2的方法 + return RetrieveRowByCache(rowid) +} + // 根据a 以b为类别 储存c func WriteConfig(sectionName, keyName, value string) error { return db.Update(func(tx *bbolt.Tx) error { diff --git a/main.go b/main.go index ac6d724b..6c3d089f 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,10 @@ func main() { // 定义faststart命令行标志。默认为false。 fastStart := flag.Bool("faststart", false, "start without initialization if set") tidy := flag.Bool("tidy", false, "backup and tidy your config.yml") - c := flag.Bool("c", false, "clean ids bucket, must backup idmap.db first!") + cleanids := flag.Bool("clean_ids", false, "clean msg_id in ids bucket.") + delids := flag.Bool("del_ids", false, "delete ids bucket, must backup idmap.db first!") + delcache := flag.Bool("del_cache", false, "delete cache bucket, it is safe") + compaction := flag.Bool("compaction", false, "compaction for apply db changes.") m := flag.Bool("m", false, "Maintenance mode") // 解析命令行参数到定义的标志。 @@ -205,12 +208,30 @@ func main() { defer idmap.CloseDB() defer botstats.CloseDB() - if *c { - mylog.Printf("开始清理ids\n") - idmap.ClearBucket("ids") + if *delids { + mylog.Printf("开始删除ids\n") + idmap.DeleteBucket("ids") + mylog.Printf("ids删除完成\n") + return + } + if *delcache { + mylog.Printf("开始删除cache\n") + idmap.DeleteBucket("cache") + mylog.Printf("cache删除完成\n") + return + } + if *cleanids { + mylog.Printf("开始清理ids中的msg_id\n") + idmap.CleanBucket("ids") mylog.Printf("ids清理完成\n") return } + if *compaction { + mylog.Printf("开始整理idmap.db\n") + idmap.CompactionIdmap() + mylog.Printf("idmap.db整理完成\n") + return + } if configURL == "" && !fix11300 { //初始化handlers handlers.BotID = me.ID diff --git a/server/getIDHandler.go b/server/getIDHandler.go index 9c01cbff..32c2cbca 100644 --- a/server/getIDHandler.go +++ b/server/getIDHandler.go @@ -238,6 +238,24 @@ func GetIDHandler(c *gin.Context) { // 如果删除成功，返回成功响应 c.JSON(http.StatusOK, gin.H{"status": "success"}) + case 16: + newRow, err := idmap.StoreCachev2(idOrRow) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"row": newRow}) + + case 17: + id, err := idmap.RetrieveRowByCachev2(idOrRow) + if err == idmap.ErrKeyNotFound { + c.JSON(http.StatusNotFound, gin.H{"error": "ID not found"}) + return + } else if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"id": id}) } }