feat: disk request body cache (#2780)
* feat: 引入通用 HTTP BodyStorage/DiskCache 缓存配置与管理 - 新增 common/body_storage.go 提供 HTTP 请求体存储抽象和文件缓存能力 - 增加 common/disk_cache_config.go 支持全局磁盘缓存配置 - main.go 挂载缓存初始化流程 - 新增和补充 controller/performance.go (及 unix/windows) 用于缓存性能监控接口 - middleware/body_cleanup.go 自动清理缓存文件 - router 挂载相关接口 - 前端 settings 页面新增性能监控设置 PerformanceSetting - 优化缓存开关状态和模块热插拔能力 - 其他相关文件同步适配缓存扩展 * fix: 修复 BodyStorage 并发安全和错误处理问题 - 修复 diskStorage.Close() 竞态条件,先获取锁再执行 CAS - 为 memoryStorage 添加互斥锁和 closed 状态检查 - 修复 CreateBodyStorageFromReader 在磁盘存储失败时的回退逻辑 - 添加缓存命中统计调用 (IncrementDiskCacheHits/IncrementMemoryCacheHits) - 修复 gin.go 中 Seek 错误被忽略的问题 - 在 api-router 添加 BodyStorageCleanup 中间件 - 修复前端 formatBytes 对异常值的处理 Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
GitHub
co-authored by
Cursor
parent
3c1675d4e4
commit
d7c55b92b2
+72
-14
@@ -18,6 +18,7 @@ import (
|
||||
)
|
||||
|
||||
const KeyRequestBody = "key_request_body"
|
||||
const KeyBodyStorage = "key_body_storage"
|
||||
|
||||
var ErrRequestBodyTooLarge = errors.New("request body too large")
|
||||
|
||||
@@ -33,42 +34,99 @@ func IsRequestBodyTooLargeError(err error) bool {
|
||||
}
|
||||
|
||||
func GetRequestBody(c *gin.Context) ([]byte, error) {
|
||||
// 首先检查是否有 BodyStorage 缓存
|
||||
if storage, exists := c.Get(KeyBodyStorage); exists && storage != nil {
|
||||
if bs, ok := storage.(BodyStorage); ok {
|
||||
if _, err := bs.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("failed to seek body storage: %w", err)
|
||||
}
|
||||
return bs.Bytes()
|
||||
}
|
||||
}
|
||||
|
||||
// 检查旧的缓存方式
|
||||
cached, exists := c.Get(KeyRequestBody)
|
||||
if exists && cached != nil {
|
||||
if b, ok := cached.([]byte); ok {
|
||||
return b, nil
|
||||
}
|
||||
}
|
||||
|
||||
maxMB := constant.MaxRequestBodyMB
|
||||
if maxMB <= 0 {
|
||||
// no limit
|
||||
body, err := io.ReadAll(c.Request.Body)
|
||||
_ = c.Request.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Set(KeyRequestBody, body)
|
||||
return body, nil
|
||||
maxMB = 128 // 默认 128MB
|
||||
}
|
||||
maxBytes := int64(maxMB) << 20
|
||||
|
||||
limited := io.LimitReader(c.Request.Body, maxBytes+1)
|
||||
body, err := io.ReadAll(limited)
|
||||
contentLength := c.Request.ContentLength
|
||||
|
||||
// 使用新的存储系统
|
||||
storage, err := CreateBodyStorageFromReader(c.Request.Body, contentLength, maxBytes)
|
||||
_ = c.Request.Body.Close()
|
||||
|
||||
if err != nil {
|
||||
_ = c.Request.Body.Close()
|
||||
if IsRequestBodyTooLargeError(err) {
|
||||
return nil, errors.Wrap(ErrRequestBodyTooLarge, fmt.Sprintf("request body exceeds %d MB", maxMB))
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
_ = c.Request.Body.Close()
|
||||
if int64(len(body)) > maxBytes {
|
||||
return nil, errors.Wrap(ErrRequestBodyTooLarge, fmt.Sprintf("request body exceeds %d MB", maxMB))
|
||||
|
||||
// 缓存存储对象
|
||||
c.Set(KeyBodyStorage, storage)
|
||||
|
||||
// 获取字节数据
|
||||
body, err := storage.Bytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 同时设置旧的缓存键以保持兼容性
|
||||
c.Set(KeyRequestBody, body)
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// GetBodyStorage 获取请求体存储对象(用于需要多次读取的场景)
|
||||
func GetBodyStorage(c *gin.Context) (BodyStorage, error) {
|
||||
// 检查是否已有存储
|
||||
if storage, exists := c.Get(KeyBodyStorage); exists && storage != nil {
|
||||
if bs, ok := storage.(BodyStorage); ok {
|
||||
if _, err := bs.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("failed to seek body storage: %w", err)
|
||||
}
|
||||
return bs, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有,调用 GetRequestBody 创建存储
|
||||
_, err := GetRequestBody(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 再次获取存储
|
||||
if storage, exists := c.Get(KeyBodyStorage); exists && storage != nil {
|
||||
if bs, ok := storage.(BodyStorage); ok {
|
||||
if _, err := bs.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("failed to seek body storage: %w", err)
|
||||
}
|
||||
return bs, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("failed to get body storage")
|
||||
}
|
||||
|
||||
// CleanupBodyStorage 清理请求体存储(应在请求结束时调用)
|
||||
func CleanupBodyStorage(c *gin.Context) {
|
||||
if storage, exists := c.Get(KeyBodyStorage); exists && storage != nil {
|
||||
if bs, ok := storage.(BodyStorage); ok {
|
||||
bs.Close()
|
||||
}
|
||||
c.Set(KeyBodyStorage, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func UnmarshalBodyReusable(c *gin.Context, v any) error {
|
||||
requestBody, err := GetRequestBody(c)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user