400 lines
12 KiB
Go
400 lines
12 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
"image/png"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
corestorage "git.ilapage.cn/OPC/chorus/internal/core/storage"
|
|
"github.com/disintegration/imaging"
|
|
_ "golang.org/x/image/webp"
|
|
)
|
|
|
|
var (
|
|
ErrInvalidKey = errors.New("storage key is invalid")
|
|
ErrInvalidMetadata = errors.New("storage ownership metadata is invalid")
|
|
ErrObjectExists = errors.New("storage object already exists")
|
|
ErrObjectTooLarge = errors.New("storage object exceeds size limit")
|
|
ErrInvalidImage = errors.New("image content is invalid")
|
|
ErrImageTooLarge = errors.New("image dimensions exceed limit")
|
|
ErrUnsupportedImage = errors.New("image format is not allowed")
|
|
)
|
|
|
|
type Config struct {
|
|
Root string
|
|
MaxObjectBytes int64
|
|
MaxImagePixels uint64
|
|
ThumbnailMaxSide int
|
|
AllowedImageMIME map[string]bool
|
|
}
|
|
|
|
type Local struct {
|
|
root string
|
|
temporaryRoot string
|
|
maxObjectBytes int64
|
|
maxImagePixels uint64
|
|
thumbnailMaxSide int
|
|
allowedImageMIME map[string]bool
|
|
}
|
|
|
|
type metadata struct {
|
|
Key string `json:"key"`
|
|
OwnerID uint64 `json:"owner_id"`
|
|
GenerationID uint64 `json:"generation_id"`
|
|
ContentType string `json:"content_type"`
|
|
Size int64 `json:"size"`
|
|
}
|
|
|
|
type ImageRequest struct {
|
|
Key string
|
|
ThumbnailKey string
|
|
OwnerID uint64
|
|
GenerationID uint64
|
|
ContentType string
|
|
Source io.Reader
|
|
}
|
|
|
|
type ImageObjects struct {
|
|
Original corestorage.Object
|
|
Thumbnail corestorage.Object
|
|
}
|
|
|
|
func NewLocal(config Config) (*Local, error) {
|
|
if strings.TrimSpace(config.Root) == "" || config.MaxObjectBytes <= 0 || config.MaxImagePixels == 0 || config.ThumbnailMaxSide <= 0 || len(config.AllowedImageMIME) == 0 {
|
|
return nil, fmt.Errorf("local storage configuration is incomplete")
|
|
}
|
|
root, err := filepath.Abs(config.Root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve storage root: %w", err)
|
|
}
|
|
if err := os.MkdirAll(root, 0o700); err != nil {
|
|
return nil, fmt.Errorf("create storage root: %w", err)
|
|
}
|
|
root, err = filepath.EvalSymlinks(root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve storage root links: %w", err)
|
|
}
|
|
temporaryRoot := filepath.Join(root, ".tmp")
|
|
if err := secureMkdirAll(root, temporaryRoot); err != nil {
|
|
return nil, fmt.Errorf("create storage temporary root: %w", err)
|
|
}
|
|
allowed := make(map[string]bool, len(config.AllowedImageMIME))
|
|
for mimeType, enabled := range config.AllowedImageMIME {
|
|
if enabled {
|
|
allowed[strings.ToLower(strings.TrimSpace(mimeType))] = true
|
|
}
|
|
}
|
|
return &Local{
|
|
root: root, temporaryRoot: temporaryRoot, maxObjectBytes: config.MaxObjectBytes,
|
|
maxImagePixels: config.MaxImagePixels, thumbnailMaxSide: config.ThumbnailMaxSide,
|
|
allowedImageMIME: allowed,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Local) Put(ctx context.Context, request corestorage.PutRequest) (object corestorage.Object, err error) {
|
|
if request.OwnerID == 0 || request.GenerationID == 0 || strings.TrimSpace(request.ContentType) == "" || request.Source == nil {
|
|
return corestorage.Object{}, ErrInvalidMetadata
|
|
}
|
|
finalDirectory, err := s.objectDirectory(request.Key)
|
|
if err != nil {
|
|
return corestorage.Object{}, err
|
|
}
|
|
if _, err := os.Lstat(finalDirectory); err == nil {
|
|
return corestorage.Object{}, ErrObjectExists
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return corestorage.Object{}, fmt.Errorf("inspect storage target: %w", err)
|
|
}
|
|
if err := secureMkdirAll(s.root, filepath.Dir(finalDirectory)); err != nil {
|
|
return corestorage.Object{}, fmt.Errorf("create storage parent: %w", err)
|
|
}
|
|
temporaryDirectory, err := os.MkdirTemp(s.temporaryRoot, "put-")
|
|
if err != nil {
|
|
return corestorage.Object{}, fmt.Errorf("create storage temporary directory: %w", err)
|
|
}
|
|
defer func() {
|
|
if temporaryDirectory != "" {
|
|
_ = os.RemoveAll(temporaryDirectory)
|
|
}
|
|
}()
|
|
|
|
contentPath := filepath.Join(temporaryDirectory, "content")
|
|
content, err := os.OpenFile(contentPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
return corestorage.Object{}, fmt.Errorf("create storage content: %w", err)
|
|
}
|
|
size, copyErr := copyLimitedContext(ctx, content, request.Source, s.maxObjectBytes)
|
|
syncErr := content.Sync()
|
|
closeErr := content.Close()
|
|
if copyErr != nil {
|
|
return corestorage.Object{}, copyErr
|
|
}
|
|
if syncErr != nil || closeErr != nil {
|
|
return corestorage.Object{}, fmt.Errorf("flush storage content")
|
|
}
|
|
|
|
storedMetadata := metadata{Key: request.Key, OwnerID: request.OwnerID, GenerationID: request.GenerationID, ContentType: request.ContentType, Size: size}
|
|
metadataBytes, err := json.Marshal(storedMetadata)
|
|
if err != nil {
|
|
return corestorage.Object{}, fmt.Errorf("encode storage metadata: %w", err)
|
|
}
|
|
if err := writeSyncedFile(filepath.Join(temporaryDirectory, "metadata.json"), metadataBytes); err != nil {
|
|
return corestorage.Object{}, err
|
|
}
|
|
if err := os.Rename(temporaryDirectory, finalDirectory); err != nil {
|
|
return corestorage.Object{}, fmt.Errorf("atomically place storage object: %w", err)
|
|
}
|
|
temporaryDirectory = ""
|
|
return objectFromMetadata(storedMetadata), nil
|
|
}
|
|
|
|
func (s *Local) Open(ctx context.Context, key string) (io.ReadCloser, corestorage.Object, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, corestorage.Object{}, err
|
|
}
|
|
directory, err := s.objectDirectory(key)
|
|
if err != nil {
|
|
return nil, corestorage.Object{}, err
|
|
}
|
|
if err := ensureNoSymlink(s.root, directory); err != nil {
|
|
return nil, corestorage.Object{}, err
|
|
}
|
|
metadataBytes, err := os.ReadFile(filepath.Join(directory, "metadata.json"))
|
|
if err != nil {
|
|
return nil, corestorage.Object{}, fmt.Errorf("read storage metadata: %w", err)
|
|
}
|
|
var storedMetadata metadata
|
|
if err := json.Unmarshal(metadataBytes, &storedMetadata); err != nil || storedMetadata.Key != key || storedMetadata.OwnerID == 0 || storedMetadata.GenerationID == 0 {
|
|
return nil, corestorage.Object{}, ErrInvalidMetadata
|
|
}
|
|
content, err := os.Open(filepath.Join(directory, "content"))
|
|
if err != nil {
|
|
return nil, corestorage.Object{}, fmt.Errorf("open storage content: %w", err)
|
|
}
|
|
return content, objectFromMetadata(storedMetadata), nil
|
|
}
|
|
|
|
func (s *Local) Delete(ctx context.Context, key string) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
directory, err := s.objectDirectory(key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := os.Lstat(directory); errors.Is(err, os.ErrNotExist) {
|
|
return nil
|
|
} else if err != nil {
|
|
return fmt.Errorf("inspect storage object: %w", err)
|
|
}
|
|
if err := ensureNoSymlink(s.root, directory); err != nil {
|
|
return err
|
|
}
|
|
if err := os.RemoveAll(directory); err != nil {
|
|
return fmt.Errorf("delete storage object: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Local) PutImage(ctx context.Context, request ImageRequest) (ImageObjects, error) {
|
|
if request.Key == "" || request.ThumbnailKey == "" || request.Key == request.ThumbnailKey {
|
|
return ImageObjects{}, ErrInvalidKey
|
|
}
|
|
data, err := readLimitedContext(ctx, request.Source, s.maxObjectBytes)
|
|
if err != nil {
|
|
return ImageObjects{}, err
|
|
}
|
|
config, format, err := image.DecodeConfig(bytes.NewReader(data))
|
|
if err != nil || config.Width <= 0 || config.Height <= 0 {
|
|
return ImageObjects{}, ErrInvalidImage
|
|
}
|
|
actualMIME := imageMIME(format)
|
|
if actualMIME == "" || !s.allowedImageMIME[actualMIME] || !strings.EqualFold(strings.TrimSpace(request.ContentType), actualMIME) {
|
|
return ImageObjects{}, ErrUnsupportedImage
|
|
}
|
|
pixels := uint64(config.Width) * uint64(config.Height)
|
|
if pixels > s.maxImagePixels {
|
|
return ImageObjects{}, ErrImageTooLarge
|
|
}
|
|
decoded, _, err := image.Decode(bytes.NewReader(data))
|
|
if err != nil {
|
|
return ImageObjects{}, ErrInvalidImage
|
|
}
|
|
thumbnail := imaging.Fit(decoded, s.thumbnailMaxSide, s.thumbnailMaxSide, imaging.Lanczos)
|
|
var thumbnailData bytes.Buffer
|
|
if err := png.Encode(&thumbnailData, thumbnail); err != nil {
|
|
return ImageObjects{}, fmt.Errorf("encode image thumbnail: %w", err)
|
|
}
|
|
|
|
original, err := s.Put(ctx, corestorage.PutRequest{
|
|
Key: request.Key, OwnerID: request.OwnerID, GenerationID: request.GenerationID,
|
|
ContentType: actualMIME, Source: bytes.NewReader(data),
|
|
})
|
|
if err != nil {
|
|
return ImageObjects{}, err
|
|
}
|
|
thumb, err := s.Put(ctx, corestorage.PutRequest{
|
|
Key: request.ThumbnailKey, OwnerID: request.OwnerID, GenerationID: request.GenerationID,
|
|
ContentType: "image/png", Source: bytes.NewReader(thumbnailData.Bytes()),
|
|
})
|
|
if err != nil {
|
|
_ = s.Delete(context.Background(), request.Key)
|
|
return ImageObjects{}, err
|
|
}
|
|
return ImageObjects{Original: original, Thumbnail: thumb}, nil
|
|
}
|
|
|
|
func (s *Local) objectDirectory(key string) (string, error) {
|
|
if key == "" || strings.Contains(key, "\\") || strings.Contains(key, ":") || strings.HasPrefix(key, "/") {
|
|
return "", ErrInvalidKey
|
|
}
|
|
parts := strings.Split(key, "/")
|
|
for _, part := range parts {
|
|
if part == "" || part == "." || part == ".." {
|
|
return "", ErrInvalidKey
|
|
}
|
|
}
|
|
path := filepath.Join(append([]string{s.root, "objects"}, parts...)...)
|
|
relative, err := filepath.Rel(s.root, path)
|
|
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
|
return "", ErrInvalidKey
|
|
}
|
|
return path, nil
|
|
}
|
|
|
|
func ensureNoSymlink(root, target string) error {
|
|
relative, err := filepath.Rel(root, target)
|
|
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
|
return ErrInvalidKey
|
|
}
|
|
current := root
|
|
for _, part := range strings.Split(relative, string(filepath.Separator)) {
|
|
if part == "." || part == "" {
|
|
continue
|
|
}
|
|
current = filepath.Join(current, part)
|
|
info, err := os.Lstat(current)
|
|
if err != nil {
|
|
return fmt.Errorf("inspect storage path: %w", err)
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 {
|
|
return ErrInvalidKey
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func secureMkdirAll(root, target string) error {
|
|
relative, err := filepath.Rel(root, target)
|
|
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
|
return ErrInvalidKey
|
|
}
|
|
current := root
|
|
for _, part := range strings.Split(relative, string(filepath.Separator)) {
|
|
if part == "." || part == "" {
|
|
continue
|
|
}
|
|
current = filepath.Join(current, part)
|
|
info, err := os.Lstat(current)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
if err := os.Mkdir(current, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
|
|
return err
|
|
}
|
|
info, err = os.Lstat(current)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
|
return ErrInvalidKey
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func copyLimitedContext(ctx context.Context, destination io.Writer, source io.Reader, limit int64) (int64, error) {
|
|
buffer := make([]byte, 32*1024)
|
|
var total int64
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return total, err
|
|
}
|
|
read, readErr := source.Read(buffer)
|
|
if read > 0 {
|
|
if total+int64(read) > limit {
|
|
return total, ErrObjectTooLarge
|
|
}
|
|
written, writeErr := destination.Write(buffer[:read])
|
|
total += int64(written)
|
|
if writeErr != nil {
|
|
return total, writeErr
|
|
}
|
|
if written != read {
|
|
return total, io.ErrShortWrite
|
|
}
|
|
}
|
|
if readErr != nil {
|
|
if errors.Is(readErr, io.EOF) {
|
|
return total, nil
|
|
}
|
|
return total, readErr
|
|
}
|
|
}
|
|
}
|
|
|
|
func readLimitedContext(ctx context.Context, source io.Reader, limit int64) ([]byte, error) {
|
|
var data bytes.Buffer
|
|
_, err := copyLimitedContext(ctx, &data, source, limit)
|
|
return data.Bytes(), err
|
|
}
|
|
|
|
func writeSyncedFile(path string, data []byte) error {
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
return fmt.Errorf("create storage metadata: %w", err)
|
|
}
|
|
if _, err := file.Write(data); err != nil {
|
|
file.Close()
|
|
return fmt.Errorf("write storage metadata: %w", err)
|
|
}
|
|
if err := file.Sync(); err != nil {
|
|
file.Close()
|
|
return fmt.Errorf("flush storage metadata: %w", err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
return fmt.Errorf("close storage metadata: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func imageMIME(format string) string {
|
|
switch strings.ToLower(format) {
|
|
case "jpeg":
|
|
return "image/jpeg"
|
|
case "png":
|
|
return "image/png"
|
|
case "gif":
|
|
return "image/gif"
|
|
case "webp":
|
|
return "image/webp"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func objectFromMetadata(stored metadata) corestorage.Object {
|
|
return corestorage.Object{
|
|
Key: stored.Key, OwnerID: stored.OwnerID, GenerationID: stored.GenerationID,
|
|
ContentType: stored.ContentType, Size: stored.Size,
|
|
}
|
|
}
|
|
|
|
var _ corestorage.Store = (*Local)(nil)
|