lexgo/server/app/lexgo/upload_test.go

package lexgo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"mime/multipart"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/gin-gonic/gin"
)

// uploadFixture is the same character set the paste contract preserves, written as the bytes
// a file would hold: CRLF and LF, a tab, curly quotes, an em dash, an ellipsis, an emoji, a
// combining acute accent, a trailing space run and an empty final line.
const uploadFixture = "Mira opened the workshop.\r\n\r\n\tThe sign read “A small step…” — café e\u0301 🙂\r\nTrailing spaces here:   \n\n"

// uploadFile posts one multipart TXT submission. A nil file means "no file part", and an
// empty name means "no file name", so the rejection paths stay testable.
func uploadFile(t *testing.T, r *gin.Engine, token, path, fileName string, content []byte, fields map[string]string) (int, string, pasteResponse) {
	t.Helper()
	var body bytes.Buffer
	writer := multipart.NewWriter(&body)
	names := make([]string, 0, len(fields))
	for name := range fields {
		names = append(names, name)
	}
	// Field order must be stable so a failure message is reproducible.
	for _, name := range []string{"requestId", "title", "language"} {
		if value, ok := fields[name]; ok {
			if err := writer.WriteField(name, value); err != nil {
				t.Fatal(err)
			}
			names = removeString(names, name)
		}
	}
	for _, name := range names {
		if err := writer.WriteField(name, fields[name]); err != nil {
			t.Fatal(err)
		}
	}
	if content != nil {
		part, err := writer.CreateFormFile("file", fileName)
		if err != nil {
			t.Fatal(err)
		}
		if _, err = part.Write(content); err != nil {
			t.Fatal(err)
		}
	}
	writer.Close()
	request := httptest.NewRequest("POST", path, &body)
	request.Header.Set("Content-Type", writer.FormDataContentType())
	if token != "" {
		request.Header.Set("Authorization", "Bearer "+token)
	}
	response := httptest.NewRecorder()
	r.ServeHTTP(response, request)
	var envelope struct {
		Msg  string          `json:"msg"`
		Data json.RawMessage `json:"data"`
	}
	if err := json.Unmarshal(response.Body.Bytes(), &envelope); err != nil {
		t.Fatalf("upload response for %s: %v", path, err)
	}
	var out pasteResponse
	if len(envelope.Data) > 0 {
		json.Unmarshal(envelope.Data, &out)
	}
	return response.Code, envelope.Msg, out
}

func removeString(values []string, target string) []string {
	result := values[:0]
	for _, value := range values {
		if value != target {
			result = append(result, value)
		}
	}
	return result
}

func TestDecodeTextUploadRules(t *testing.T) {
	// Valid UTF-8 is returned exactly as received, including unusual but legal characters.
	if text, err := decodeTextUpload([]byte(uploadFixture)); err != nil || text != uploadFixture {
		t.Fatalf("valid file: %q %v", text, err)
	}
	// A UTF-8 BOM is removed and never becomes part of the original text.
	withBOM := append(append([]byte{}, utf8BOM...), []byte("BOM before text\n")...)
	if text, err := decodeTextUpload(withBOM); err != nil || text != "BOM before text\n" {
		t.Fatalf("BOM file: %q %v", text, err)
	}
	// Only a BOM leaves an empty text, which the paste rules then reject.
	if text, err := decodeTextUpload(append([]byte{}, utf8BOM...)); err != nil || text != "" {
		t.Fatalf("BOM only: %q %v", text, err)
	}
	rejected := []struct {
		name    string
		content []byte
		message string
	}{
		{"invalid UTF-8", []byte{0x41, 0x80, 0x42}, "UTF-8"},
		{"latin-1 text", []byte("caf\xe9 plain\n"), "UTF-8"},
		{"UTF-16 little endian", []byte{0xFF, 0xFE, 0x41, 0x00}, "UTF-16"},
		{"UTF-16 big endian", []byte{0xFE, 0xFF, 0x00, 0x41}, "UTF-16"},
		{"NUL byte", []byte("text\x00more"), "无法处理"},
	}
	for _, tc := range rejected {
		text, err := decodeTextUpload(tc.content)
		if err == nil || text != "" {
			t.Fatalf("%s was accepted as %q", tc.name, text)
		}
		api, ok := err.(*apiError)
		if !ok || api.status != 400 || !strings.Contains(api.message, tc.message) {
			t.Fatalf("%s message: %v", tc.name, err)
		}
	}
	// The size boundary is exact on both sides, and the byte limit is checked before decoding.
	if _, err := decodeTextUpload(bytes.Repeat([]byte("a"), maxTextUploadBytes)); err != nil {
		t.Fatalf("a file at the size limit must be accepted: %v", err)
	}
	if _, err := decodeTextUpload(bytes.Repeat([]byte("a"), maxTextUploadBytes+1)); err == nil {
		t.Fatal("a file over the size limit must be rejected")
	}
	// The paste rules still bound one chapter, so the byte limit cannot smuggle in more text.
	if _, _, _, err := validatePaste("title", strings.Repeat("a", maxChapterRunes)); err != nil {
		t.Fatalf("the exact chapter limit must be accepted: %v", err)
	}
	if _, _, _, err := validatePaste("title", strings.Repeat("a", maxChapterRunes+1)); err == nil {
		t.Fatal("the chapter code point limit must still apply to uploaded text")
	}
}

// TestMySQLTextUploadImportPath covers the accepted file: it creates the same book, chapter
// and job as a paste, and the reader text equals the file byte for byte.
func TestMySQLTextUploadImportPath(t *testing.T) {
	db, r, owner := libraryFixture(t)
	learner := newLearner(t, r, owner.Token)
	drainIngest(t, db)

	fields := map[string]string{"requestId": "upload-fixture-0001", "title": "The Workshop Upload", "language": "en"}
	code, msg, uploaded := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "workshop.txt", []byte(uploadFixture), fields)
	if code != 201 {
		t.Fatalf("upload status %d (%s)", code, msg)
	}
	if uploaded.Book == nil || uploaded.Book.Title != "The Workshop Upload" || uploaded.Book.Language != "en" {
		t.Fatalf("unexpected book %+v", uploaded.Book)
	}
	if uploaded.Chapter.Ordinal != 1 || uploaded.Chapter.Status != statusPending || uploaded.Duplicate {
		t.Fatalf("unexpected chapter %+v", uploaded.Chapter)
	}

	// The worker publishes the chapter, and the reader shows exactly the file content.
	drainIngest(t, db)
	code, ready := readChapter(t, r, learner.Token, uploaded.Chapter.ID)
	if code != 200 || ready.Chapter.Status != statusReady || ready.Chapter.OriginalText != uploadFixture {
		t.Fatalf("reader text: status %d, %+v", code, ready.Chapter)
	}
	if want := len([]rune(uploadFixture)); ready.Chapter.CharCount != want {
		t.Fatalf("charCount %d, want %d", ready.Chapter.CharCount, want)
	}

	// A UTF-8 BOM is stripped, so the reader never shows it.
	bomFields := map[string]string{"requestId": "upload-bom-000002", "title": "BOM Upload", "language": "en"}
	withBOM := append(append([]byte{}, utf8BOM...), []byte("Plain text with BOM.\n")...)
	code, msg, bom := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "bom.txt", withBOM, bomFields)
	if code != 201 {
		t.Fatalf("BOM upload status %d (%s)", code, msg)
	}
	drainIngest(t, db)
	if code, read := readChapter(t, r, learner.Token, bom.Chapter.ID); code != 200 || read.Chapter.OriginalText != "Plain text with BOM.\n" {
		t.Fatalf("BOM text: status %d, %q", code, read.Chapter.OriginalText)
	}
}

// TestMySQLTextUploadIdempotencyAndAppend reuses the paste job rules: one file yields one
// chapter, a repeated upload answers with that chapter, and the same request id with other
// content is a conflict.
func TestMySQLTextUploadIdempotencyAndAppend(t *testing.T) {
	db, r, owner := libraryFixture(t)
	learner := newLearner(t, r, owner.Token)
	other := newLearner(t, r, owner.Token)
	drainIngest(t, db)

	fields := map[string]string{"requestId": "upload-repeat-0001", "title": "Repeat Upload", "language": "en"}
	code, msg, first := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "repeat.txt", []byte("First upload body.\n"), fields)
	if code != 201 {
		t.Fatalf("first upload %d (%s)", code, msg)
	}
	code, msg, again := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "repeat.txt", []byte("First upload body.\n"), fields)
	if code != 200 || !again.Duplicate || again.Chapter.ID != first.Chapter.ID {
		t.Fatalf("repeat upload %d (%s): %+v", code, msg, again)
	}
	var chapters int64
	db.Model(&Chapter{}).Where("book_id = ?", first.Book.ID).Count(&chapters)
	if chapters != 1 {
		t.Fatalf("a repeated upload created %d chapters", chapters)
	}
	// The same request id with other content is a conflict, not a second chapter.
	code, msg, _ = uploadFile(t, r, learner.Token, "/api/v1/books/upload", "repeat.txt", []byte("Different body.\n"), fields)
	if code != 409 {
		t.Fatalf("changed content status %d (%s)", code, msg)
	}

	// Appending uses the same rules and the same job pipeline.
	appendFields := map[string]string{"requestId": "upload-append-0002", "title": "Second Chapter"}
	path := fmt.Sprintf("/api/v1/books/%d/chapters/upload", first.Book.ID)
	code, msg, appended := uploadFile(t, r, learner.Token, path, "second.txt", []byte("A single plain paragraph.\n"), appendFields)
	if code != 201 {
		t.Fatalf("append upload %d (%s)", code, msg)
	}
	if appended.Chapter.Ordinal != 2 || appended.Chapter.BookID != first.Book.ID {
		t.Fatalf("unexpected appended chapter %+v", appended.Chapter)
	}
	code, msg, againAppend := uploadFile(t, r, learner.Token, path, "second.txt", []byte("A single plain paragraph.\n"), appendFields)
	if code != 200 || !againAppend.Duplicate || againAppend.Chapter.ID != appended.Chapter.ID {
		t.Fatalf("repeated append %d (%s): %+v", code, msg, againAppend)
	}
	// The append endpoint does not accept a language field: the book owns the language.
	code, msg, _ = uploadFile(t, r, learner.Token, path, "second.txt", []byte("Another body.\n"),
		map[string]string{"requestId": "upload-append-lang-0004", "title": "Second Chapter", "language": "en"})
	if code != 400 {
		t.Fatalf("append with a language field: status %d (%s)", code, msg)
	}

	// Another account cannot append into this book, and never learns whether it exists.
	code, foreignMsg, _ := uploadFile(t, r, other.Token, path, "second.txt", []byte("Foreign body.\n"), map[string]string{"requestId": "upload-foreign-0003", "title": "Foreign"})
	if code != 404 {
		t.Fatalf("foreign append status %d (%s)", code, foreignMsg)
	}
	// The other account's own library stays empty.
	_, list := bookList(t, r, other.Token)
	if len(list.Items) != 0 {
		t.Fatalf("the other account must not see this book: %+v", list.Items)
	}
}

// TestMySQLTextUploadRejectsInvalidSubmissions covers the validation surface, including the
// client file name, which is never used as a path.
func TestMySQLTextUploadRejectsInvalidSubmissions(t *testing.T) {
	db, r, owner := libraryFixture(t)
	learner := newLearner(t, r, owner.Token)
	drainIngest(t, db)

	base := map[string]string{"requestId": "upload-invalid-0001", "title": "Invalid Upload", "language": "en"}
	cases := []struct {
		name     string
		fileName string
		content  []byte
		fields   map[string]string
		status   int
	}{
		{"no file part", "", nil, base, 400},
		{"empty file", "empty.txt", []byte{}, base, 400},
		{"empty file part", "empty.txt", []byte{}, base, 400},
		{"whitespace only", "blank.txt", []byte("   \n\t\n"), base, 400},
		{"BOM only", "bom.txt", append([]byte{}, utf8BOM...), base, 400},
		{"invalid UTF-8", "latin1.txt", []byte("caf\xe9\n"), base, 400},
		{"UTF-16 file", "unicode.txt", []byte{0xFF, 0xFE, 0x41, 0x00}, base, 400},
		{"oversized file", "big.txt", bytes.Repeat([]byte("a"), maxTextUploadBytes+1), base, 400},
		{"missing request id", "text.txt", []byte("Body.\n"), map[string]string{"title": "Invalid Upload", "language": "en"}, 400},
		{"missing title", "text.txt", []byte("Body.\n"), map[string]string{"requestId": "upload-invalid-0002", "language": "en"}, 400},
		{"unsupported language", "text.txt", []byte("Body.\n"), map[string]string{"requestId": "upload-invalid-0004", "title": "Invalid Upload", "language": "fr"}, 400},
		{"short request id", "text.txt", []byte("Body.\n"), map[string]string{"requestId": "short", "title": "Invalid Upload", "language": "en"}, 400},
		{"unknown field", "text.txt", []byte("Body.\n"), map[string]string{"requestId": "upload-invalid-0005", "title": "Invalid Upload", "language": "en", "ownerId": "9"}, 400},
	}
	for _, tc := range cases {
		code, msg, _ := uploadFile(t, r, learner.Token, "/api/v1/books/upload", tc.fileName, tc.content, tc.fields)
		if code != tc.status {
			t.Fatalf("%s: status %d (%s), want %d", tc.name, code, msg, tc.status)
		}
		if msg == "" {
			t.Fatalf("%s: rejection without a readable message", tc.name)
		}
	}
	// No rejected submission left a book behind.
	_, list := bookList(t, r, learner.Token)
	if len(list.Items) != 0 {
		t.Fatalf("rejected uploads created books: %+v", list.Items)
	}

	// A missing language field follows the paste rule and defaults to English.
	code, msg, defaulted := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "default.txt", []byte("Body without language.\n"),
		map[string]string{"requestId": "upload-default-lang-0007", "title": "Default Language"})
	if code != 201 || defaulted.Book == nil || defaulted.Book.Language != "en" {
		t.Fatalf("missing language must default to English: status %d (%s) book %+v", code, msg, defaulted.Book)
	}

	// The uploaded name is only a display string: a traversal-shaped name changes nothing.
	hostile := map[string]string{"requestId": "upload-hostile-0006", "title": "Hostile Name", "language": "en"}
	code, msg, uploaded := uploadFile(t, r, learner.Token, "/api/v1/books/upload", `..\..\windows\system32\evil.txt`, []byte("Hostile but harmless.\n"), hostile)
	if code != 201 {
		t.Fatalf("hostile name status %d (%s)", code, msg)
	}
	var chapter Chapter
	if err := db.First(&chapter, uploaded.Chapter.ID).Error; err != nil {
		t.Fatal(err)
	}
	var book Book
	if err := db.First(&book, uploaded.Book.ID).Error; err != nil {
		t.Fatal(err)
	}
	for _, field := range []string{chapter.Title, chapter.OriginalText, book.Title} {
		if strings.Contains(field, "evil") || strings.Contains(field, "system32") || strings.Contains(field, `..`) {
			t.Fatalf("the uploaded name leaked into stored data: %q", field)
		}
	}

	// An unrelated content type is not a multipart upload.
	request := httptest.NewRequest("POST", "/api/v1/books/upload", strings.NewReader(`{"requestId":"json-upload-0001"}`))
	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Authorization", "Bearer "+learner.Token)
	response := httptest.NewRecorder()
	r.ServeHTTP(response, request)
	if response.Code != 400 {
		t.Fatalf("JSON body accepted as an upload: %d", response.Code)
	}
	// Uploading without a session is rejected before any parsing.
	code, _, _ = uploadFile(t, r, "", "/api/v1/books/upload", "text.txt", []byte("Body.\n"), base)
	if code != 401 {
		t.Fatalf("anonymous upload status %d", code)
	}
}

// TestMySQLTextUploadKeepsChapterLimit proves the byte cap cannot bypass the one-chapter
// code point rule, and that a large but legal file is stored completely.
func TestMySQLTextUploadKeepsChapterLimit(t *testing.T) {
	db, r, owner := libraryFixture(t)
	learner := newLearner(t, r, owner.Token)
	drainIngest(t, db)

	// Exactly at the chapter limit: accepted, and the stored text keeps its full length.
	atLimit := strings.Repeat("a", maxChapterRunes-1) + "\n"
	fields := map[string]string{"requestId": "upload-limit-0001", "title": "At The Limit", "language": "en"}
	code, msg, uploaded := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "limit.txt", []byte(atLimit), fields)
	if code != 201 {
		t.Fatalf("upload at the chapter limit %d (%s)", code, msg)
	}
	drainIngest(t, db)
	if code, read := readChapter(t, r, learner.Token, uploaded.Chapter.ID); code != 200 || read.Chapter.OriginalText != atLimit {
		t.Fatalf("chapter at the limit: status %d, length %d", code, len([]rune(read.Chapter.OriginalText)))
	}
	// One code point more is rejected by the same rule that already applies to a paste.
	overLimit := strings.Repeat("a", maxChapterRunes+1)
	code, msg, _ = uploadFile(t, r, learner.Token, "/api/v1/books/upload", "over.txt", []byte(overLimit), map[string]string{"requestId": "upload-limit-0002", "title": "Over The Limit", "language": "en"})
	if code != 400 || !strings.Contains(msg, "100000") {
		t.Fatalf("upload over the chapter limit: status %d (%s)", code, msg)
	}
}

// TestMySQLTextUploadAndPasteShareOnePipeline checks the two entry points cannot produce a
// second chapter for the same submitted content when the client stays on one request id.
func TestMySQLTextUploadAndPasteShareOnePipeline(t *testing.T) {
	db, r, owner := libraryFixture(t)
	learner := newLearner(t, r, owner.Token)
	drainIngest(t, db)

	fields := map[string]string{"requestId": "upload-shared-0001", "title": "Shared Pipeline", "language": "en"}
	code, msg, uploaded := uploadFile(t, r, learner.Token, "/api/v1/books/upload", "shared.txt", []byte("Shared body.\n"), fields)
	if code != 201 {
		t.Fatalf("upload %d (%s)", code, msg)
	}
	// The same request id through the paste endpoint answers with the uploaded chapter.
	code, pasted := pasteBook(t, r, learner.Token, map[string]string{
		"requestId": "upload-shared-0001", "title": "Shared Pipeline", "text": "Shared body.\n", "language": "en"})
	if code != 200 || !pasted.Duplicate || pasted.Chapter.ID != uploaded.Chapter.ID {
		t.Fatalf("paste after upload %d: %+v", code, pasted)
	}
	var chapters int64
	db.Model(&Chapter{}).Where("book_id = ?", uploaded.Book.ID).Count(&chapters)
	if chapters != 1 {
		t.Fatalf("the two entry points created %d chapters", chapters)
	}
}