godoxy/internal/logging/accesslog/back_scanner_test.go

package accesslog

import (
	"bytes"
	"fmt"
	"math/rand/v2"
	"net/http"
	"net/http/httptest"
	"os"
	"strconv"
	"strings"
	"testing"

	"github.com/spf13/afero"

	strutils "github.com/yusing/goutils/strings"
	"github.com/yusing/goutils/task"
)

func TestBackScanner(t *testing.T) {
	tests := []struct {
		name     string
		input    string
		expected []string
	}{
		{
			name:     "empty file",
			input:    "",
			expected: []string{},
		},
		{
			name:     "single line without newline",
			input:    "single line",
			expected: []string{"single line"},
		},
		{
			name:     "single line with newline",
			input:    "single line\n",
			expected: []string{"single line"},
		},
		{
			name:     "multiple lines",
			input:    "first\nsecond\nthird\n",
			expected: []string{"third", "second", "first"},
		},
		{
			name:     "multiple lines without final newline",
			input:    "first\nsecond\nthird",
			expected: []string{"third", "second", "first"},
		},
		{
			name:     "lines longer than chunk size",
			input:    "short\n" + strings.Repeat("a", 20) + "\nshort\n",
			expected: []string{"short", strings.Repeat("a", 20), "short"},
		},
		{
			name:     "empty lines",
			input:    "first\n\n\nlast\n",
			expected: []string{"last", "first"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// Setup mock file
			mockFile := NewMockFile(false)
			_, err := mockFile.Write([]byte(tt.input))
			if err != nil {
				t.Fatalf("failed to write to mock file: %v", err)
			}

			// Create scanner with small chunk size to test chunking
			scanner := NewBackScanner(mockFile, mockFile.MustSize(), 10)

			// Collect all lines
			var lines [][]byte
			for scanner.Scan() {
				lines = append(lines, scanner.Bytes())
			}

			// Check for scanning errors
			if err := scanner.Err(); err != nil {
				t.Errorf("scanner error: %v", err)
			}

			// Compare results
			if len(lines) != len(tt.expected) {
				t.Errorf("got %d lines, want %d lines", len(lines), len(tt.expected))
				return
			}

			for i, line := range lines {
				if string(line) != tt.expected[i] {
					t.Errorf("line %d: got %q, want %q", i, line, tt.expected[i])
				}
			}
		})
	}
}

func TestBackScannerWithVaryingChunkSizes(t *testing.T) {
	input := "first\nsecond\nthird\nfourth\nfifth\n"
	expected := []string{"fifth", "fourth", "third", "second", "first"}
	chunkSizes := []int{1, 2, 3, 5, 10, 20, 100}

	for _, chunkSize := range chunkSizes {
		t.Run(fmt.Sprintf("chunk_size_%d", chunkSize), func(t *testing.T) {
			mockFile := NewMockFile(false)
			_, err := mockFile.Write([]byte(input))
			if err != nil {
				t.Fatalf("failed to write to mock file: %v", err)
			}

			scanner := NewBackScanner(mockFile, mockFile.MustSize(), chunkSize)

			var lines [][]byte
			for scanner.Scan() {
				lines = append(lines, scanner.Bytes())
			}

			if err := scanner.Err(); err != nil {
				t.Errorf("scanner error: %v", err)
			}

			if len(lines) != len(expected) {
				t.Errorf("got %d lines, want %d lines", len(lines), len(expected))
				return
			}

			for i, line := range lines {
				if string(line) != expected[i] {
					t.Errorf("chunk size %d, line %d: got %q, want %q",
						chunkSize, i, line, expected[i])
				}
			}
		})
	}
}

var logEntry = func() func() []byte {
	accesslog := NewMockAccessLogger(task.RootTask("test", false), &RequestLoggerConfig{
		Format: FormatJSON,
	})

	contentTypes := []string{"application/json", "text/html", "text/plain", "application/xml", "application/x-www-form-urlencoded"}
	userAgents := []string{"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/120.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/120.0", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/120.0"}
	methods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"}
	paths := []string{"/", "/about", "/contact", "/login", "/logout", "/register", "/profile"}

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		allocSize := rand.IntN(8192)
		w.Header().Set("Content-Type", contentTypes[rand.IntN(len(contentTypes))])
		w.Header().Set("Content-Length", strconv.Itoa(allocSize))
		w.WriteHeader(http.StatusOK)
	}))
	srv.URL = "http://localhost:8080"

	return func() []byte {
		// make a request to the server
		req, _ := http.NewRequest(http.MethodGet, srv.URL, nil)
		res := httptest.NewRecorder()
		req.Header.Set("User-Agent", userAgents[rand.IntN(len(userAgents))])
		req.Method = methods[rand.IntN(len(methods))]
		req.URL.Path = paths[rand.IntN(len(paths))]
		// server the request
		srv.Config.Handler.ServeHTTP(res, req)
		b := bytes.NewBuffer(make([]byte, 0, 1024))
		accesslog.(RequestFormatter).AppendRequestLog(b, req, res.Result())
		return b.Bytes()
	}
}()

// 100000 log entries.
func BenchmarkBackScannerRealFile(b *testing.B) {
	file, err := afero.TempFile(afero.NewOsFs(), "", "accesslog")
	if err != nil {
		b.Fatalf("failed to create temp file: %v", err)
	}
	defer os.Remove(file.Name())

	buf := bytes.NewBuffer(nil)
	for range 100000 {
		buf.Write(logEntry())
	}

	fSize := int64(buf.Len())
	_, err = file.Write(buf.Bytes())
	if err != nil {
		b.Fatalf("failed to write to file: %v", err)
	}

	// file position does not matter, Seek not needed

	for i := range 12 {
		chunkSize := (2 << i) * kilobyte
		name := strutils.FormatByteSize(chunkSize)
		b.ResetTimer()
		b.Run(name, func(b *testing.B) {
			for b.Loop() {
				scanner := NewBackScanner(file, fSize, chunkSize)
				for scanner.Scan() {
				}
				scanner.Release()
			}
		})
	}
}

/*
BenchmarkBackScannerRealFile
BenchmarkBackScannerRealFile/2_KiB
BenchmarkBackScannerRealFile/2_KiB-10                 21          51796773 ns/op             619 B/op          1 allocs/op
BenchmarkBackScannerRealFile/4_KiB
BenchmarkBackScannerRealFile/4_KiB-10                 36          32081281 ns/op             699 B/op          1 allocs/op
BenchmarkBackScannerRealFile/8_KiB
BenchmarkBackScannerRealFile/8_KiB-10                 57          22155619 ns/op             847 B/op          1 allocs/op
BenchmarkBackScannerRealFile/16_KiB
BenchmarkBackScannerRealFile/16_KiB-10                62          21323125 ns/op            1449 B/op          1 allocs/op
BenchmarkBackScannerRealFile/32_KiB
BenchmarkBackScannerRealFile/32_KiB-10                63          17534883 ns/op            2729 B/op          1 allocs/op
BenchmarkBackScannerRealFile/64_KiB
BenchmarkBackScannerRealFile/64_KiB-10                73          17877029 ns/op            4617 B/op          1 allocs/op
BenchmarkBackScannerRealFile/128_KiB
BenchmarkBackScannerRealFile/128_KiB-10               75          17797267 ns/op            8866 B/op          1 allocs/op
BenchmarkBackScannerRealFile/256_KiB
BenchmarkBackScannerRealFile/256_KiB-10               67          16732108 ns/op           19691 B/op          1 allocs/op
BenchmarkBackScannerRealFile/512_KiB
BenchmarkBackScannerRealFile/512_KiB-10               70          17121683 ns/op           37577 B/op          1 allocs/op
BenchmarkBackScannerRealFile/1_MiB
BenchmarkBackScannerRealFile/1_MiB-10                 51          19615791 ns/op          102930 B/op          1 allocs/op
BenchmarkBackScannerRealFile/2_MiB
BenchmarkBackScannerRealFile/2_MiB-10                 26          41744928 ns/op        77595287 B/op         57 allocs/op
BenchmarkBackScannerRealFile/4_MiB
BenchmarkBackScannerRealFile/4_MiB-10                 22          48081521 ns/op        79692224 B/op         49 allocs/op
*/