mirror of
https://github.com/yusing/godoxy.git
synced 2026-04-23 01:08:47 +02:00
refactor: move accesslog to logging/accesslog
This commit is contained in:
yusing
268
internal/logging/accesslog/back_scanner_test.go
Normal file
268
internal/logging/accesslog/back_scanner_test.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/yusing/go-proxy/internal/task"
|
||||
"github.com/yusing/go-proxy/internal/utils/strutils"
|
||||
expect "github.com/yusing/go-proxy/internal/utils/testing"
|
||||
)
|
||||
|
||||
func TestBackScanner(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "empty file",
|
||||
input: "",
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "single line without newline",
|
||||
input: "single line",
|
||||
expected: []string{"single line"},
|
||||
},
|
||||
{
|
||||
name: "single line with newline",
|
||||
input: "single line\n",
|
||||
expected: []string{"single line"},
|
||||
},
|
||||
{
|
||||
name: "multiple lines",
|
||||
input: "first\nsecond\nthird\n",
|
||||
expected: []string{"third", "second", "first"},
|
||||
},
|
||||
{
|
||||
name: "multiple lines without final newline",
|
||||
input: "first\nsecond\nthird",
|
||||
expected: []string{"third", "second", "first"},
|
||||
},
|
||||
{
|
||||
name: "lines longer than chunk size",
|
||||
input: "short\n" + strings.Repeat("a", 20) + "\nshort\n",
|
||||
expected: []string{"short", strings.Repeat("a", 20), "short"},
|
||||
},
|
||||
{
|
||||
name: "empty lines",
|
||||
input: "first\n\n\nlast\n",
|
||||
expected: []string{"last", "first"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Setup mock file
|
||||
mockFile := NewMockFile()
|
||||
_, err := mockFile.Write([]byte(tt.input))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write to mock file: %v", err)
|
||||
}
|
||||
|
||||
// Create scanner with small chunk size to test chunking
|
||||
scanner := NewBackScanner(mockFile, 10)
|
||||
|
||||
// Collect all lines
|
||||
var lines [][]byte
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Bytes())
|
||||
}
|
||||
|
||||
// Check for scanning errors
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Errorf("scanner error: %v", err)
|
||||
}
|
||||
|
||||
// Compare results
|
||||
if len(lines) != len(tt.expected) {
|
||||
t.Errorf("got %d lines, want %d lines", len(lines), len(tt.expected))
|
||||
return
|
||||
}
|
||||
|
||||
for i, line := range lines {
|
||||
if string(line) != tt.expected[i] {
|
||||
t.Errorf("line %d: got %q, want %q", i, line, tt.expected[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackScannerWithVaryingChunkSizes(t *testing.T) {
|
||||
input := "first\nsecond\nthird\nfourth\nfifth\n"
|
||||
expected := []string{"fifth", "fourth", "third", "second", "first"}
|
||||
chunkSizes := []int{1, 2, 3, 5, 10, 20, 100}
|
||||
|
||||
for _, chunkSize := range chunkSizes {
|
||||
t.Run(fmt.Sprintf("chunk_size_%d", chunkSize), func(t *testing.T) {
|
||||
mockFile := NewMockFile()
|
||||
_, err := mockFile.Write([]byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write to mock file: %v", err)
|
||||
}
|
||||
|
||||
scanner := NewBackScanner(mockFile, chunkSize)
|
||||
|
||||
var lines [][]byte
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Bytes())
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Errorf("scanner error: %v", err)
|
||||
}
|
||||
|
||||
if len(lines) != len(expected) {
|
||||
t.Errorf("got %d lines, want %d lines", len(lines), len(expected))
|
||||
return
|
||||
}
|
||||
|
||||
for i, line := range lines {
|
||||
if string(line) != expected[i] {
|
||||
t.Errorf("chunk size %d, line %d: got %q, want %q",
|
||||
chunkSize, i, line, expected[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func logEntry() []byte {
|
||||
accesslog := NewMockAccessLogger(task.RootTask("test", false), &Config{
|
||||
Format: FormatJSON,
|
||||
})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("hello"))
|
||||
}))
|
||||
srv.URL = "http://localhost:8080"
|
||||
defer srv.Close()
|
||||
// make a request to the server
|
||||
req, _ := http.NewRequest("GET", srv.URL, nil)
|
||||
res := httptest.NewRecorder()
|
||||
// server the request
|
||||
srv.Config.Handler.ServeHTTP(res, req)
|
||||
b := accesslog.AppendLog(nil, req, res.Result())
|
||||
if b[len(b)-1] != '\n' {
|
||||
b = append(b, '\n')
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestReset(t *testing.T) {
|
||||
file, err := afero.TempFile(afero.NewOsFs(), "", "accesslog")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create temp file: %v", err)
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
line := logEntry()
|
||||
nLines := 1000
|
||||
for range nLines {
|
||||
_, err := file.Write(line)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write to temp file: %v", err)
|
||||
}
|
||||
}
|
||||
linesRead := 0
|
||||
s := NewBackScanner(file, defaultChunkSize)
|
||||
for s.Scan() {
|
||||
linesRead++
|
||||
}
|
||||
if err := s.Err(); err != nil {
|
||||
t.Errorf("scanner error: %v", err)
|
||||
}
|
||||
expect.Equal(t, linesRead, nLines)
|
||||
s.Reset()
|
||||
|
||||
linesRead = 0
|
||||
for s.Scan() {
|
||||
linesRead++
|
||||
}
|
||||
if err := s.Err(); err != nil {
|
||||
t.Errorf("scanner error: %v", err)
|
||||
}
|
||||
expect.Equal(t, linesRead, nLines)
|
||||
}
|
||||
|
||||
// 100000 log entries
|
||||
func BenchmarkBackScanner(b *testing.B) {
|
||||
mockFile := NewMockFile()
|
||||
line := logEntry()
|
||||
for range 100000 {
|
||||
_, _ = mockFile.Write(line)
|
||||
}
|
||||
for i := range 14 {
|
||||
chunkSize := (2 << i) * kilobyte
|
||||
scanner := NewBackScanner(mockFile, chunkSize)
|
||||
name := strutils.FormatByteSize(chunkSize)
|
||||
b.ResetTimer()
|
||||
b.Run(name, func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
_ = scanner.Reset()
|
||||
for scanner.Scan() {
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBackScannerRealFile(b *testing.B) {
|
||||
file, err := afero.TempFile(afero.NewOsFs(), "", "accesslog")
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create temp file: %v", err)
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
|
||||
for range 10000 {
|
||||
_, err = file.Write(logEntry())
|
||||
if err != nil {
|
||||
b.Fatalf("failed to write to temp file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
scanner := NewBackScanner(file, 256*kilobyte)
|
||||
b.ResetTimer()
|
||||
for scanner.Scan() {
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
b.Errorf("scanner error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
BenchmarkBackScanner
|
||||
BenchmarkBackScanner/2_KiB
|
||||
BenchmarkBackScanner/2_KiB-20 52 23254071 ns/op 67596663 B/op 26420 allocs/op
|
||||
BenchmarkBackScanner/4_KiB
|
||||
BenchmarkBackScanner/4_KiB-20 55 20961059 ns/op 62529378 B/op 13211 allocs/op
|
||||
BenchmarkBackScanner/8_KiB
|
||||
BenchmarkBackScanner/8_KiB-20 64 18242460 ns/op 62951141 B/op 6608 allocs/op
|
||||
BenchmarkBackScanner/16_KiB
|
||||
BenchmarkBackScanner/16_KiB-20 52 20162076 ns/op 62940256 B/op 3306 allocs/op
|
||||
BenchmarkBackScanner/32_KiB
|
||||
BenchmarkBackScanner/32_KiB-20 54 19247968 ns/op 67553645 B/op 1656 allocs/op
|
||||
BenchmarkBackScanner/64_KiB
|
||||
BenchmarkBackScanner/64_KiB-20 60 20909046 ns/op 64053342 B/op 827 allocs/op
|
||||
BenchmarkBackScanner/128_KiB
|
||||
BenchmarkBackScanner/128_KiB-20 68 17759890 ns/op 62201945 B/op 414 allocs/op
|
||||
BenchmarkBackScanner/256_KiB
|
||||
BenchmarkBackScanner/256_KiB-20 52 19531877 ns/op 61030487 B/op 208 allocs/op
|
||||
BenchmarkBackScanner/512_KiB
|
||||
BenchmarkBackScanner/512_KiB-20 54 19124656 ns/op 61030485 B/op 208 allocs/op
|
||||
BenchmarkBackScanner/1_MiB
|
||||
BenchmarkBackScanner/1_MiB-20 67 17078936 ns/op 61030495 B/op 208 allocs/op
|
||||
BenchmarkBackScanner/2_MiB
|
||||
BenchmarkBackScanner/2_MiB-20 66 18467421 ns/op 61030492 B/op 208 allocs/op
|
||||
BenchmarkBackScanner/4_MiB
|
||||
BenchmarkBackScanner/4_MiB-20 68 17214573 ns/op 61030486 B/op 208 allocs/op
|
||||
BenchmarkBackScanner/8_MiB
|
||||
BenchmarkBackScanner/8_MiB-20 57 18235229 ns/op 61030492 B/op 208 allocs/op
|
||||
BenchmarkBackScanner/16_MiB
|
||||
BenchmarkBackScanner/16_MiB-20 57 19343441 ns/op 61030499 B/op 208 allocs/op
|
||||
*/
|
||||
Reference in New Issue
Block a user