mirror of
https://github.com/yusing/godoxy.git
synced 2026-02-23 17:24:58 +01:00
refactor(accesslog): restructure access logging; enhance console output format
Major refactoring of the access logging infrastructure to improve code organization and add proper console/stdout logging support. - Renamed `Writer` interface to `File` and consolidated with `SupportRotate` - Renamed `Log(req, res)` to `LogRequest(req, res)` for clarity - Added new `ConsoleLogger` with zerolog console writer for formatted stdout output - Moved type definitions to new `types.go` file - Changed buffer handling from `[]byte` returns to `*bytes.Buffer` parameters - Renamed internal files for clarity (`access_logger.go` → `file_access_logger.go`) - Fixed fileserver access logging timing: moved logging after handler execution with defer - Correct response handling in Fileserver - Remove deprecated field `buffer_size` - Simplify and removed unnecessary code All callers have been updated to use the new APIs.
This commit is contained in:
committed by
github-actions[bot]
parent
235af71343
commit
bd1ff9731d
2
goutils
2
goutils
Submodule goutils updated: 061245e969...3aa9b8d869
@@ -4191,12 +4191,6 @@
|
||||
"RequestLoggerConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"buffer_size": {
|
||||
"description": "Deprecated: buffer size is adjusted dynamically",
|
||||
"type": "integer",
|
||||
"x-nullable": false,
|
||||
"x-omitempty": false
|
||||
},
|
||||
"fields": {
|
||||
"$ref": "#/definitions/accesslog.Fields",
|
||||
"x-nullable": false,
|
||||
|
||||
@@ -879,9 +879,6 @@ definitions:
|
||||
type: object
|
||||
RequestLoggerConfig:
|
||||
properties:
|
||||
buffer_size:
|
||||
description: 'Deprecated: buffer size is adjusted dynamically'
|
||||
type: integer
|
||||
fields:
|
||||
$ref: '#/definitions/accesslog.Fields'
|
||||
filters:
|
||||
|
||||
@@ -100,7 +100,7 @@ func (ep *Entrypoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
rec := accesslog.GetResponseRecorder(w)
|
||||
w = rec
|
||||
defer func() {
|
||||
ep.accessLogger.Log(r, rec.Response())
|
||||
ep.accessLogger.LogRequest(r, rec.Response())
|
||||
accesslog.PutResponseRecorder(rec)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/yusing/godoxy/internal/serialization"
|
||||
@@ -9,16 +10,15 @@ import (
|
||||
|
||||
type (
|
||||
ConfigBase struct {
|
||||
B int `json:"buffer_size"` // Deprecated: buffer size is adjusted dynamically
|
||||
Path string `json:"path"`
|
||||
Stdout bool `json:"stdout"`
|
||||
Retention *Retention `json:"retention" aliases:"keep"`
|
||||
RotateInterval time.Duration `json:"rotate_interval,omitempty" swaggertype:"primitive,integer"`
|
||||
}
|
||||
} // @name AccessLoggerConfigBase
|
||||
ACLLoggerConfig struct {
|
||||
ConfigBase
|
||||
LogAllowed bool `json:"log_allowed"`
|
||||
}
|
||||
} // @name ACLLoggerConfig
|
||||
RequestLoggerConfig struct {
|
||||
ConfigBase
|
||||
Format Format `json:"format" validate:"oneof=common combined json"`
|
||||
@@ -32,7 +32,7 @@ type (
|
||||
}
|
||||
AnyConfig interface {
|
||||
ToConfig() *Config
|
||||
Writers() ([]Writer, error)
|
||||
Writers() ([]File, error)
|
||||
}
|
||||
|
||||
Format string
|
||||
@@ -66,17 +66,17 @@ func (cfg *ConfigBase) Validate() gperr.Error {
|
||||
}
|
||||
|
||||
// Writers returns a list of writers for the config.
|
||||
func (cfg *ConfigBase) Writers() ([]Writer, error) {
|
||||
writers := make([]Writer, 0, 2)
|
||||
func (cfg *ConfigBase) Writers() ([]File, error) {
|
||||
writers := make([]File, 0, 2)
|
||||
if cfg.Path != "" {
|
||||
io, err := NewFileIO(cfg.Path)
|
||||
f, err := OpenFile(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writers = append(writers, io)
|
||||
writers = append(writers, f)
|
||||
}
|
||||
if cfg.Stdout {
|
||||
writers = append(writers, NewStdout())
|
||||
writers = append(writers, stdout)
|
||||
}
|
||||
return writers, nil
|
||||
}
|
||||
@@ -95,6 +95,16 @@ func (cfg *RequestLoggerConfig) ToConfig() *Config {
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg *Config) ShouldLogRequest(req *http.Request, res *http.Response) bool {
|
||||
if cfg.req == nil {
|
||||
return true
|
||||
}
|
||||
return cfg.req.Filters.StatusCodes.CheckKeep(req, res) &&
|
||||
cfg.req.Filters.Method.CheckKeep(req, res) &&
|
||||
cfg.req.Filters.Headers.CheckKeep(req, res) &&
|
||||
cfg.req.Filters.CIDR.CheckKeep(req, res)
|
||||
}
|
||||
|
||||
func DefaultRequestLoggerConfig() *RequestLoggerConfig {
|
||||
return &RequestLoggerConfig{
|
||||
ConfigBase: ConfigBase{
|
||||
|
||||
73
internal/logging/accesslog/console_logger.go
Normal file
73
internal/logging/accesslog/console_logger.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
maxmind "github.com/yusing/godoxy/internal/maxmind/types"
|
||||
)
|
||||
|
||||
type ConsoleLogger struct {
|
||||
cfg *Config
|
||||
|
||||
formatter ConsoleFormatter
|
||||
}
|
||||
|
||||
var stdoutLogger = func() *zerolog.Logger {
|
||||
l := zerolog.New(zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
|
||||
w.Out = os.Stdout
|
||||
w.TimeFormat = zerolog.TimeFieldFormat
|
||||
w.FieldsOrder = []string{
|
||||
"uri", "protocol", "type", "size",
|
||||
"useragent", "query", "headers", "cookies",
|
||||
"error", "iso_code", "time_zone"}
|
||||
})).With().Str("level", zerolog.InfoLevel.String()).Timestamp().Logger()
|
||||
return &l
|
||||
}()
|
||||
|
||||
// placeholder for console logger
|
||||
var stdout File = &sharedFileHandle{}
|
||||
|
||||
func NewConsoleLogger(cfg *Config) AccessLogger {
|
||||
if cfg == nil {
|
||||
panic("accesslog: NewConsoleLogger called with nil config")
|
||||
}
|
||||
l := &ConsoleLogger{
|
||||
cfg: cfg,
|
||||
}
|
||||
if cfg.req != nil {
|
||||
l.formatter = ConsoleFormatter{cfg: &cfg.req.Fields}
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (l *ConsoleLogger) Config() *Config {
|
||||
return l.cfg
|
||||
}
|
||||
|
||||
func (l *ConsoleLogger) LogRequest(req *http.Request, res *http.Response) {
|
||||
if !l.cfg.ShouldLogRequest(req, res) {
|
||||
return
|
||||
}
|
||||
|
||||
l.formatter.LogRequestZeroLog(stdoutLogger, req, res)
|
||||
}
|
||||
|
||||
func (l *ConsoleLogger) LogError(req *http.Request, err error) {
|
||||
log := stdoutLogger.With().Err(err).Logger()
|
||||
l.formatter.LogRequestZeroLog(&log, req, internalErrorResponse)
|
||||
}
|
||||
|
||||
func (l *ConsoleLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
|
||||
ConsoleACLFormatter{}.LogACLZeroLog(stdoutLogger, info, blocked)
|
||||
}
|
||||
|
||||
func (l *ConsoleLogger) Flush() {
|
||||
// No-op for console logger
|
||||
}
|
||||
|
||||
func (l *ConsoleLogger) Close() error {
|
||||
// No-op for console logger
|
||||
return nil
|
||||
}
|
||||
@@ -20,25 +20,20 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
AccessLogger interface {
|
||||
Log(req *http.Request, res *http.Response)
|
||||
LogError(req *http.Request, err error)
|
||||
LogACL(info *maxmind.IPInfo, blocked bool)
|
||||
|
||||
Config() *Config
|
||||
|
||||
Flush()
|
||||
Close() error
|
||||
File interface {
|
||||
io.WriteCloser
|
||||
supportRotate
|
||||
Name() string
|
||||
}
|
||||
|
||||
accessLogger struct {
|
||||
fileAccessLogger struct {
|
||||
task *task.Task
|
||||
cfg *Config
|
||||
|
||||
writer BufferedWriter
|
||||
supportRotate SupportRotate
|
||||
writeLock *sync.Mutex
|
||||
closed bool
|
||||
writer BufferedWriter
|
||||
file File
|
||||
writeLock *sync.Mutex
|
||||
closed bool
|
||||
|
||||
writeCount int64
|
||||
bufSize int
|
||||
@@ -48,32 +43,7 @@ type (
|
||||
logger zerolog.Logger
|
||||
|
||||
RequestFormatter
|
||||
ACLFormatter
|
||||
}
|
||||
|
||||
Writer interface {
|
||||
io.WriteCloser
|
||||
ShouldBeBuffered() bool
|
||||
Name() string // file name or path
|
||||
}
|
||||
|
||||
SupportRotate interface {
|
||||
io.Writer
|
||||
supportRotate
|
||||
Name() string
|
||||
}
|
||||
|
||||
AccessLogRotater interface {
|
||||
Rotate(result *RotateResult) (rotated bool, err error)
|
||||
}
|
||||
|
||||
RequestFormatter interface {
|
||||
// AppendRequestLog appends a log line to line with or without a trailing newline
|
||||
AppendRequestLog(line []byte, req *http.Request, res *http.Response) []byte
|
||||
}
|
||||
ACLFormatter interface {
|
||||
// AppendACLLog appends a log line to line with or without a trailing newline
|
||||
AppendACLLog(line []byte, info *maxmind.IPInfo, blocked bool) []byte
|
||||
ACLLogFormatter
|
||||
}
|
||||
)
|
||||
|
||||
@@ -96,112 +66,87 @@ const (
|
||||
var bytesPool = synk.GetUnsizedBytesPool()
|
||||
var sizedPool = synk.GetSizedBytesPool()
|
||||
|
||||
func NewAccessLogger(parent task.Parent, cfg AnyConfig) (AccessLogger, error) {
|
||||
writers, err := cfg.Writers()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewMultiAccessLogger(parent, cfg, writers), nil
|
||||
}
|
||||
|
||||
func NewMockAccessLogger(parent task.Parent, cfg *RequestLoggerConfig) AccessLogger {
|
||||
return NewAccessLoggerWithIO(parent, NewMockFile(true), cfg)
|
||||
}
|
||||
|
||||
func NewAccessLoggerWithIO(parent task.Parent, writer Writer, anyCfg AnyConfig) AccessLogger {
|
||||
func NewFileAccessLogger(parent task.Parent, file File, anyCfg AnyConfig) AccessLogger {
|
||||
cfg := anyCfg.ToConfig()
|
||||
if cfg.RotateInterval == 0 {
|
||||
cfg.RotateInterval = defaultRotateInterval
|
||||
}
|
||||
|
||||
l := &accessLogger{
|
||||
task: parent.Subtask("accesslog."+writer.Name(), true),
|
||||
name := file.Name()
|
||||
l := &fileAccessLogger{
|
||||
task: parent.Subtask("accesslog."+name, true),
|
||||
cfg: cfg,
|
||||
bufSize: InitialBufferSize,
|
||||
errRateLimiter: rate.NewLimiter(rate.Every(errRateLimit), errBurst),
|
||||
logger: log.With().Str("file", writer.Name()).Logger(),
|
||||
logger: log.With().Str("file", name).Logger(),
|
||||
}
|
||||
|
||||
l.writeLock, _ = writerLocks.LoadOrStore(writer.Name(), &sync.Mutex{})
|
||||
l.writeLock, _ = writerLocks.LoadOrStore(name, &sync.Mutex{})
|
||||
|
||||
if writer.ShouldBeBuffered() {
|
||||
l.writer = ioutils.NewBufferedWriter(writer, InitialBufferSize)
|
||||
} else {
|
||||
l.writer = NewUnbufferedWriter(writer)
|
||||
}
|
||||
|
||||
if supportRotate, ok := writer.(SupportRotate); ok {
|
||||
l.supportRotate = supportRotate
|
||||
}
|
||||
l.writer = ioutils.NewBufferedWriter(file, InitialBufferSize)
|
||||
l.file = file
|
||||
|
||||
if cfg.req != nil {
|
||||
fmt := CommonFormatter{cfg: &cfg.req.Fields}
|
||||
switch cfg.req.Format {
|
||||
case FormatCommon:
|
||||
l.RequestFormatter = &fmt
|
||||
l.RequestFormatter = CommonFormatter{cfg: &cfg.req.Fields}
|
||||
case FormatCombined:
|
||||
l.RequestFormatter = &CombinedFormatter{fmt}
|
||||
l.RequestFormatter = CombinedFormatter{CommonFormatter{cfg: &cfg.req.Fields}}
|
||||
case FormatJSON:
|
||||
l.RequestFormatter = &JSONFormatter{fmt}
|
||||
l.RequestFormatter = JSONFormatter{cfg: &cfg.req.Fields}
|
||||
default: // should not happen, validation has done by validate tags
|
||||
panic("invalid access log format")
|
||||
}
|
||||
} else {
|
||||
l.ACLFormatter = ACLLogFormatter{}
|
||||
}
|
||||
|
||||
go l.start()
|
||||
return l
|
||||
}
|
||||
|
||||
func (l *accessLogger) Config() *Config {
|
||||
func (l *fileAccessLogger) Config() *Config {
|
||||
return l.cfg
|
||||
}
|
||||
|
||||
func (l *accessLogger) shouldLog(req *http.Request, res *http.Response) bool {
|
||||
if !l.cfg.req.Filters.StatusCodes.CheckKeep(req, res) ||
|
||||
!l.cfg.req.Filters.Method.CheckKeep(req, res) ||
|
||||
!l.cfg.req.Filters.Headers.CheckKeep(req, res) ||
|
||||
!l.cfg.req.Filters.CIDR.CheckKeep(req, res) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *accessLogger) Log(req *http.Request, res *http.Response) {
|
||||
if !l.shouldLog(req, res) {
|
||||
func (l *fileAccessLogger) LogRequest(req *http.Request, res *http.Response) {
|
||||
if !l.cfg.ShouldLogRequest(req, res) {
|
||||
return
|
||||
}
|
||||
|
||||
line := bytesPool.Get()
|
||||
line = l.AppendRequestLog(line, req, res)
|
||||
if line[len(line)-1] != '\n' {
|
||||
line = append(line, '\n')
|
||||
line := bytesPool.GetBuffer()
|
||||
defer bytesPool.PutBuffer(line)
|
||||
l.AppendRequestLog(line, req, res)
|
||||
// line is never empty
|
||||
if line.Bytes()[line.Len()-1] != '\n' {
|
||||
line.WriteByte('\n')
|
||||
}
|
||||
l.write(line)
|
||||
bytesPool.Put(line)
|
||||
l.write(line.Bytes())
|
||||
}
|
||||
|
||||
func (l *accessLogger) LogError(req *http.Request, err error) {
|
||||
l.Log(req, &http.Response{StatusCode: http.StatusInternalServerError, Status: err.Error()})
|
||||
var internalErrorResponse = &http.Response{
|
||||
StatusCode: http.StatusInternalServerError,
|
||||
Status: http.StatusText(http.StatusInternalServerError),
|
||||
}
|
||||
|
||||
func (l *accessLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
|
||||
line := bytesPool.Get()
|
||||
line = l.AppendACLLog(line, info, blocked)
|
||||
if line[len(line)-1] != '\n' {
|
||||
line = append(line, '\n')
|
||||
func (l *fileAccessLogger) LogError(req *http.Request, err error) {
|
||||
l.LogRequest(req, internalErrorResponse)
|
||||
}
|
||||
|
||||
func (l *fileAccessLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
|
||||
line := bytesPool.GetBuffer()
|
||||
defer bytesPool.PutBuffer(line)
|
||||
l.AppendACLLog(line, info, blocked)
|
||||
// line is never empty
|
||||
if line.Bytes()[line.Len()-1] != '\n' {
|
||||
line.WriteByte('\n')
|
||||
}
|
||||
l.write(line)
|
||||
bytesPool.Put(line)
|
||||
l.write(line.Bytes())
|
||||
}
|
||||
|
||||
func (l *accessLogger) ShouldRotate() bool {
|
||||
return l.supportRotate != nil && l.cfg.Retention.IsValid()
|
||||
func (l *fileAccessLogger) ShouldRotate() bool {
|
||||
return l.cfg.Retention.IsValid()
|
||||
}
|
||||
|
||||
func (l *accessLogger) Rotate(result *RotateResult) (rotated bool, err error) {
|
||||
func (l *fileAccessLogger) Rotate(result *RotateResult) (rotated bool, err error) {
|
||||
if !l.ShouldRotate() {
|
||||
return false, nil
|
||||
}
|
||||
@@ -210,11 +155,11 @@ func (l *accessLogger) Rotate(result *RotateResult) (rotated bool, err error) {
|
||||
l.writeLock.Lock()
|
||||
defer l.writeLock.Unlock()
|
||||
|
||||
rotated, err = rotateLogFile(l.supportRotate, l.cfg.Retention, result)
|
||||
rotated, err = rotateLogFile(l.file, l.cfg.Retention, result)
|
||||
return
|
||||
}
|
||||
|
||||
func (l *accessLogger) handleErr(err error) {
|
||||
func (l *fileAccessLogger) handleErr(err error) {
|
||||
if l.errRateLimiter.Allow() {
|
||||
gperr.LogError("failed to write access log", err, &l.logger)
|
||||
} else {
|
||||
@@ -223,7 +168,7 @@ func (l *accessLogger) handleErr(err error) {
|
||||
}
|
||||
}
|
||||
|
||||
func (l *accessLogger) start() {
|
||||
func (l *fileAccessLogger) start() {
|
||||
defer func() {
|
||||
l.Flush()
|
||||
l.Close()
|
||||
@@ -259,7 +204,7 @@ func (l *accessLogger) start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (l *accessLogger) Close() error {
|
||||
func (l *fileAccessLogger) Close() error {
|
||||
l.writeLock.Lock()
|
||||
defer l.writeLock.Unlock()
|
||||
if l.closed {
|
||||
@@ -270,7 +215,7 @@ func (l *accessLogger) Close() error {
|
||||
return l.writer.Close()
|
||||
}
|
||||
|
||||
func (l *accessLogger) Flush() {
|
||||
func (l *fileAccessLogger) Flush() {
|
||||
l.writeLock.Lock()
|
||||
defer l.writeLock.Unlock()
|
||||
if l.closed {
|
||||
@@ -279,7 +224,7 @@ func (l *accessLogger) Flush() {
|
||||
l.writer.Flush()
|
||||
}
|
||||
|
||||
func (l *accessLogger) write(data []byte) {
|
||||
func (l *fileAccessLogger) write(data []byte) {
|
||||
l.writeLock.Lock()
|
||||
defer l.writeLock.Unlock()
|
||||
if l.closed {
|
||||
@@ -294,7 +239,7 @@ func (l *accessLogger) write(data []byte) {
|
||||
atomic.AddInt64(&l.writeCount, int64(n))
|
||||
}
|
||||
|
||||
func (l *accessLogger) adjustBuffer() {
|
||||
func (l *fileAccessLogger) adjustBuffer() {
|
||||
wps := int(atomic.SwapInt64(&l.writeCount, 0)) / int(bufferAdjustInterval.Seconds())
|
||||
origBufSize := l.bufSize
|
||||
newBufSize := origBufSize
|
||||
@@ -1,6 +1,7 @@
|
||||
package accesslog_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -53,13 +54,13 @@ var (
|
||||
)
|
||||
|
||||
func fmtLog(cfg *RequestLoggerConfig) (ts string, line string) {
|
||||
buf := make([]byte, 0, 1024)
|
||||
buf := bytes.NewBuffer(make([]byte, 0, 1024))
|
||||
|
||||
t := time.Now()
|
||||
logger := NewMockAccessLogger(testTask, cfg)
|
||||
mockable.MockTimeNow(t)
|
||||
buf = logger.(RequestFormatter).AppendRequestLog(buf, req, resp)
|
||||
return t.Format(LogTimeFormat), string(buf)
|
||||
logger.(RequestFormatter).AppendRequestLog(buf, req, resp)
|
||||
return t.Format(LogTimeFormat), buf.String()
|
||||
}
|
||||
|
||||
func TestAccessLoggerCommon(t *testing.T) {
|
||||
@@ -141,9 +142,6 @@ func TestAccessLoggerJSON(t *testing.T) {
|
||||
expect.Equal(t, entry.UserAgent, ua)
|
||||
expect.Equal(t, len(entry.Headers), 0)
|
||||
expect.Equal(t, len(entry.Cookies), 0)
|
||||
if status >= 400 {
|
||||
expect.Equal(t, entry.Error, http.StatusText(status))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAccessLoggerJSON(b *testing.B) {
|
||||
@@ -152,7 +150,7 @@ func BenchmarkAccessLoggerJSON(b *testing.B) {
|
||||
logger := NewMockAccessLogger(testTask, config)
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,6 +160,6 @@ func BenchmarkAccessLoggerCombined(b *testing.B) {
|
||||
logger := NewMockAccessLogger(testTask, config)
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
}
|
||||
@@ -16,9 +16,11 @@ type (
|
||||
CommonFormatter struct {
|
||||
cfg *Fields
|
||||
}
|
||||
CombinedFormatter struct{ CommonFormatter }
|
||||
JSONFormatter struct{ CommonFormatter }
|
||||
ACLLogFormatter struct{}
|
||||
CombinedFormatter struct{ CommonFormatter }
|
||||
JSONFormatter struct{ cfg *Fields }
|
||||
ConsoleFormatter struct{ cfg *Fields }
|
||||
ACLLogFormatter struct{}
|
||||
ConsoleACLFormatter struct{}
|
||||
)
|
||||
|
||||
const LogTimeFormat = "02/Jan/2006:15:04:05 -0700"
|
||||
@@ -30,24 +32,26 @@ func scheme(req *http.Request) string {
|
||||
return "http"
|
||||
}
|
||||
|
||||
func appendRequestURI(line []byte, req *http.Request, query iter.Seq2[string, []string]) []byte {
|
||||
func appendRequestURI(line *bytes.Buffer, req *http.Request, query iter.Seq2[string, []string]) {
|
||||
uri := req.URL.EscapedPath()
|
||||
line = append(line, uri...)
|
||||
line.WriteString(uri)
|
||||
isFirst := true
|
||||
for k, v := range query {
|
||||
if isFirst {
|
||||
line = append(line, '?')
|
||||
line.WriteByte('?')
|
||||
isFirst = false
|
||||
} else {
|
||||
line = append(line, '&')
|
||||
line.WriteByte('&')
|
||||
}
|
||||
line = append(line, k...)
|
||||
line = append(line, '=')
|
||||
for _, v := range v {
|
||||
line = append(line, v...)
|
||||
for i, val := range v {
|
||||
if i > 0 {
|
||||
line.WriteByte('&')
|
||||
}
|
||||
line.WriteString(k)
|
||||
line.WriteByte('=')
|
||||
line.WriteString(val)
|
||||
}
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func clientIP(req *http.Request) string {
|
||||
@@ -58,50 +62,51 @@ func clientIP(req *http.Request) string {
|
||||
return req.RemoteAddr
|
||||
}
|
||||
|
||||
func (f *CommonFormatter) AppendRequestLog(line []byte, req *http.Request, res *http.Response) []byte {
|
||||
func (f CommonFormatter) AppendRequestLog(line *bytes.Buffer, req *http.Request, res *http.Response) {
|
||||
query := f.cfg.Query.IterQuery(req.URL.Query())
|
||||
|
||||
line = append(line, req.Host...)
|
||||
line = append(line, ' ')
|
||||
line.WriteString(req.Host)
|
||||
line.WriteByte(' ')
|
||||
|
||||
line = append(line, clientIP(req)...)
|
||||
line = append(line, " - - ["...)
|
||||
line.WriteString(clientIP(req))
|
||||
line.WriteString(" - - [")
|
||||
|
||||
line = mockable.TimeNow().AppendFormat(line, LogTimeFormat)
|
||||
line = append(line, `] "`...)
|
||||
line.WriteString(mockable.TimeNow().Format(LogTimeFormat))
|
||||
line.WriteString("] \"")
|
||||
|
||||
line = append(line, req.Method...)
|
||||
line = append(line, ' ')
|
||||
line = appendRequestURI(line, req, query)
|
||||
line = append(line, ' ')
|
||||
line = append(line, req.Proto...)
|
||||
line = append(line, '"')
|
||||
line = append(line, ' ')
|
||||
line.WriteString(req.Method)
|
||||
line.WriteByte(' ')
|
||||
appendRequestURI(line, req, query)
|
||||
line.WriteByte(' ')
|
||||
line.WriteString(req.Proto)
|
||||
line.WriteByte('"')
|
||||
line.WriteByte(' ')
|
||||
|
||||
line = strconv.AppendInt(line, int64(res.StatusCode), 10)
|
||||
line = append(line, ' ')
|
||||
line = strconv.AppendInt(line, res.ContentLength, 10)
|
||||
return line
|
||||
line.WriteString(strconv.FormatInt(int64(res.StatusCode), 10))
|
||||
line.WriteByte(' ')
|
||||
line.WriteString(strconv.FormatInt(res.ContentLength, 10))
|
||||
}
|
||||
|
||||
func (f *CombinedFormatter) AppendRequestLog(line []byte, req *http.Request, res *http.Response) []byte {
|
||||
line = f.CommonFormatter.AppendRequestLog(line, req, res)
|
||||
line = append(line, " \""...)
|
||||
line = append(line, req.Referer()...)
|
||||
line = append(line, "\" \""...)
|
||||
line = append(line, req.UserAgent()...)
|
||||
line = append(line, '"')
|
||||
return line
|
||||
func (f CombinedFormatter) AppendRequestLog(line *bytes.Buffer, req *http.Request, res *http.Response) {
|
||||
f.CommonFormatter.AppendRequestLog(line, req, res)
|
||||
line.WriteString(" \"")
|
||||
line.WriteString(req.Referer())
|
||||
line.WriteString("\" \"")
|
||||
line.WriteString(req.UserAgent())
|
||||
line.WriteByte('"')
|
||||
}
|
||||
|
||||
func (f *JSONFormatter) AppendRequestLog(line []byte, req *http.Request, res *http.Response) []byte {
|
||||
func (f JSONFormatter) AppendRequestLog(line *bytes.Buffer, req *http.Request, res *http.Response) {
|
||||
logger := zerolog.New(line)
|
||||
f.LogRequestZeroLog(&logger, req, res)
|
||||
}
|
||||
|
||||
func (f JSONFormatter) LogRequestZeroLog(logger *zerolog.Logger, req *http.Request, res *http.Response) {
|
||||
query := f.cfg.Query.ZerologQuery(req.URL.Query())
|
||||
headers := f.cfg.Headers.ZerologHeaders(req.Header)
|
||||
cookies := f.cfg.Cookies.ZerologCookies(req.Cookies())
|
||||
contentType := res.Header.Get("Content-Type")
|
||||
|
||||
writer := bytes.NewBuffer(line)
|
||||
logger := zerolog.New(writer)
|
||||
event := logger.Info().
|
||||
Str("time", mockable.TimeNow().Format(LogTimeFormat)).
|
||||
Str("ip", clientIP(req)).
|
||||
@@ -119,22 +124,33 @@ func (f *JSONFormatter) AppendRequestLog(line []byte, req *http.Request, res *ht
|
||||
Object("headers", headers).
|
||||
Object("cookies", cookies)
|
||||
|
||||
if res.StatusCode >= 400 {
|
||||
if res.Status != "" {
|
||||
event.Str("error", res.Status)
|
||||
} else {
|
||||
event.Str("error", http.StatusText(res.StatusCode))
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: zerolog will append a newline to the buffer
|
||||
event.Send()
|
||||
return writer.Bytes()
|
||||
}
|
||||
|
||||
func (f ACLLogFormatter) AppendACLLog(line []byte, info *maxmind.IPInfo, blocked bool) []byte {
|
||||
writer := bytes.NewBuffer(line)
|
||||
logger := zerolog.New(writer)
|
||||
func (f ConsoleFormatter) LogRequestZeroLog(logger *zerolog.Logger, req *http.Request, res *http.Response) {
|
||||
contentType := res.Header.Get("Content-Type")
|
||||
|
||||
var reqURI bytes.Buffer
|
||||
appendRequestURI(&reqURI, req, f.cfg.Query.IterQuery(req.URL.Query()))
|
||||
|
||||
event := logger.Info().
|
||||
Bytes("uri", reqURI.Bytes()).
|
||||
Str("protocol", req.Proto).
|
||||
Str("type", contentType).
|
||||
Int64("size", res.ContentLength).
|
||||
Str("useragent", req.UserAgent())
|
||||
|
||||
// NOTE: zerolog will append a newline to the buffer
|
||||
event.Msgf("[%d] %s %s://%s from %s", res.StatusCode, req.Method, scheme(req), req.Host, clientIP(req))
|
||||
}
|
||||
|
||||
func (f ACLLogFormatter) AppendACLLog(line *bytes.Buffer, info *maxmind.IPInfo, blocked bool) {
|
||||
logger := zerolog.New(line)
|
||||
f.LogACLZeroLog(&logger, info, blocked)
|
||||
}
|
||||
|
||||
func (f ACLLogFormatter) LogACLZeroLog(logger *zerolog.Logger, info *maxmind.IPInfo, blocked bool) {
|
||||
event := logger.Info().
|
||||
Str("time", mockable.TimeNow().Format(LogTimeFormat)).
|
||||
Str("ip", info.Str)
|
||||
@@ -144,10 +160,32 @@ func (f ACLLogFormatter) AppendACLLog(line []byte, info *maxmind.IPInfo, blocked
|
||||
event.Str("action", "allow")
|
||||
}
|
||||
if info.City != nil {
|
||||
event.Str("iso_code", info.City.Country.IsoCode)
|
||||
event.Str("time_zone", info.City.Location.TimeZone)
|
||||
if isoCode := info.City.Country.IsoCode; isoCode != "" {
|
||||
event.Str("iso_code", isoCode)
|
||||
}
|
||||
if timeZone := info.City.Location.TimeZone; timeZone != "" {
|
||||
event.Str("time_zone", timeZone)
|
||||
}
|
||||
}
|
||||
// NOTE: zerolog will append a newline to the buffer
|
||||
event.Send()
|
||||
return writer.Bytes()
|
||||
}
|
||||
|
||||
func (f ConsoleACLFormatter) LogACLZeroLog(logger *zerolog.Logger, info *maxmind.IPInfo, blocked bool) {
|
||||
event := logger.Info()
|
||||
if info.City != nil {
|
||||
if isoCode := info.City.Country.IsoCode; isoCode != "" {
|
||||
event.Str("iso_code", isoCode)
|
||||
}
|
||||
if timeZone := info.City.Location.TimeZone; timeZone != "" {
|
||||
event.Str("time_zone", timeZone)
|
||||
}
|
||||
}
|
||||
action := "accepted"
|
||||
if blocked {
|
||||
action = "denied"
|
||||
}
|
||||
|
||||
// NOTE: zerolog will append a newline to the buffer
|
||||
event.Msgf("request %s from %s", action, info.Str)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ type MockFile struct {
|
||||
buffered bool
|
||||
}
|
||||
|
||||
var _ SupportRotate = (*MockFile)(nil)
|
||||
var _ File = (*MockFile)(nil)
|
||||
|
||||
func NewMockFile(buffered bool) *MockFile {
|
||||
f, _ := afero.TempFile(afero.NewMemMapFs(), "", "")
|
||||
@@ -52,14 +52,9 @@ func (m *MockFile) NumLines() int {
|
||||
return count
|
||||
}
|
||||
|
||||
func (m *MockFile) Size() (int64, error) {
|
||||
stat, _ := m.Stat()
|
||||
return stat.Size(), nil
|
||||
}
|
||||
|
||||
func (m *MockFile) MustSize() int64 {
|
||||
size, _ := m.Size()
|
||||
return size
|
||||
stat, _ := m.Stat()
|
||||
return stat.Size()
|
||||
}
|
||||
|
||||
func (m *MockFile) Close() error {
|
||||
|
||||
@@ -15,14 +15,21 @@ type MultiAccessLogger struct {
|
||||
//
|
||||
// If there is only one writer, it will return a single AccessLogger.
|
||||
// Otherwise, it will return a MultiAccessLogger that writes to all the writers.
|
||||
func NewMultiAccessLogger(parent task.Parent, cfg AnyConfig, writers []Writer) AccessLogger {
|
||||
func NewMultiAccessLogger(parent task.Parent, cfg AnyConfig, writers []File) AccessLogger {
|
||||
if len(writers) == 1 {
|
||||
return NewAccessLoggerWithIO(parent, writers[0], cfg)
|
||||
if writers[0] == stdout {
|
||||
return NewConsoleLogger(cfg.ToConfig())
|
||||
}
|
||||
return NewFileAccessLogger(parent, writers[0], cfg)
|
||||
}
|
||||
|
||||
accessLoggers := make([]AccessLogger, len(writers))
|
||||
for i, writer := range writers {
|
||||
accessLoggers[i] = NewAccessLoggerWithIO(parent, writer, cfg)
|
||||
if writer == stdout {
|
||||
accessLoggers[i] = NewConsoleLogger(cfg.ToConfig())
|
||||
} else {
|
||||
accessLoggers[i] = NewFileAccessLogger(parent, writer, cfg)
|
||||
}
|
||||
}
|
||||
return &MultiAccessLogger{accessLoggers}
|
||||
}
|
||||
@@ -31,9 +38,9 @@ func (m *MultiAccessLogger) Config() *Config {
|
||||
return m.accessLoggers[0].Config()
|
||||
}
|
||||
|
||||
func (m *MultiAccessLogger) Log(req *http.Request, res *http.Response) {
|
||||
func (m *MultiAccessLogger) LogRequest(req *http.Request, res *http.Response) {
|
||||
for _, accessLogger := range m.accessLoggers {
|
||||
accessLogger.Log(req, res)
|
||||
accessLogger.LogRequest(req, res)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestNewMultiAccessLogger(t *testing.T) {
|
||||
testTask := task.RootTask("test", false)
|
||||
cfg := DefaultRequestLoggerConfig()
|
||||
|
||||
writers := []Writer{
|
||||
writers := []File{
|
||||
NewMockFile(true),
|
||||
NewMockFile(true),
|
||||
}
|
||||
@@ -30,7 +30,7 @@ func TestMultiAccessLoggerConfig(t *testing.T) {
|
||||
cfg := DefaultRequestLoggerConfig()
|
||||
cfg.Format = FormatCommon
|
||||
|
||||
writers := []Writer{
|
||||
writers := []File{
|
||||
NewMockFile(true),
|
||||
NewMockFile(true),
|
||||
}
|
||||
@@ -48,7 +48,7 @@ func TestMultiAccessLoggerLog(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -68,7 +68,7 @@ func TestMultiAccessLoggerLog(t *testing.T) {
|
||||
ContentLength: 100,
|
||||
}
|
||||
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
logger.Flush()
|
||||
|
||||
expect.Equal(t, writer1.NumLines(), 1)
|
||||
@@ -81,7 +81,7 @@ func TestMultiAccessLoggerLogError(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -107,7 +107,7 @@ func TestMultiAccessLoggerLogACL(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -129,7 +129,7 @@ func TestMultiAccessLoggerFlush(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -143,7 +143,7 @@ func TestMultiAccessLoggerFlush(t *testing.T) {
|
||||
StatusCode: http.StatusOK,
|
||||
}
|
||||
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
logger.Flush()
|
||||
|
||||
expect.Equal(t, writer1.NumLines(), 1)
|
||||
@@ -156,7 +156,7 @@ func TestMultiAccessLoggerClose(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -170,7 +170,7 @@ func TestMultiAccessLoggerMultipleLogs(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -185,7 +185,7 @@ func TestMultiAccessLoggerMultipleLogs(t *testing.T) {
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
}
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
|
||||
logger.Flush()
|
||||
@@ -199,7 +199,7 @@ func TestMultiAccessLoggerSingleWriter(t *testing.T) {
|
||||
cfg := DefaultRequestLoggerConfig()
|
||||
|
||||
writer := NewMockFile(true)
|
||||
writers := []Writer{writer}
|
||||
writers := []File{writer}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
expect.NotNil(t, logger)
|
||||
@@ -214,7 +214,7 @@ func TestMultiAccessLoggerSingleWriter(t *testing.T) {
|
||||
StatusCode: http.StatusOK,
|
||||
}
|
||||
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
logger.Flush()
|
||||
|
||||
expect.Equal(t, writer.NumLines(), 1)
|
||||
@@ -226,7 +226,7 @@ func TestMultiAccessLoggerMixedOperations(t *testing.T) {
|
||||
|
||||
writer1 := NewMockFile(true)
|
||||
writer2 := NewMockFile(true)
|
||||
writers := []Writer{writer1, writer2}
|
||||
writers := []File{writer1, writer2}
|
||||
|
||||
logger := NewMultiAccessLogger(testTask, cfg, writers)
|
||||
|
||||
@@ -241,7 +241,7 @@ func TestMultiAccessLoggerMixedOperations(t *testing.T) {
|
||||
StatusCode: http.StatusOK,
|
||||
}
|
||||
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
logger.Flush()
|
||||
|
||||
info := &maxmind.IPInfo{
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
@@ -17,7 +18,7 @@ type supportRotate interface {
|
||||
io.ReaderAt
|
||||
io.WriterAt
|
||||
Truncate(size int64) error
|
||||
Size() (int64, error)
|
||||
Stat() (fs.FileInfo, error)
|
||||
}
|
||||
|
||||
type RotateResult struct {
|
||||
@@ -93,10 +94,11 @@ func rotateLogFileByPolicy(file supportRotate, config *Retention, result *Rotate
|
||||
return false, nil // should not happen
|
||||
}
|
||||
|
||||
fileSize, err := file.Size()
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
fileSize := stat.Size()
|
||||
|
||||
// nothing to rotate, return the nothing
|
||||
if fileSize == 0 {
|
||||
@@ -104,6 +106,7 @@ func rotateLogFileByPolicy(file supportRotate, config *Retention, result *Rotate
|
||||
}
|
||||
|
||||
s := NewBackScanner(file, fileSize, defaultChunkSize)
|
||||
defer s.Release()
|
||||
result.OriginalSize = fileSize
|
||||
|
||||
// Store the line positions and sizes we want to keep
|
||||
@@ -216,16 +219,17 @@ func fileContentMove(file supportRotate, srcPos, dstPos int64, size int) error {
|
||||
//
|
||||
// Invalid lines will not be detected and included in the result.
|
||||
func rotateLogFileBySize(file supportRotate, config *Retention, result *RotateResult) (rotated bool, err error) {
|
||||
filesize, err := file.Size()
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
fileSize := stat.Size()
|
||||
|
||||
result.OriginalSize = filesize
|
||||
result.OriginalSize = fileSize
|
||||
|
||||
keepSize := int64(config.KeepSize)
|
||||
if keepSize >= filesize {
|
||||
result.NumBytesKeep = filesize
|
||||
if keepSize >= fileSize {
|
||||
result.NumBytesKeep = fileSize
|
||||
return false, nil
|
||||
}
|
||||
result.NumBytesKeep = keepSize
|
||||
|
||||
@@ -57,13 +57,13 @@ func TestRotateKeepLast(t *testing.T) {
|
||||
t.Run(string(format)+" keep last", func(t *testing.T) {
|
||||
file := NewMockFile(true)
|
||||
mockable.MockTimeNow(testTime)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
Format: format,
|
||||
})
|
||||
expect.Nil(t, logger.Config().Retention)
|
||||
|
||||
for range 10 {
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
logger.Flush()
|
||||
|
||||
@@ -87,14 +87,14 @@ func TestRotateKeepLast(t *testing.T) {
|
||||
|
||||
t.Run(string(format)+" keep days", func(t *testing.T) {
|
||||
file := NewMockFile(true)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
Format: format,
|
||||
})
|
||||
expect.Nil(t, logger.Config().Retention)
|
||||
nLines := 10
|
||||
for i := range nLines {
|
||||
mockable.MockTimeNow(testTime.AddDate(0, 0, -nLines+i+1))
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
logger.Flush()
|
||||
expect.Equal(t, file.NumLines(), nLines)
|
||||
@@ -133,14 +133,14 @@ func TestRotateKeepFileSize(t *testing.T) {
|
||||
for _, format := range ReqLoggerFormats {
|
||||
t.Run(string(format)+" keep size no rotation", func(t *testing.T) {
|
||||
file := NewMockFile(true)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
Format: format,
|
||||
})
|
||||
expect.Nil(t, logger.Config().Retention)
|
||||
nLines := 10
|
||||
for i := range nLines {
|
||||
mockable.MockTimeNow(testTime.AddDate(0, 0, -nLines+i+1))
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
logger.Flush()
|
||||
expect.Equal(t, file.NumLines(), nLines)
|
||||
@@ -165,14 +165,14 @@ func TestRotateKeepFileSize(t *testing.T) {
|
||||
|
||||
t.Run("keep size with rotation", func(t *testing.T) {
|
||||
file := NewMockFile(true)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
Format: FormatJSON,
|
||||
})
|
||||
expect.Nil(t, logger.Config().Retention)
|
||||
nLines := 100
|
||||
for i := range nLines {
|
||||
mockable.MockTimeNow(testTime.AddDate(0, 0, -nLines+i+1))
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
logger.Flush()
|
||||
expect.Equal(t, file.NumLines(), nLines)
|
||||
@@ -199,14 +199,14 @@ func TestRotateSkipInvalidTime(t *testing.T) {
|
||||
for _, format := range ReqLoggerFormats {
|
||||
t.Run(string(format), func(t *testing.T) {
|
||||
file := NewMockFile(true)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
Format: format,
|
||||
})
|
||||
expect.Nil(t, logger.Config().Retention)
|
||||
nLines := 10
|
||||
for i := range nLines {
|
||||
mockable.MockTimeNow(testTime.AddDate(0, 0, -nLines+i+1))
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
logger.Flush()
|
||||
|
||||
n, err := file.Write([]byte("invalid time\n"))
|
||||
@@ -241,7 +241,7 @@ func BenchmarkRotate(b *testing.B) {
|
||||
for _, retention := range tests {
|
||||
b.Run(fmt.Sprintf("retention_%s", retention.String()), func(b *testing.B) {
|
||||
file := NewMockFile(true)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
ConfigBase: ConfigBase{
|
||||
Retention: retention,
|
||||
},
|
||||
@@ -249,7 +249,7 @@ func BenchmarkRotate(b *testing.B) {
|
||||
})
|
||||
for i := range 100 {
|
||||
mockable.MockTimeNow(testTime.AddDate(0, 0, -100+i+1))
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
}
|
||||
logger.Flush()
|
||||
content := file.Content()
|
||||
@@ -275,7 +275,7 @@ func BenchmarkRotateWithInvalidTime(b *testing.B) {
|
||||
for _, retention := range tests {
|
||||
b.Run(fmt.Sprintf("retention_%s", retention.String()), func(b *testing.B) {
|
||||
file := NewMockFile(true)
|
||||
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
logger := NewFileAccessLogger(task.RootTask("test", false), file, &RequestLoggerConfig{
|
||||
ConfigBase: ConfigBase{
|
||||
Retention: retention,
|
||||
},
|
||||
@@ -283,7 +283,7 @@ func BenchmarkRotateWithInvalidTime(b *testing.B) {
|
||||
})
|
||||
for i := range 10000 {
|
||||
mockable.MockTimeNow(testTime.AddDate(0, 0, -10000+i+1))
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
if i%10 == 0 {
|
||||
_, _ = file.Write([]byte("invalid time\n"))
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"github.com/yusing/goutils/synk"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
f *os.File
|
||||
type sharedFileHandle struct {
|
||||
*os.File
|
||||
|
||||
// os.File.Name() may not equal to key of `openedFiles`.
|
||||
// Store it for later delete from `openedFiles`.
|
||||
@@ -22,18 +22,18 @@ type File struct {
|
||||
}
|
||||
|
||||
var (
|
||||
openedFiles = make(map[string]*File)
|
||||
openedFiles = make(map[string]*sharedFileHandle)
|
||||
openedFilesMu sync.Mutex
|
||||
)
|
||||
|
||||
// NewFileIO creates a new file writer with cleaned path.
|
||||
// OpenFile creates a new file writer with cleaned path.
|
||||
//
|
||||
// If the file is already opened, it will be returned.
|
||||
func NewFileIO(path string) (Writer, error) {
|
||||
func OpenFile(path string) (File, error) {
|
||||
openedFilesMu.Lock()
|
||||
defer openedFilesMu.Unlock()
|
||||
|
||||
var file *File
|
||||
var file *sharedFileHandle
|
||||
var err error
|
||||
|
||||
// make it absolute path, so that we can use it as key of `openedFiles` and shared lock
|
||||
@@ -53,65 +53,38 @@ func NewFileIO(path string) (Writer, error) {
|
||||
return nil, fmt.Errorf("access log open error: %w", err)
|
||||
}
|
||||
if _, err := f.Seek(0, io.SeekEnd); err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("access log seek error: %w", err)
|
||||
}
|
||||
file = &File{f: f, path: path, refCount: synk.NewRefCounter()}
|
||||
|
||||
file = &sharedFileHandle{File: f, path: path, refCount: synk.NewRefCounter()}
|
||||
openedFiles[path] = file
|
||||
|
||||
log.Debug().Str("path", path).Msg("file opened")
|
||||
|
||||
go file.closeOnZero()
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Name returns the absolute path of the file.
|
||||
func (f *File) Name() string {
|
||||
func (f *sharedFileHandle) Name() string {
|
||||
return f.path
|
||||
}
|
||||
|
||||
func (f *File) ShouldBeBuffered() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (f *File) Write(p []byte) (n int, err error) {
|
||||
return f.f.Write(p)
|
||||
}
|
||||
|
||||
func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
return f.f.ReadAt(p, off)
|
||||
}
|
||||
|
||||
func (f *File) WriteAt(p []byte, off int64) (n int, err error) {
|
||||
return f.f.WriteAt(p, off)
|
||||
}
|
||||
|
||||
func (f *File) Seek(offset int64, whence int) (int64, error) {
|
||||
return f.f.Seek(offset, whence)
|
||||
}
|
||||
|
||||
func (f *File) Size() (int64, error) {
|
||||
stat, err := f.f.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return stat.Size(), nil
|
||||
}
|
||||
|
||||
func (f *File) Truncate(size int64) error {
|
||||
return f.f.Truncate(size)
|
||||
}
|
||||
|
||||
func (f *File) Close() error {
|
||||
func (f *sharedFileHandle) Close() error {
|
||||
f.refCount.Sub()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *File) closeOnZero() {
|
||||
defer log.Debug().
|
||||
Str("path", f.path).
|
||||
Msg("access log closed")
|
||||
func (f *sharedFileHandle) closeOnZero() {
|
||||
defer log.Debug().Str("path", f.path).Msg("file closed")
|
||||
|
||||
<-f.refCount.Zero()
|
||||
|
||||
openedFilesMu.Lock()
|
||||
delete(openedFiles, f.path)
|
||||
openedFilesMu.Unlock()
|
||||
f.f.Close()
|
||||
err := f.File.Close()
|
||||
if err != nil {
|
||||
log.Error().Str("path", f.path).Err(err).Msg("failed to close file")
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yusing/goutils/task"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func TestConcurrentFileLoggersShareSameAccessLogIO(t *testing.T) {
|
||||
@@ -18,7 +19,7 @@ func TestConcurrentFileLoggersShareSameAccessLogIO(t *testing.T) {
|
||||
cfg.Path = "test.log"
|
||||
|
||||
loggerCount := runtime.GOMAXPROCS(0)
|
||||
accessLogIOs := make([]Writer, loggerCount)
|
||||
accessLogIOs := make([]File, loggerCount)
|
||||
|
||||
// make test log file
|
||||
file, err := os.Create(cfg.Path)
|
||||
@@ -28,16 +29,20 @@ func TestConcurrentFileLoggersShareSameAccessLogIO(t *testing.T) {
|
||||
assert.NoError(t, os.Remove(cfg.Path))
|
||||
})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var errs errgroup.Group
|
||||
for i := range loggerCount {
|
||||
wg.Go(func() {
|
||||
file, err := NewFileIO(cfg.Path)
|
||||
assert.NoError(t, err)
|
||||
errs.Go(func() error {
|
||||
file, err := OpenFile(cfg.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accessLogIOs[i] = file
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
err = errs.Wait()
|
||||
assert.NoError(t, err)
|
||||
|
||||
firstIO := accessLogIOs[0]
|
||||
for _, io := range accessLogIOs {
|
||||
@@ -58,7 +63,7 @@ func TestConcurrentAccessLoggerLogAndFlush(t *testing.T) {
|
||||
loggers := make([]AccessLogger, loggerCount)
|
||||
|
||||
for i := range loggerCount {
|
||||
loggers[i] = NewAccessLoggerWithIO(parent, file, cfg)
|
||||
loggers[i] = NewFileAccessLogger(parent, file, cfg)
|
||||
}
|
||||
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
|
||||
@@ -87,7 +92,7 @@ func concurrentLog(logger AccessLogger, req *http.Request, resp *http.Response,
|
||||
var wg sync.WaitGroup
|
||||
for range n {
|
||||
wg.Go(func() {
|
||||
logger.Log(req, resp)
|
||||
logger.LogRequest(req, resp)
|
||||
if rand.IntN(2) == 0 {
|
||||
logger.Flush()
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/yusing/godoxy/internal/logging"
|
||||
)
|
||||
|
||||
type Stdout struct {
|
||||
logger zerolog.Logger
|
||||
}
|
||||
|
||||
func NewStdout() Writer {
|
||||
return &Stdout{logger: logging.NewLoggerWithFixedLevel(zerolog.InfoLevel, os.Stdout)}
|
||||
}
|
||||
|
||||
func (s Stdout) Name() string {
|
||||
return "stdout"
|
||||
}
|
||||
|
||||
func (s Stdout) ShouldBeBuffered() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s Stdout) Write(p []byte) (n int, err error) {
|
||||
return s.logger.Write(p)
|
||||
}
|
||||
|
||||
func (s Stdout) Close() error {
|
||||
return nil
|
||||
}
|
||||
55
internal/logging/accesslog/types.go
Normal file
55
internal/logging/accesslog/types.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
maxmind "github.com/yusing/godoxy/internal/maxmind/types"
|
||||
"github.com/yusing/goutils/task"
|
||||
)
|
||||
|
||||
type (
|
||||
AccessLogger interface {
|
||||
LogRequest(req *http.Request, res *http.Response)
|
||||
LogError(req *http.Request, err error)
|
||||
LogACL(info *maxmind.IPInfo, blocked bool)
|
||||
|
||||
Config() *Config
|
||||
|
||||
Flush()
|
||||
Close() error
|
||||
}
|
||||
|
||||
AccessLogRotater interface {
|
||||
Rotate(result *RotateResult) (rotated bool, err error)
|
||||
}
|
||||
|
||||
RequestFormatter interface {
|
||||
// AppendRequestLog appends a log line to line with or without a trailing newline
|
||||
AppendRequestLog(line *bytes.Buffer, req *http.Request, res *http.Response)
|
||||
}
|
||||
RequestFormatterZeroLog interface {
|
||||
// LogRequestZeroLog logs a request log to the logger
|
||||
LogRequestZeroLog(logger *zerolog.Logger, req *http.Request, res *http.Response)
|
||||
}
|
||||
ACLFormatter interface {
|
||||
// AppendACLLog appends a log line to line with or without a trailing newline
|
||||
AppendACLLog(line *bytes.Buffer, info *maxmind.IPInfo, blocked bool)
|
||||
// LogACLZeroLog logs an ACL log to the logger
|
||||
LogACLZeroLog(logger *zerolog.Logger, info *maxmind.IPInfo, blocked bool)
|
||||
}
|
||||
)
|
||||
|
||||
func NewAccessLogger(parent task.Parent, cfg AnyConfig) (AccessLogger, error) {
|
||||
writers, err := cfg.Writers()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewMultiAccessLogger(parent, cfg, writers), nil
|
||||
}
|
||||
|
||||
func NewMockAccessLogger(parent task.Parent, cfg *RequestLoggerConfig) AccessLogger {
|
||||
return NewFileAccessLogger(parent, NewMockFile(true), cfg)
|
||||
}
|
||||
@@ -143,8 +143,13 @@ func (s *FileServer) RootPath() string {
|
||||
|
||||
// ServeHTTP implements http.Handler.
|
||||
func (s *FileServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
s.handler.ServeHTTP(w, req)
|
||||
if s.accessLogger != nil {
|
||||
s.accessLogger.Log(req, req.Response)
|
||||
rec := accesslog.GetResponseRecorder(w)
|
||||
w = rec
|
||||
defer func() {
|
||||
s.accessLogger.LogRequest(req, rec.Response())
|
||||
accesslog.PutResponseRecorder(rec)
|
||||
}()
|
||||
}
|
||||
s.handler.ServeHTTP(w, req)
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ func openFile(path string) (io.WriteCloser, gperr.Error) {
|
||||
return noopWriteCloser{buf}, nil
|
||||
}
|
||||
|
||||
f, err := accesslog.NewFileIO(path)
|
||||
f, err := accesslog.OpenFile(path)
|
||||
if err != nil {
|
||||
return nil, ErrInvalidArguments.With(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user