mirror of
https://github.com/yusing/godoxy.git
synced 2026-02-19 17:07:42 +01:00
077e0bc03b
Replace per-scan byte slice allocations with a sized buffer pool, significantly reducing memory pressure during log file scanning. - Add Release() method to return buffers to pool (callers must invoke) - Remove Reset() method - create new scanner instead for simpler lifecycle - Refactor chunk prepending to reuse pooled buffers instead of append Benchmark results show allocations dropped from ~26k to 1 per scan for small chunk sizes, with better throughput. BREAKING CHANGE: Reset() removed; callers must call Release() and create a new BackScanner instance instead.
134 lines
2.9 KiB
Go
134 lines
2.9 KiB
Go
package accesslog
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
)
|
|
|
|
type ReaderAtSeeker interface {
|
|
io.ReaderAt
|
|
io.Seeker
|
|
}
|
|
|
|
// BackScanner provides an interface to read a file backward line by line.
|
|
type BackScanner struct {
|
|
file ReaderAtSeeker
|
|
size int64
|
|
chunkBuf []byte
|
|
|
|
offset int64
|
|
chunk []byte
|
|
line []byte
|
|
err error
|
|
}
|
|
|
|
// NewBackScanner creates a new Scanner to read the file backward.
|
|
// chunkSize determines the size of each read chunk from the end of the file.
|
|
func NewBackScanner(file ReaderAtSeeker, fileSize int64, chunkSize int) *BackScanner {
|
|
return newBackScanner(file, fileSize, sizedPool.GetSized(chunkSize))
|
|
}
|
|
|
|
func newBackScanner(file ReaderAtSeeker, fileSize int64, buf []byte) *BackScanner {
|
|
return &BackScanner{
|
|
file: file,
|
|
size: fileSize,
|
|
offset: fileSize,
|
|
chunkBuf: buf,
|
|
}
|
|
}
|
|
|
|
// Release releases the buffer back to the pool.
|
|
func (s *BackScanner) Release() {
|
|
sizedPool.Put(s.chunkBuf)
|
|
s.chunkBuf = nil
|
|
if s.chunk != nil {
|
|
sizedPool.Put(s.chunk)
|
|
s.chunk = nil
|
|
}
|
|
}
|
|
|
|
// Scan advances the scanner to the previous line, which will then be available
|
|
// via the Bytes method. It returns false when there are no more lines.
|
|
func (s *BackScanner) Scan() bool {
|
|
if s.err != nil {
|
|
return false
|
|
}
|
|
|
|
// Read chunks until a newline is found or the file is fully read
|
|
for {
|
|
// Check if there's a line in the buffer
|
|
if idx := bytes.LastIndexByte(s.chunk, '\n'); idx >= 0 {
|
|
s.line = s.chunk[idx+1:]
|
|
s.chunk = s.chunk[:idx]
|
|
if len(s.line) > 0 {
|
|
return true
|
|
}
|
|
continue
|
|
}
|
|
|
|
for {
|
|
if s.offset <= 0 {
|
|
// No more data to read; check remaining buffer
|
|
if len(s.chunk) > 0 {
|
|
s.line = s.chunk
|
|
sizedPool.Put(s.chunk)
|
|
s.chunk = nil
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
newOffset := max(0, s.offset-int64(len(s.chunkBuf)))
|
|
chunkSize := s.offset - newOffset
|
|
chunk := s.chunkBuf[:chunkSize]
|
|
|
|
n, err := s.file.ReadAt(chunk, newOffset)
|
|
if err != nil {
|
|
if !errors.Is(err, io.EOF) {
|
|
s.err = err
|
|
}
|
|
return false
|
|
} else if n == 0 {
|
|
return false
|
|
}
|
|
|
|
// Prepend the chunk to the buffer
|
|
if s.chunk == nil { // first chunk
|
|
s.chunk = sizedPool.GetSized(2 * len(s.chunkBuf))
|
|
copy(s.chunk, chunk[:n])
|
|
s.chunk = s.chunk[:n]
|
|
} else {
|
|
neededSize := n + len(s.chunk)
|
|
newChunk := sizedPool.GetSized(max(neededSize, 2*len(s.chunkBuf)))
|
|
copy(newChunk, chunk[:n])
|
|
copy(newChunk[n:], s.chunk)
|
|
sizedPool.Put(s.chunk)
|
|
s.chunk = newChunk[:neededSize]
|
|
}
|
|
|
|
s.offset = newOffset
|
|
|
|
// Check for newline in the updated buffer
|
|
if idx := bytes.LastIndexByte(s.chunk, '\n'); idx >= 0 {
|
|
s.line = s.chunk[idx+1:]
|
|
s.chunk = s.chunk[:idx]
|
|
if len(s.line) > 0 {
|
|
return true
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Bytes returns the most recent line generated by a call to Scan.
|
|
func (s *BackScanner) Bytes() []byte {
|
|
return s.line
|
|
}
|
|
|
|
// Err returns the first non-EOF error encountered by the scanner.
|
|
func (s *BackScanner) Err() error {
|
|
return s.err
|
|
}
|