v0.5.0-rc5: check release

This commit is contained in:
yusing
2024-09-19 20:40:03 +08:00
parent be7a766cb2
commit 4a2d42bfa9
68 changed files with 1971 additions and 1107 deletions

View File

@@ -3,6 +3,7 @@ package docker
import (
"net/http"
"sync"
"sync/atomic"
"github.com/docker/cli/cli/connhelper"
"github.com/docker/docker/client"
@@ -11,14 +12,37 @@ import (
E "github.com/yusing/go-proxy/error"
)
type Client = *client.Client
type Client struct {
key string
refCount *atomic.Int32
*client.Client
}
func (c Client) DaemonHostname() string {
url, _ := client.ParseHostURL(c.DaemonHost())
return url.Hostname()
}
// if the client is still referenced, this is no-op
func (c Client) Close() error {
if c.refCount.Load() > 0 {
c.refCount.Add(-1)
return nil
}
clientMapMu.Lock()
defer clientMapMu.Unlock()
delete(clientMap, c.key)
return c.Client.Close()
}
// ConnectClient creates a new Docker client connection to the specified host.
//
// Returns existing client if available.
//
// Parameters:
// - host: the host to connect to (either a URL or "FROM_ENV").
// - host: the host to connect to (either a URL or common.DockerHostFromEnv).
//
// Returns:
// - Client: the Docker client connection.
@@ -29,7 +53,8 @@ func ConnectClient(host string) (Client, E.NestedError) {
// check if client exists
if client, ok := clientMap[host]; ok {
return client, E.Nil()
client.refCount.Add(1)
return client, nil
}
// create client
@@ -41,7 +66,7 @@ func ConnectClient(host string) (Client, E.NestedError) {
default:
helper, err := E.Check(connhelper.GetConnectionHelper(host))
if err.HasError() {
logger.Fatalf("unexpected error: %s", err)
return Client{}, E.UnexpectedError(err.Error())
}
if helper != nil {
httpClient := &http.Client{
@@ -66,11 +91,16 @@ func ConnectClient(host string) (Client, E.NestedError) {
client, err := E.Check(client.NewClientWithOpts(opt...))
if err.HasError() {
return nil, err
return Client{}, err
}
clientMap[host] = client
return client, E.Nil()
clientMap[host] = Client{
Client: client,
key: host,
refCount: &atomic.Int32{},
}
clientMap[host].refCount.Add(1)
return clientMap[host], nil
}
func CloseAllClients() {
@@ -83,12 +113,13 @@ func CloseAllClients() {
logger.Debug("closed all clients")
}
var clientMap map[string]Client = make(map[string]Client)
var clientMapMu sync.Mutex
var (
clientMap map[string]Client = make(map[string]Client)
clientMapMu sync.Mutex
clientOptEnvHost = []client.Opt{
client.WithHostFromEnv(),
client.WithAPIVersionNegotiation(),
}
var clientOptEnvHost = []client.Opt{
client.WithHostFromEnv(),
client.WithAPIVersionNegotiation(),
}
var logger = logrus.WithField("module", "docker")
logger = logrus.WithField("module", "docker")
)

View File

@@ -12,35 +12,41 @@ import (
)
type ClientInfo struct {
Host string
Client Client
Containers []types.Container
}
func GetClientInfo(clientHost string) (*ClientInfo, E.NestedError) {
var listOptions = container.ListOptions{
// Filters: filters.NewArgs(
// filters.Arg("health", "healthy"),
// filters.Arg("health", "none"),
// filters.Arg("health", "starting"),
// ),
All: true,
}
func GetClientInfo(clientHost string, getContainer bool) (*ClientInfo, E.NestedError) {
dockerClient, err := ConnectClient(clientHost)
if err.HasError() {
return nil, E.Failure("create docker client").With(err)
return nil, E.FailWith("connect to docker", err)
}
defer dockerClient.Close()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
containers, err := E.Check(dockerClient.ContainerList(ctx, container.ListOptions{}))
if err.HasError() {
return nil, E.Failure("list containers").With(err)
var containers []types.Container
if getContainer {
containers, err = E.Check(dockerClient.ContainerList(ctx, listOptions))
if err.HasError() {
return nil, E.FailWith("list containers", err)
}
}
// extract host from docker client url
// since the services being proxied to
// should have the same IP as the docker client
url, err := E.Check(client.ParseHostURL(dockerClient.DaemonHost()))
if err.HasError() {
return nil, E.Invalid("host url", dockerClient.DaemonHost()).With(err)
}
if url.Scheme == "unix" {
return &ClientInfo{Host: "localhost", Containers: containers}, E.Nil()
}
return &ClientInfo{Host: url.Hostname(), Containers: containers}, E.Nil()
return &ClientInfo{
Client: dockerClient,
Containers: containers,
}, nil
}
func IsErrConnectionFailed(err error) bool {

109
src/docker/container.go Normal file
View File

@@ -0,0 +1,109 @@
package docker
import (
"fmt"
"strconv"
"strings"
"github.com/docker/docker/api/types"
U "github.com/yusing/go-proxy/utils"
)
type ProxyProperties struct {
DockerHost string `yaml:"docker_host" json:"docker_host"`
ContainerName string `yaml:"container_name" json:"container_name"`
ImageName string `yaml:"image_name" json:"image_name"`
Aliases []string `yaml:"aliases" json:"aliases"`
IsExcluded bool `yaml:"is_excluded" json:"is_excluded"`
FirstPort string `yaml:"first_port" json:"first_port"`
IdleTimeout string `yaml:"idle_timeout" json:"idle_timeout"`
WakeTimeout string `yaml:"wake_timeout" json:"wake_timeout"`
StopMethod string `yaml:"stop_method" json:"stop_method"`
StopTimeout string `yaml:"stop_timeout" json:"stop_timeout"` // stop_method = "stop" only
StopSignal string `yaml:"stop_signal" json:"stop_signal"` // stop_method = "stop" | "kill" only
}
type Container struct {
*types.Container
*ProxyProperties
}
func FromDocker(c *types.Container, dockerHost string) (res Container) {
res.Container = c
res.ProxyProperties = &ProxyProperties{
DockerHost: dockerHost,
ContainerName: res.getName(),
ImageName: res.getImageName(),
Aliases: res.getAliases(),
IsExcluded: U.ParseBool(res.getDeleteLabel(LableExclude)),
FirstPort: res.firstPortOrEmpty(),
IdleTimeout: res.getDeleteLabel(LabelIdleTimeout),
WakeTimeout: res.getDeleteLabel(LabelWakeTimeout),
StopMethod: res.getDeleteLabel(LabelStopMethod),
StopTimeout: res.getDeleteLabel(LabelStopTimeout),
StopSignal: res.getDeleteLabel(LabelStopSignal),
}
return
}
func FromJson(json types.ContainerJSON, dockerHost string) Container {
ports := make([]types.Port, 0)
for k, bindings := range json.NetworkSettings.Ports {
for _, v := range bindings {
pubPort, _ := strconv.Atoi(v.HostPort)
privPort, _ := strconv.Atoi(k.Port())
ports = append(ports, types.Port{
IP: v.HostIP,
PublicPort: uint16(pubPort),
PrivatePort: uint16(privPort),
})
}
}
return FromDocker(&types.Container{
ID: json.ID,
Names: []string{json.Name},
Image: json.Image,
Ports: ports,
Labels: json.Config.Labels,
State: json.State.Status,
Status: json.State.Status,
}, dockerHost)
}
func (c Container) getDeleteLabel(label string) string {
if l, ok := c.Labels[label]; ok {
delete(c.Labels, label)
return l
}
return ""
}
func (c Container) getAliases() []string {
if l := c.getDeleteLabel(LableAliases); l != "" {
return U.CommaSeperatedList(l)
} else {
return []string{c.getName()}
}
}
func (c Container) getName() string {
return strings.TrimPrefix(c.Names[0], "/")
}
func (c Container) getImageName() string {
colonSep := strings.Split(c.Image, ":")
slashSep := strings.Split(colonSep[len(colonSep)-1], "/")
return slashSep[len(slashSep)-1]
}
func (c Container) firstPortOrEmpty() string {
if len(c.Ports) == 0 {
return ""
}
for _, p := range c.Ports {
if p.PublicPort != 0 {
return fmt.Sprint(p.PublicPort)
}
}
return ""
}

View File

@@ -0,0 +1,14 @@
package idlewatcher
import "net/http"
type (
roundTripper struct {
patched roundTripFunc
}
roundTripFunc func(*http.Request) (*http.Response, error)
)
func (rt roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
return rt.patched(req)
}

View File

@@ -0,0 +1,329 @@
package idlewatcher
import (
"bytes"
"context"
"io"
"net/http"
"sync"
"sync/atomic"
"time"
"github.com/docker/docker/api/types/container"
"github.com/sirupsen/logrus"
D "github.com/yusing/go-proxy/docker"
E "github.com/yusing/go-proxy/error"
P "github.com/yusing/go-proxy/proxy"
PT "github.com/yusing/go-proxy/proxy/fields"
)
type watcher struct {
*P.ReverseProxyEntry
client D.Client
refCount atomic.Int32
stopByMethod StopCallback
wakeCh chan struct{}
wakeDone chan E.NestedError
ctx context.Context
cancel context.CancelFunc
l logrus.FieldLogger
}
type (
WakeDone <-chan error
WakeFunc func() WakeDone
StopCallback func() (bool, E.NestedError)
)
func Register(entry *P.ReverseProxyEntry) (*watcher, E.NestedError) {
failure := E.Failure("idle_watcher register")
if entry.IdleTimeout == 0 {
return nil, failure.With(E.Invalid("idle_timeout", 0))
}
watcherMapMu.Lock()
defer watcherMapMu.Unlock()
if w, ok := watcherMap[entry.ContainerName]; ok {
w.refCount.Add(1)
return w, nil
}
client, err := D.ConnectClient(entry.DockerHost)
if err.HasError() {
return nil, failure.With(err)
}
w := &watcher{
ReverseProxyEntry: entry,
client: client,
wakeCh: make(chan struct{}, 1),
wakeDone: make(chan E.NestedError, 1),
l: logger.WithField("container", entry.ContainerName),
}
w.refCount.Add(1)
w.stopByMethod = w.getStopCallback()
watcherMap[w.ContainerName] = w
go func() {
newWatcherCh <- w
}()
return w, nil
}
// If the container is not registered, this is no-op
func Unregister(containerName string) {
watcherMapMu.Lock()
defer watcherMapMu.Unlock()
if w, ok := watcherMap[containerName]; ok {
if w.refCount.Load() == 0 {
w.cancel()
close(w.wakeCh)
delete(watcherMap, containerName)
} else {
w.refCount.Add(-1)
}
}
}
func Start() {
logger.Debug("started")
defer logger.Debug("stopped")
mainLoopCtx, mainLoopCancel = context.WithCancel(context.Background())
defer mainLoopWg.Wait()
for {
select {
case <-mainLoopCtx.Done():
return
case w := <-newWatcherCh:
w.l.Debug("registered")
mainLoopWg.Add(1)
go func() {
w.watch()
Unregister(w.ContainerName)
w.l.Debug("unregistered")
mainLoopWg.Done()
}()
}
}
}
func Stop() {
mainLoopCancel()
mainLoopWg.Wait()
}
func (w *watcher) PatchRoundTripper(rtp http.RoundTripper) roundTripper {
return roundTripper{patched: func(r *http.Request) (*http.Response, error) {
return w.roundTrip(rtp.RoundTrip, r)
}}
}
func (w *watcher) roundTrip(origRoundTrip roundTripFunc, req *http.Request) (*http.Response, error) {
timeout := time.After(w.WakeTimeout)
w.wakeCh <- struct{}{}
for {
select {
case err := <-w.wakeDone:
if err != nil {
return nil, err.Error()
}
return origRoundTrip(req)
case <-timeout:
resp := loadingResponse
resp.TLS = req.TLS
return &resp, nil
}
}
}
func (w *watcher) containerStop() error {
return w.client.ContainerStop(w.ctx, w.ContainerName, container.StopOptions{
Signal: string(w.StopSignal),
Timeout: &w.StopTimeout})
}
func (w *watcher) containerPause() error {
return w.client.ContainerPause(w.ctx, w.ContainerName)
}
func (w *watcher) containerKill() error {
return w.client.ContainerKill(w.ctx, w.ContainerName, string(w.StopSignal))
}
func (w *watcher) containerUnpause() error {
return w.client.ContainerUnpause(w.ctx, w.ContainerName)
}
func (w *watcher) containerStart() error {
return w.client.ContainerStart(w.ctx, w.ContainerName, container.StartOptions{})
}
func (w *watcher) containerStatus() (string, E.NestedError) {
json, err := w.client.ContainerInspect(w.ctx, w.ContainerName)
if err != nil {
return "", E.FailWith("inspect container", err)
}
return json.State.Status, nil
}
func (w *watcher) wakeIfStopped() (bool, E.NestedError) {
failure := E.Failure("wake")
status, err := w.containerStatus()
if err.HasError() {
return false, failure.With(err)
}
// "created", "running", "paused", "restarting", "removing", "exited", or "dead"
switch status {
case "exited", "dead":
err = E.From(w.containerStart())
case "paused":
err = E.From(w.containerUnpause())
case "running":
return false, nil
default:
return false, failure.With(E.Unexpected("container state", status))
}
if err.HasError() {
return false, failure.With(err)
}
status, err = w.containerStatus()
if err.HasError() {
return false, failure.With(err)
} else if status != "running" {
return false, failure.With(E.Unexpected("container state", status))
} else {
return true, nil
}
}
func (w *watcher) getStopCallback() StopCallback {
var cb func() error
switch w.StopMethod {
case PT.StopMethodPause:
cb = w.containerPause
case PT.StopMethodStop:
cb = w.containerStop
case PT.StopMethodKill:
cb = w.containerKill
default:
panic("should not reach here")
}
return func() (bool, E.NestedError) {
status, err := w.containerStatus()
if err.HasError() {
return false, E.FailWith("stop", err)
}
if status != "running" {
return false, nil
}
err = E.From(cb())
if err.HasError() {
return false, E.FailWith("stop", err)
}
return true, nil
}
}
func (w *watcher) watch() {
watcherCtx, watcherCancel := context.WithCancel(context.Background())
w.ctx = watcherCtx
w.cancel = watcherCancel
ticker := time.NewTicker(w.IdleTimeout)
defer ticker.Stop()
for {
select {
case <-mainLoopCtx.Done():
watcherCancel()
case <-watcherCtx.Done():
w.l.Debug("stopped")
return
case <-ticker.C:
w.l.Debug("timeout")
stopped, err := w.stopByMethod()
if err.HasError() {
w.l.Error(err.Extraf("stop method: %s", w.StopMethod))
} else if stopped {
w.l.Infof("%s: ok", w.StopMethod)
} else {
ticker.Stop()
}
case <-w.wakeCh:
w.l.Debug("wake received")
go func() {
started, err := w.wakeIfStopped()
if err != nil {
w.l.Error(err)
} else if started {
w.l.Infof("awaken")
ticker.Reset(w.IdleTimeout)
}
w.wakeDone <- err // this is passed to roundtrip
}()
}
}
}
var (
mainLoopCtx context.Context
mainLoopCancel context.CancelFunc
mainLoopWg sync.WaitGroup
watcherMap = make(map[string]*watcher)
watcherMapMu sync.Mutex
newWatcherCh = make(chan *watcher)
logger = logrus.WithField("module", "idle_watcher")
loadingResponse = http.Response{
StatusCode: http.StatusAccepted,
Header: http.Header{
"Content-Type": {"text/html"},
"Cache-Control": {
"no-cache",
"no-store",
"must-revalidate",
},
},
Body: io.NopCloser(bytes.NewReader((loadingPage))),
ContentLength: int64(len(loadingPage)),
}
loadingPage = []byte(`
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Loading...</title>
</head>
<body>
<script>
window.onload = function() {
setTimeout(function() {
location.reload();
}, 1000); // 1000 milliseconds = 1 second
};
</script>
<p>Container is starting... Please wait</p>
</body>
</html>
`[1:])
)

19
src/docker/inspect.go Normal file
View File

@@ -0,0 +1,19 @@
package docker
import (
"context"
"time"
E "github.com/yusing/go-proxy/error"
)
func (c Client) Inspect(containerID string) (Container, E.NestedError) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
json, err := c.ContainerInspect(ctx, containerID)
if err != nil {
return Container{}, E.From(err)
}
return FromJson(json, c.key), nil
}

View File

@@ -36,7 +36,7 @@ func ParseLabel(label string, value string) (*Label, E.NestedError) {
return &Label{
Namespace: label,
Value: value,
}, E.Nil()
}, nil
}
l := &Label{
@@ -54,12 +54,12 @@ func ParseLabel(label string, value string) (*Label, E.NestedError) {
// find if namespace has value parser
pm, ok := labelValueParserMap[l.Namespace]
if !ok {
return l, E.Nil()
return l, nil
}
// find if attribute has value parser
p, ok := pm[l.Attribute]
if !ok {
return l, E.Nil()
return l, nil
}
// try to parse value
v, err := p(value)
@@ -67,7 +67,7 @@ func ParseLabel(label string, value string) (*Label, E.NestedError) {
return nil, err
}
l.Value = v
return l, E.Nil()
return l, nil
}
func RegisterNamespace(namespace string, pm ValueParserMap) {

View File

@@ -10,7 +10,7 @@ import (
func yamlListParser(value string) (any, E.NestedError) {
value = strings.TrimSpace(value)
if value == "" {
return []string{}, E.Nil()
return []string{}, nil
}
var data []string
err := E.From(yaml.Unmarshal([]byte(value), &data))
@@ -34,23 +34,15 @@ func yamlStringMappingParser(value string) (any, E.NestedError) {
h[key] = val
}
}
return h, E.Nil()
}
func commaSepParser(value string) (any, E.NestedError) {
v := strings.Split(value, ",")
for i := range v {
v[i] = strings.TrimSpace(v[i])
}
return v, E.Nil()
return h, nil
}
func boolParser(value string) (any, E.NestedError) {
switch strings.ToLower(value) {
case "true", "yes", "1":
return true, E.Nil()
return true, nil
case "false", "no", "0":
return false, E.Nil()
return false, nil
default:
return nil, E.Invalid("boolean value", value)
}

View File

@@ -7,7 +7,7 @@ import (
"testing"
E "github.com/yusing/go-proxy/error"
. "github.com/yusing/go-proxy/utils"
. "github.com/yusing/go-proxy/utils/testing"
)
func makeLabel(namespace string, alias string, field string) string {
@@ -19,7 +19,7 @@ func TestHomePageLabel(t *testing.T) {
field := "ip"
v := "bar"
pl, err := ParseLabel(makeLabel(NSHomePage, alias, field), v)
ExpectNoError(t, err)
ExpectNoError(t, err.Error())
if pl.Target != alias {
t.Errorf("Expected alias=%s, got %s", alias, pl.Target)
}
@@ -34,8 +34,8 @@ func TestHomePageLabel(t *testing.T) {
func TestStringProxyLabel(t *testing.T) {
v := "bar"
pl, err := ParseLabel(makeLabel(NSProxy, "foo", "ip"), v)
ExpectNoError(t, err)
ExpectEqual(t, pl.Value, v)
ExpectNoError(t, err.Error())
ExpectEqual(t, pl.Value.(string), v)
}
func TestBoolProxyLabelValid(t *testing.T) {
@@ -52,8 +52,8 @@ func TestBoolProxyLabelValid(t *testing.T) {
for k, v := range tests {
pl, err := ParseLabel(makeLabel(NSProxy, "foo", "no_tls_verify"), k)
ExpectNoError(t, err)
ExpectEqual(t, pl.Value, v)
ExpectNoError(t, err.Error())
ExpectEqual(t, pl.Value.(bool), v)
}
}
@@ -78,7 +78,7 @@ X-Custom-Header2: boo`
}
pl, err := ParseLabel(makeLabel(NSProxy, "foo", "set_headers"), v)
ExpectNoError(t, err)
ExpectNoError(t, err.Error())
hGot := ExpectType[map[string]string](t, pl.Value)
if hGot != nil && !reflect.DeepEqual(h, hGot) {
t.Errorf("Expected %v, got %v", h, hGot)
@@ -109,33 +109,32 @@ func TestHideHeadersProxyLabel(t *testing.T) {
`
v = strings.TrimPrefix(v, "\n")
pl, err := ParseLabel(makeLabel(NSProxy, "foo", "hide_headers"), v)
ExpectNoError(t, err)
ExpectNoError(t, err.Error())
sGot := ExpectType[[]string](t, pl.Value)
sWant := []string{"X-Custom-Header1", "X-Custom-Header2", "X-Custom-Header3"}
if sGot != nil {
ExpectEqual(t, sGot, sWant)
ExpectDeepEqual(t, sGot, sWant)
}
}
func TestCommaSepProxyLabelSingle(t *testing.T) {
v := "a"
pl, err := ParseLabel("proxy.aliases", v)
ExpectNoError(t, err)
sGot := ExpectType[[]string](t, pl.Value)
sWant := []string{"a"}
if sGot != nil {
ExpectEqual(t, sGot, sWant)
}
// func TestCommaSepProxyLabelSingle(t *testing.T) {
// v := "a"
// pl, err := ParseLabel("proxy.aliases", v)
// ExpectNoError(t, err)
// sGot := ExpectType[[]string](t, pl.Value)
// sWant := []string{"a"}
// if sGot != nil {
// ExpectEqual(t, sGot, sWant)
// }
// }
}
func TestCommaSepProxyLabelMulti(t *testing.T) {
v := "X-Custom-Header1, X-Custom-Header2,X-Custom-Header3"
pl, err := ParseLabel("proxy.aliases", v)
ExpectNoError(t, err)
sGot := ExpectType[[]string](t, pl.Value)
sWant := []string{"X-Custom-Header1", "X-Custom-Header2", "X-Custom-Header3"}
if sGot != nil {
ExpectEqual(t, sGot, sWant)
}
}
// func TestCommaSepProxyLabelMulti(t *testing.T) {
// v := "X-Custom-Header1, X-Custom-Header2,X-Custom-Header3"
// pl, err := ParseLabel("proxy.aliases", v)
// ExpectNoError(t, err)
// sGot := ExpectType[[]string](t, pl.Value)
// sWant := []string{"X-Custom-Header1", "X-Custom-Header2", "X-Custom-Header3"}
// if sGot != nil {
// ExpectEqual(t, sGot, sWant)
// }
// }

13
src/docker/labels.go Normal file
View File

@@ -0,0 +1,13 @@
package docker
const (
WildcardAlias = "*"
LableAliases = NSProxy + ".aliases"
LableExclude = NSProxy + ".exclude"
LabelIdleTimeout = NSProxy + ".idle_timeout"
LabelWakeTimeout = NSProxy + ".wake_timeout"
LabelStopMethod = NSProxy + ".stop_method"
LabelStopTimeout = NSProxy + ".stop_timeout"
LabelStopSignal = NSProxy + ".stop_signal"
)