helix-proxy/internal/proxy/engine.go

package proxy

import (
	"golang.org/x/crypto/bcrypt"
	"context"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"helix-proxy/internal/config"
	"helix-proxy/internal/store"
	"helix-proxy/internal/www"
)

// Engine manages the live HTTP reverse proxy (and later streams) based on the store.
// It replaces all nginx config generation / reload.
type Engine struct {
	mu      sync.RWMutex
	hosts   map[string]*Host // domain (or wildcard key) -> config + handler
	st      store.Store
	srv     *http.Server // the :80 / :443 listener(s)
	started bool

	// streams
	streamMu        sync.Mutex
	streamHandles   map[int]*streamHandle // stream id -> live listeners
	streamRuntimeMu sync.RWMutex
	streamRuntime   map[int]*streamRuntime // incoming port -> live listener state

	// TLS certs per host (from certificates)
	certMu    sync.RWMutex
	hostCerts map[string]*tls.Certificate
	certByID  map[int]*tls.Certificate // for streams SSL termination etc.

	// redirection hosts
	redirMu    sync.RWMutex
	redirHosts map[string]store.RedirectionHost

	// dead hosts
	deadMu    sync.RWMutex
	deadHosts map[string]store.DeadHost

	// simple response cache for cachingEnabled
	cacheMu sync.Mutex
	cache   map[string]struct {
		body    []byte
		headers http.Header
		status  int
		expires time.Time
	}
}

// Host holds the live config for one (or more) domain(s).
type Host struct {
	ID            int
	Domains       []string
	ForwardURL    string // default backend
	Enabled       bool
	ForwardScheme string
	SslForced     bool
	BlockExploits bool
	Locations     []store.Location

	// Access list (resolved at reload)
	AccessClients    []store.AccessClient
	AccessSatisfyAny bool
	AccessItems      []store.AccessItem
	AccessPassAuth   bool

	// Additional flags from ProxyHost (applied in Handler)
	HstsEnabled           bool
	HstsSubdomains        bool
	AllowWebsocketUpgrade bool
	TrustForwardedProto   bool
	ReqHeaderSets         map[string]string
	RespHeaderAdds        map[string]string
	RespHeaderHides       []string
	CachingEnabled        bool
	Http2Support          bool // noted for future http2 backend config
}

func NewEngine(st store.Store) *Engine {
	return &Engine{
		hosts:         make(map[string]*Host),
		st:            st,
		streamHandles: make(map[int]*streamHandle),
		streamRuntime: make(map[int]*streamRuntime),
		hostCerts:     make(map[string]*tls.Certificate),
		certByID:      make(map[int]*tls.Certificate),
		redirHosts:    make(map[string]store.RedirectionHost),
		deadHosts:     make(map[string]store.DeadHost),
		cache: make(map[string]struct {
			body    []byte
			headers http.Header
			status  int
			expires time.Time
		}),
	}
}

// ReloadFromStore rebuilds the in-memory routing table from the store.
// Called on startup and after any host mutation.
func (e *Engine) ReloadFromStore() {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.hosts = make(map[string]*Host)

	proxyHosts := e.st.GetProxyHosts()
	slog.Info("store root for reload", "proxyHosts", len(proxyHosts), "accessLists", len(e.st.GetAccessLists()))
	for _, ph := range proxyHosts {
		if !ph.Enabled || len(ph.DomainNames) == 0 {
			continue
		}
		scheme := ph.ForwardScheme
		if scheme == "" {
			scheme = "http"
		}
		u := &url.URL{Scheme: scheme, Host: net.JoinHostPort(ph.ForwardHost, "80")}
		if ph.ForwardPort > 0 {
			u.Host = net.JoinHostPort(ph.ForwardHost, itoa(ph.ForwardPort))
		}
		h := &Host{
			ID:            ph.ID,
			Domains:       append([]string(nil), ph.DomainNames...),
			ForwardURL:    u.String(),
			Enabled:       ph.Enabled,
			ForwardScheme: scheme,
			SslForced:     ph.SslForced,
			BlockExploits: ph.BlockExploits,
			Locations:     append([]store.Location(nil), ph.Locations...),

			// access list
			AccessSatisfyAny: false,

			// additional flags
			HstsEnabled:           ph.HstsEnabled,
			HstsSubdomains:        ph.HstsSubdomains,
			AllowWebsocketUpgrade: ph.AllowWebsocketUpgrade,
			TrustForwardedProto:   ph.TrustForwardedProto,
			CachingEnabled:        ph.CachingEnabled,
			Http2Support:          ph.Http2Support,
		}
		h.ReqHeaderSets, h.RespHeaderAdds, h.RespHeaderHides = resolveProxyHeaders(ph)
		if ph.AccessListID > 0 {
			if al, ok := e.st.GetAccessList(ph.AccessListID); ok {
				h.AccessClients = append([]store.AccessClient(nil), al.Clients...)
				h.AccessSatisfyAny = al.SatisfyAny
				h.AccessItems = append([]store.AccessItem(nil), al.Items...)
				h.AccessPassAuth = al.PassAuth
				slog.Info("loaded access list for host", "hostID", ph.ID, "listID", ph.AccessListID, "clients", len(h.AccessClients), "authItems", len(h.AccessItems))
			} else {
				slog.Warn("access list not found for host", "hostID", ph.ID, "listID", ph.AccessListID)
			}
		}
		for _, d := range h.Domains {
			e.hosts[strings.ToLower(d)] = h // later: support *.example.com wildcard matching
		}
	}
	slog.Info("proxy engine reloaded", "hosts", len(e.hosts))
	e.reloadStreams()
	e.loadCerts()
	e.loadRedirs()
	e.loadDeads()
	e.cacheMu.Lock()
	e.cache = make(map[string]struct {
		body    []byte
		headers http.Header
		status  int
		expires time.Time
	})
	e.cacheMu.Unlock()
}

func itoa(i int) string { return strconv.Itoa(i) }

// Handler returns the http.Handler for the main proxy listener(s).
// It does SNI/Host based lookup and reverse proxy.
func (e *Engine) Handler() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Serve ACME challenges for LE (before host lookup)
		// lego webroot provider writes tokens to <CHALLENGE_DIR>/.well-known/acme-challenge/<token>
		// so our serve must look there (standard layout).
		if strings.HasPrefix(r.URL.Path, "/.well-known/acme-challenge/") {
			challengeDir := config.Resolve("letsencrypt-acme-challenge")
			acmeDir := filepath.Join(challengeDir, ".well-known", "acme-challenge")
			file := filepath.Join(acmeDir, filepath.Base(r.URL.Path))
			if data, err := os.ReadFile(file); err == nil {
				w.Header().Set("Content-Type", "text/plain")
				w.Write(data)
				return
			}
			http.NotFound(w, r)
			return
		}

		host := r.Host
		if h, _, err := net.SplitHostPort(host); err == nil {
			host = h
		}
		host = strings.ToLower(host)

		// Check redirection hosts first (independent)
		e.redirMu.RLock()
		rh, isRedir := e.redirHosts[host]
		e.redirMu.RUnlock()
		slog.Debug("redir check", "host", host, "isRedir", isRedir, "numRedirs", len(e.redirHosts))
		if isRedir {
			code := rh.ForwardHTTPCode
			if code == 0 {
				code = 302
			}
			target := rh.ForwardDomainName
			scheme := rh.ForwardScheme
			if scheme == "" || scheme == "auto" {
				scheme = "http"
			}
			target = scheme + "://" + target
			if rh.PreservePath {
				target += r.URL.RequestURI()
			}
			http.Redirect(w, r, target, code)
			return
		}

		// Check dead hosts (blocks access, serves 404 or custom)
		e.deadMu.RLock()
		dh, isDead := e.deadHosts[host]
		e.deadMu.RUnlock()
		if isDead {
			// per-dead custom html via meta.html (for full parity/custom content)
			if dh.Meta != nil {
				if html, ok := dh.Meta["html"].(string); ok && html != "" {
					w.Header().Set("Content-Type", "text/html; charset=utf-8")
					w.WriteHeader(http.StatusNotFound)
					w.Write([]byte(html))
					return
				}
			}
			// Serve 404 page (data/www/404.html overrides embedded default).
			if data := www.LoadPage("404.html", map[string]string{
				"MESSAGE": fmt.Sprintf("This host (%s) is configured as dead/blocked.", host),
			}); data != nil {
				w.Header().Set("Content-Type", "text/html; charset=utf-8")
				w.WriteHeader(http.StatusNotFound)
				w.Write(data)
				return
			}
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("404 Not Found (this host is configured as dead/blocked)"))
			return
		}

		e.mu.RLock()
		hcfg, ok := e.hosts[host]
		e.mu.RUnlock()

		if !ok || !hcfg.Enabled {
			// default site behavior driven by settings
			ds := "congratulations"
			if settings := e.st.GetSettings(); settings != nil {
				if s, ok := settings["default_site"].(string); ok && s != "" {
					ds = s
				}
			}
			switch ds {
			case "404":
				if data := www.LoadPage("404.html", map[string]string{
					"MESSAGE": fmt.Sprintf("No virtual host configured for %s.", host),
				}); data != nil {
					w.Header().Set("Content-Type", "text/html; charset=utf-8")
					w.WriteHeader(http.StatusNotFound)
					w.Write(data)
					return
				}
				w.WriteHeader(http.StatusNotFound)
				w.Write([]byte("404 Not Found"))
				return
			case "444":
				w.WriteHeader(444) // close like nginx
				return
			case "redirect":
				tgt := "https://example.com"
				if settings := e.st.GetSettings(); settings != nil {
					if r, ok := settings["default_site_redirect"].(string); ok && r != "" {
						tgt = r
					}
				}
				http.Redirect(w, r, tgt, http.StatusFound)
				return
			default: // congratulations
				if data := www.LoadPage("index.html", map[string]string{"HOST": host}); data != nil {
					w.Header().Set("Content-Type", "text/html; charset=utf-8")
					w.Write(data)
					return
				}
				w.Header().Set("Content-Type", "text/html; charset=utf-8")
				fmt.Fprintf(w, `<h1>Welcome to Helix Proxy</h1><p>No virtual host for %s. Use the admin UI on :81 to configure hosts.</p>`, host)
				return
			}
		}

		slog.Debug("host matched", "host", host, "hasAccessClients", len(hcfg.AccessClients))

		// Access list IP check (before other processing)
		// Reuses getRealClientIP (defined below) with TrustForwardedProto gating to avoid dupe of XFF-trust logic (see also injection path at ~431).
		if len(hcfg.AccessClients) > 0 {
			clientIP := getRealClientIP(r, hcfg.TrustForwardedProto)
			slog.Debug("access check", "host", host, "clients", len(hcfg.AccessClients), "satisfyAny", hcfg.AccessSatisfyAny, "ip", clientIP)
			if !checkAccess(hcfg.AccessClients, hcfg.AccessSatisfyAny, clientIP) {
				w.WriteHeader(http.StatusForbidden)
				w.Write([]byte("access denied by access list"))
				return
			}
		}

		// Basic auth if access list has items
		if len(hcfg.AccessItems) > 0 {
			authHeader := r.Header.Get("Authorization")
			if authHeader == "" || !strings.HasPrefix(authHeader, "Basic ") {
				w.Header().Set("WWW-Authenticate", `Basic realm="Authorization required"`)
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			payload, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Basic "))
			if err != nil {
				w.Header().Set("WWW-Authenticate", `Basic realm="Authorization required"`)
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			creds := strings.SplitN(string(payload), ":", 2)
			if len(creds) != 2 {
				w.Header().Set("WWW-Authenticate", `Basic realm="Authorization required"`)
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			user, pass := creds[0], creds[1]
			authed := false
			for _, item := range hcfg.AccessItems {
				if item.Username == user && accessListPasswordOK(item.Password, pass) {
					authed = true
					break
				}
			}
			if !authed {
				w.Header().Set("WWW-Authenticate", `Basic realm="Authorization required"`)
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			// if not pass_auth, strip the header so backend doesn't see the creds
			if !hcfg.AccessPassAuth {
				r.Header.Del("Authorization")
			}
		}

		// Basic ssl_forced: if configured and request looks http, redirect to https
		if hcfg.SslForced {
			proto := r.Header.Get("X-Forwarded-Proto")
			if proto == "http" || r.TLS == nil {
				httpsURL := "https://" + r.Host + r.URL.RequestURI()
				http.Redirect(w, r, httpsURL, http.StatusMovedPermanently)
				return
			}
		}

		// Basic block_exploits
		if hcfg.BlockExploits {
			badPaths := []string{".env", "wp-admin", ".git/", "config.php", "phpinfo", "shell.php"}
			for _, bad := range badPaths {
				if strings.Contains(r.URL.Path, bad) {
					w.WriteHeader(http.StatusForbidden)
					w.Write([]byte("blocked by exploits rule"))
					return
				}
			}
		}

		// HSTS (set on responses for this vhost if enabled; typically observed on https)
		if hcfg.HstsEnabled {
			hsts := "max-age=31536000"
			if hcfg.HstsSubdomains {
				hsts += "; includeSubDomains"
			}
			w.Header().Set("Strict-Transport-Security", hsts)
		}

		// Support custom locations (path prefix routing to different backends)
		targetURL := hcfg.ForwardURL
		reqPath := r.URL.Path
		for _, loc := range hcfg.Locations {
			if loc.Path != "" && strings.HasPrefix(reqPath, loc.Path) {
				scheme := loc.ForwardScheme
				if scheme == "" {
					scheme = hcfg.ForwardScheme
					if scheme == "" {
						scheme = "http"
					}
				}
				port := loc.ForwardPort
				if port == 0 {
					port = 80
				}
				u := &url.URL{
					Scheme: scheme,
					Host:   net.JoinHostPort(loc.ForwardHost, strconv.Itoa(port)),
				}
				if loc.ForwardPath != "" {
					// simple path rewrite (mutate request path only; target URL stays host-level
					// so ReverseProxy joinURLPath does not duplicate the prefix)
					rest := strings.TrimPrefix(reqPath, loc.Path)
					r.URL.Path = loc.ForwardPath + rest
				}
				targetURL = u.String()
				break
			}
		}

		// Websocket upgrade support (if enabled for this host)
		if hcfg.AllowWebsocketUpgrade && isWebsocketRequest(r) {
			r.Header.Set("Connection", "Upgrade")
			r.Header.Set("Upgrade", "websocket")
			// ReverseProxy will handle hijacking the conn for the upgrade
		}

		// Compute real client info (respect trust-forwarded if configured)
		clientIP := getRealClientIP(r, hcfg.TrustForwardedProto)
		forwardedProto := "http"
		if r.TLS != nil {
			forwardedProto = "https"
		} else if hcfg.TrustForwardedProto {
			if p := r.Header.Get("X-Forwarded-Proto"); p != "" {
				forwardedProto = strings.ToLower(strings.Split(p, ",")[0])
			}
		}

		// Apply configured request/response header overrides.
		for k, v := range hcfg.ReqHeaderSets {
			r.Header.Set(k, v)
		}

		target, _ := url.Parse(targetURL)
		proxy := httputil.NewSingleHostReverseProxy(target)

		// Director to set X-Forwarded* and any request header overrides (after default director)
		origDirector := proxy.Director
		proxy.Director = func(req *http.Request) {
			origDirector(req)
			req.Header.Set("X-Forwarded-Host", r.Host)
			req.Header.Set("X-Forwarded-Proto", forwardedProto)
			req.Header.Set("X-Forwarded-For", clientIP)
			req.Header.Set("X-Real-IP", clientIP)
			for k, v := range hcfg.ReqHeaderSets {
				req.Header.Set(k, v)
			}
		}

		// ModifyResponse to apply response header adds/hides
		proxy.ModifyResponse = func(resp *http.Response) error {
			for k, v := range hcfg.RespHeaderAdds {
				resp.Header.Set(k, v)
			}
			for _, k := range hcfg.RespHeaderHides {
				resp.Header.Del(k)
			}
			return nil
		}

		// Basic caching support if enabled (for GET; simple in-mem with 1h TTL + HIT/MISS + no-store skip)
		if hcfg.CachingEnabled && r.Method == "GET" {
			key := host + r.URL.RequestURI()
			e.cacheMu.Lock()
			if c, ok := e.cache[key]; ok && time.Now().Before(c.expires) {
				for k, vs := range c.headers {
					for _, v := range vs {
						w.Header().Add(k, v)
					}
				}
				w.Header().Set("X-Proxy-Cache", "HIT")
				status := c.status
				if status == 0 {
					status = 200
				}
				w.WriteHeader(status)
				w.Write(c.body)
				e.cacheMu.Unlock()
				return
			}
			e.cacheMu.Unlock()

			// capture response for cache (note: buffers full body in RAM; no size cap yet, see review)
			rec := httptest.NewRecorder()
			proxy.ServeHTTP(rec, r)

			// decide to cache: skip if no-store/private in resp
			doCache := true
			if cc := rec.Header().Get("Cache-Control"); cc != "" {
				lcc := strings.ToLower(cc)
				if strings.Contains(lcc, "no-store") || strings.Contains(lcc, "private") {
					doCache = false
				}
			}
			if doCache {
				e.cacheMu.Lock()
				e.cache[key] = struct {
					body    []byte
					headers http.Header
					status  int
					expires time.Time
				}{
					body:    rec.Body.Bytes(),
					headers: rec.Header(),
					status:  rec.Code,
					expires: time.Now().Add(1 * time.Hour),
				}
				e.cacheMu.Unlock()
			}
			// write to client (MISS only set for actual cacheable paths; omitted/BYPASS for no-store)
			for k, vs := range rec.Header() {
				for _, v := range vs {
					w.Header().Add(k, v)
				}
			}
			if doCache {
				w.Header().Set("X-Proxy-Cache", "MISS")
			}
			w.WriteHeader(rec.Code)
			w.Write(rec.Body.Bytes())
			return
		}

		proxy.ServeHTTP(w, r)
	})
}

// Start begins listening on the proxy ports (80/443 in real; for dev we can use high ports or require root).
// In production docker this will be able to bind because of how the image runs.
// Respects PROXY_HTTP_PORT env (e.g. 80 or 8080).
func (e *Engine) Start(ctx context.Context) error {
	e.ReloadFromStore()

	port := "8080"
	if p := os.Getenv("PROXY_HTTP_PORT"); p != "" {
		port = p
	}
	addr := net.JoinHostPort("", port)
	if os.Getenv("DISABLE_IPV6") == "1" {
		addr = "0.0.0.0:" + port
	}

	e.srv = &http.Server{
		Addr:    addr,
		Handler: e.Handler(),
	}

	ln, err := net.Listen("tcp", e.srv.Addr)
	if err != nil {
		return err
	}
	slog.Info("pure-Go proxy HTTP listening (no nginx). Use PROXY_HTTP_PORT=80 (and privileges) or in docker for real low port", "addr", e.srv.Addr)

	go func() {
		<-ctx.Done()
		_ = e.srv.Shutdown(context.Background())
		_ = ln.Close()
	}()

	e.started = true
	return e.srv.Serve(ln)
}

// StreamStatus is the live listener state for an enabled stream (not persisted).
type StreamStatus struct {
	Listening   bool   `json:"listening"`
	ListenError string `json:"listenError,omitempty"`
}

type streamRuntime struct {
	wantTCP bool
	wantUDP bool
	tcpOK   bool
	udpOK   bool
	tcpErr  string
	udpErr  string
	tcpDone bool
	udpDone bool
}

func (r *streamRuntime) status() StreamStatus {
	if r.wantTCP && !r.tcpDone {
		return StreamStatus{}
	}
	if r.wantUDP && !r.udpDone {
		return StreamStatus{}
	}
	var errs []string
	if r.wantTCP && !r.tcpOK && r.tcpErr != "" {
		errs = append(errs, r.tcpErr)
	}
	if r.wantUDP && !r.udpOK && r.udpErr != "" {
		errs = append(errs, r.udpErr)
	}
	if len(errs) > 0 {
		return StreamStatus{ListenError: strings.Join(errs, "; ")}
	}
	return StreamStatus{Listening: r.wantTCP || r.wantUDP}
}

func (e *Engine) StreamStatus(port int) StreamStatus {
	e.streamRuntimeMu.RLock()
	rt := e.streamRuntime[port]
	e.streamRuntimeMu.RUnlock()
	if rt == nil {
		return StreamStatus{}
	}
	return rt.status()
}

func (e *Engine) initStreamRuntime(s store.Stream) {
	e.streamRuntimeMu.Lock()
	defer e.streamRuntimeMu.Unlock()
	e.streamRuntime[s.IncomingPort] = &streamRuntime{
		wantTCP: s.TCPForwarding,
		wantUDP: s.UDPForwarding,
	}
}

func (e *Engine) markStreamTCPResult(port int, ok bool, err error) {
	e.streamRuntimeMu.Lock()
	defer e.streamRuntimeMu.Unlock()
	rt := e.streamRuntime[port]
	if rt == nil {
		return
	}
	rt.tcpDone = true
	rt.tcpOK = ok
	if err != nil {
		rt.tcpErr = err.Error()
	}
}

func (e *Engine) markStreamUDPResult(port int, ok bool, err error) {
	e.streamRuntimeMu.Lock()
	defer e.streamRuntimeMu.Unlock()
	rt := e.streamRuntime[port]
	if rt == nil {
		return
	}
	rt.udpDone = true
	rt.udpOK = ok
	if err != nil {
		rt.udpErr = err.Error()
	}
}

type streamHandle struct {
	cancel  context.CancelFunc
	running store.Stream
	tcpLn   net.Listener
	udpConn *net.UDPConn
	wg      sync.WaitGroup
}

func streamListenerConfigEqual(a, b store.Stream) bool {
	return a.IncomingPort == b.IncomingPort &&
		a.ForwardingHost == b.ForwardingHost &&
		a.ForwardingPort == b.ForwardingPort &&
		a.TCPForwarding == b.TCPForwarding &&
		a.UDPForwarding == b.UDPForwarding &&
		a.CertificateID == b.CertificateID
}

func duplicateStreamListenPorts(streams map[int]store.Stream) map[int]string {
	byPort := make(map[int]int, len(streams))
	errs := make(map[int]string)
	for id, s := range streams {
		if owner, ok := byPort[s.IncomingPort]; ok {
			msg := fmt.Sprintf("incoming port %d already assigned to stream %d", s.IncomingPort, owner)
			if _, has := errs[id]; !has {
				errs[id] = msg
			}
			if _, has := errs[owner]; !has {
				errs[owner] = fmt.Sprintf("incoming port %d conflict with stream %d", s.IncomingPort, id)
			}
			continue
		}
		byPort[s.IncomingPort] = id
	}
	return errs
}

func (e *Engine) markStreamFailed(s store.Stream, err error) {
	if s.TCPForwarding {
		e.markStreamTCPResult(s.IncomingPort, false, err)
	}
	if s.UDPForwarding {
		e.markStreamUDPResult(s.IncomingPort, false, err)
	}
}

func (e *Engine) streamPortHeldByOther(streamID, port int) (int, bool) {
	e.streamMu.Lock()
	defer e.streamMu.Unlock()
	for id, h := range e.streamHandles {
		if id != streamID && h.running.IncomingPort == port {
			return id, true
		}
	}
	return 0, false
}

func (e *Engine) clearStreamRuntime(port int) {
	e.streamRuntimeMu.Lock()
	delete(e.streamRuntime, port)
	e.streamRuntimeMu.Unlock()
}

func (e *Engine) stopStreamByID(id int) {
	e.streamMu.Lock()
	h := e.streamHandles[id]
	delete(e.streamHandles, id)
	e.streamMu.Unlock()
	if h == nil {
		return
	}
	if h.tcpLn != nil {
		_ = h.tcpLn.Close()
	}
	if h.udpConn != nil {
		_ = h.udpConn.Close()
	}
	h.cancel()
	h.wg.Wait()
	e.clearStreamRuntime(h.running.IncomingPort)
}

func (e *Engine) reloadStreams() {
	all := e.st.GetStreams()
	desired := make(map[int]store.Stream, len(all))
	for _, s := range all {
		if s.Enabled && (s.TCPForwarding || s.UDPForwarding) {
			desired[s.ID] = s
		}
	}

	e.streamMu.Lock()
	running := make(map[int]*streamHandle, len(e.streamHandles))
	for id, h := range e.streamHandles {
		running[id] = h
	}
	e.streamMu.Unlock()

	portConflicts := duplicateStreamListenPorts(desired)
	for id, msg := range portConflicts {
		s := desired[id]
		e.initStreamRuntime(s)
		e.markStreamFailed(s, fmt.Errorf("%s", msg))
		delete(desired, id)
		slog.Error("stream port conflict", "streamID", id, "port", s.IncomingPort, "err", msg)
	}

	stopIDs := make(map[int]struct{})
	for id, h := range running {
		want, ok := desired[id]
		if !ok || !streamListenerConfigEqual(want, h.running) {
			stopIDs[id] = struct{}{}
		}
	}

	for id := range stopIDs {
		e.stopStreamByID(id)
	}

	for id, want := range desired {
		e.streamMu.Lock()
		h := e.streamHandles[id]
		e.streamMu.Unlock()
		if h != nil && streamListenerConfigEqual(want, h.running) {
			continue
		}
		e.startStream(want)
	}

	e.streamRuntimeMu.Lock()
	activePorts := make(map[int]struct{})
	e.streamMu.Lock()
	for _, h := range e.streamHandles {
		activePorts[h.running.IncomingPort] = struct{}{}
	}
	e.streamMu.Unlock()
	for port := range e.streamRuntime {
		if _, ok := activePorts[port]; ok {
			continue
		}
		keep := false
		for _, s := range desired {
			if s.IncomingPort == port {
				keep = true
				break
			}
		}
		if !keep {
			delete(e.streamRuntime, port)
		}
	}
	e.streamRuntimeMu.Unlock()
}

func (e *Engine) startStream(s store.Stream) {
	if !s.TCPForwarding && !s.UDPForwarding {
		return
	}
	ctx, cancel := context.WithCancel(context.Background())
	h := &streamHandle{cancel: cancel, running: s}

	e.streamMu.Lock()
	e.streamHandles[s.ID] = h
	e.streamMu.Unlock()

	e.initStreamRuntime(s)

	if holderID, held := e.streamPortHeldByOther(s.ID, s.IncomingPort); held {
		err := fmt.Errorf("incoming port %d already used by stream %d", s.IncomingPort, holderID)
		slog.Error("stream listen blocked", "streamID", s.ID, "port", s.IncomingPort, "holder", holderID)
		e.markStreamFailed(s, err)
		return
	}

	if s.TCPForwarding {
		listenAddr := ":" + itoa(s.IncomingPort)
		if os.Getenv("DISABLE_IPV6") == "1" {
			listenAddr = "0.0.0.0:" + itoa(s.IncomingPort)
		}
		if s.CertificateID > 0 {
			cert := e.getStreamCert(s.CertificateID)
			if cert == nil {
				err := fmt.Errorf("no certificate for stream TLS (cert id %d)", s.CertificateID)
				slog.Error("tcp stream tls listen failed", "port", s.IncomingPort, "err", err)
				e.markStreamTCPResult(s.IncomingPort, false, err)
			} else {
				tlsCfg := &tls.Config{Certificates: []tls.Certificate{*cert}}
				ln, err := tls.Listen("tcp", listenAddr, tlsCfg)
				if err != nil {
					slog.Error("tcp stream tls listen failed", "port", s.IncomingPort, "err", err)
					e.markStreamTCPResult(s.IncomingPort, false, err)
				} else {
					h.tcpLn = ln
					e.markStreamTCPResult(s.IncomingPort, true, nil)
					h.wg.Add(1)
					go func() {
						defer h.wg.Done()
						e.serveTCPStream(ctx, s, ln)
					}()
				}
			}
		} else {
			ln, err := net.Listen("tcp", listenAddr)
			if err != nil {
				slog.Error("tcp stream listen failed", "port", s.IncomingPort, "err", err)
				e.markStreamTCPResult(s.IncomingPort, false, err)
			} else {
				h.tcpLn = ln
				e.markStreamTCPResult(s.IncomingPort, true, nil)
				h.wg.Add(1)
				go func() {
					defer h.wg.Done()
					e.serveTCPStream(ctx, s, ln)
				}()
			}
		}
	}
	if s.UDPForwarding {
		udpAddr := ":" + itoa(s.IncomingPort)
		if os.Getenv("DISABLE_IPV6") == "1" {
			udpAddr = "0.0.0.0:" + itoa(s.IncomingPort)
		}
		addr, _ := net.ResolveUDPAddr("udp", udpAddr)
		conn, err := net.ListenUDP("udp", addr)
		if err != nil {
			slog.Error("udp stream listen failed", "port", s.IncomingPort, "err", err)
			e.markStreamUDPResult(s.IncomingPort, false, err)
		} else {
			h.udpConn = conn
			e.markStreamUDPResult(s.IncomingPort, true, nil)
			h.wg.Add(1)
			go func() {
				defer h.wg.Done()
				e.serveUDPStream(ctx, s, conn)
			}()
		}
	}
	st := e.StreamStatus(s.IncomingPort)
	if st.Listening {
		slog.Info("started stream listener", "port", s.IncomingPort, "tcp", s.TCPForwarding, "udp", s.UDPForwarding, "ssl", s.CertificateID > 0)
	} else if st.ListenError != "" {
		slog.Warn("stream listener failed to start", "port", s.IncomingPort, "tcp", s.TCPForwarding, "udp", s.UDPForwarding, "ssl", s.CertificateID > 0, "err", st.ListenError)
		if h.tcpLn == nil && h.udpConn == nil {
			e.streamMu.Lock()
			delete(e.streamHandles, s.ID)
			e.streamMu.Unlock()
			h.cancel()
		}
	}
}

func (e *Engine) serveTCPStream(ctx context.Context, s store.Stream, ln net.Listener) {
	for {
		conn, err := ln.Accept()
		if err != nil {
			if ctx.Err() != nil {
				return
			}
			continue
		}
		go func(c net.Conn) {
			defer c.Close()
			target, err := net.Dial("tcp", net.JoinHostPort(s.ForwardingHost, itoa(s.ForwardingPort)))
			if err != nil {
				return
			}
			defer target.Close()
			done := make(chan struct{}, 2)
			go func() { io.Copy(target, c); done <- struct{}{} }()
			go func() { io.Copy(c, target); done <- struct{}{} }()
			<-done
		}(conn)
	}
}

func (e *Engine) getStreamCert(id int) *tls.Certificate {
	e.certMu.RLock()
	defer e.certMu.RUnlock()
	return e.certByID[id]
}

func (e *Engine) serveUDPStream(ctx context.Context, s store.Stream, conn *net.UDPConn) {
	buf := make([]byte, 65535)
	for {
		n, clientAddr, err := conn.ReadFromUDP(buf)
		if err != nil {
			if ctx.Err() != nil {
				return
			}
			continue
		}
		go func(data []byte, caddr *net.UDPAddr) {
			taddr, _ := net.ResolveUDPAddr("udp", net.JoinHostPort(s.ForwardingHost, itoa(s.ForwardingPort)))
			tconn, err := net.DialUDP("udp", nil, taddr)
			if err != nil {
				return
			}
			defer tconn.Close()
			tconn.Write(data)
			tconn.SetReadDeadline(time.Now().Add(5 * time.Second))
			resp := make([]byte, 65535)
			m, _, _ := tconn.ReadFromUDP(resp)
			if m > 0 {
				conn.WriteToUDP(resp[:m], caddr)
			}
		}(append([]byte{}, buf[:n]...), clientAddr)
	}
}

func (e *Engine) loadCerts() {
	e.certMu.Lock()
	defer e.certMu.Unlock()
	e.hostCerts = make(map[string]*tls.Certificate)
	e.certByID = make(map[int]*tls.Certificate)

	certs := e.st.GetCertificates()
	certByID := map[int]store.Certificate{}
	for _, c := range certs {
		certByID[c.ID] = c
	}

	for id, c := range certByID {
		certPEM, _ := c.Meta["certificate"].(string)
		if certPEM == "" {
			certPEM, _ = c.Meta["cert"].(string) // compat our old + UI
		}
		keyPEM, _ := c.Meta["certificate_key"].(string)
		if keyPEM == "" {
			keyPEM, _ = c.Meta["key"].(string)
		}
		if certPEM != "" && keyPEM != "" {
			tcert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM))
			if err == nil {
				e.certByID[id] = &tcert
				// also index by domains for http proxy hosts
				for _, d := range c.DomainNames {
					e.hostCerts[strings.ToLower(d)] = &tcert
				}
			}
		}
	}

	// legacy per-host for proxy (now covered above, but keep for compat)
	// re-attaches using ph.CertificateID even if cert's DomainNames differ; supports hosts referencing cert by id only
	proxyHosts := e.st.GetProxyHosts()
	for _, ph := range proxyHosts {
		if ph.CertificateID > 0 {
			if tcert, ok := e.certByID[ph.CertificateID]; ok {
				for _, d := range ph.DomainNames {
					e.hostCerts[strings.ToLower(d)] = tcert
				}
			}
		}
	}
}

func (e *Engine) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
	e.certMu.RLock()
	defer e.certMu.RUnlock()
	if cert, ok := e.hostCerts[hello.ServerName]; ok {
		return cert, nil
	}
	// fallback to any
	for _, c := range e.hostCerts {
		return c, nil
	}
	return nil, fmt.Errorf("no certificate for %s", hello.ServerName)
}

func (e *Engine) loadRedirs() {
	e.redirMu.Lock()
	defer e.redirMu.Unlock()
	e.redirHosts = make(map[string]store.RedirectionHost)

	redirs := e.st.GetRedirectionHosts()
	keys := []string{}
	for _, rh := range redirs {
		if rh.Enabled {
			for _, d := range rh.DomainNames {
				lower := strings.ToLower(d)
				e.redirHosts[lower] = rh
				keys = append(keys, lower)
			}
		}
	}
	slog.Info("loadRedirs done", "num", len(e.redirHosts), "keys", keys)
}

func (e *Engine) loadDeads() {
	e.deadMu.Lock()
	defer e.deadMu.Unlock()
	e.deadHosts = make(map[string]store.DeadHost)

	deads := e.st.GetDeadHosts()
	keys := []string{}
	for _, dh := range deads {
		if dh.Enabled {
			for _, d := range dh.DomainNames {
				e.deadHosts[strings.ToLower(d)] = dh
				keys = append(keys, d)
			}
		}
	}
	slog.Info("loadDeads done", "num", len(e.deadHosts), "keys", keys)
}

func (e *Engine) TLSConfig() *tls.Config {
	return &tls.Config{
		GetCertificate: e.GetCertificate,
	}
}

// TLS + streams: https listener (with SNI via GetCertificate) and dynamic stream mgmt
// (reload + per-port listeners/cancels) are implemented in cmd/helix-proxy/main.go
// (using eng.TLSConfig()/Handler() and reloadStreams on every ReloadFromStore).

// checkAccess implements basic allow/deny from access list clients.
// satisfyAny means any allow is enough (after denies).
func checkAccess(clients []store.AccessClient, satisfyAny bool, ipStr string) bool {
	if len(clients) == 0 {
		return true
	}
	ip := ipStr
	if h, _, err := net.SplitHostPort(ipStr); err == nil {
		ip = h
	}
	parsed := net.ParseIP(ip)
	if parsed == nil {
		slog.Info("access check bad ip", "ip", ipStr)
		return false
	}

	hasAllow := false
	for _, c := range clients {
		addr := c.Address
		if addr == "all" {
			if c.Directive == "deny" {
				slog.Info("access deny all", "ip", ipStr)
				return false
			}
			hasAllow = true
			continue
		}
		// try CIDR
		if _, cidr, err := net.ParseCIDR(addr); err == nil {
			if cidr.Contains(parsed) {
				if c.Directive == "deny" {
					slog.Info("access deny cidr", "ip", ipStr, "cidr", addr)
					return false
				}
				hasAllow = true
			}
			continue
		}
		// exact IP
		if net.ParseIP(addr).Equal(parsed) {
			if c.Directive == "deny" {
				slog.Info("access deny exact", "ip", ipStr, "addr", addr)
				return false
			}
			hasAllow = true
		}
	}

	if satisfyAny {
		slog.Info("access satisfyAny result", "ip", ipStr, "hasAllow", hasAllow)
		return hasAllow
	}
	// satisfy all: if we didn't hit any deny, and there were allows or no rules, allow
	slog.Info("access satisfyAll result", "ip", ipStr, "hasAllow", hasAllow)
	return true
}

// (no more config ref needed here)

// --- helpers for new engine features (hsts/ws/headers/advanced) ---

func isWebsocketRequest(r *http.Request) bool {
	return strings.EqualFold(r.Header.Get("Upgrade"), "websocket") &&
		strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade")
}

func getRealClientIP(r *http.Request, trustForwarded bool) string {
	// X-Forwarded-For may be comma list; first is original client (when trusted)
	// gated on host's TrustForwardedProto (also controls proto; used for IP trust per security review)
	if trustForwarded {
		if xf := r.Header.Get("X-Forwarded-For"); xf != "" {
			parts := strings.Split(xf, ",")
			return strings.TrimSpace(parts[0])
		}
	}
	ip := r.RemoteAddr
	if h, _, err := net.SplitHostPort(ip); err == nil {
		return h
	}
	return ip
}

// resolveProxyHeaders returns header overrides from structured fields, falling back to legacy advancedConfig.
func resolveProxyHeaders(ph store.ProxyHost) (reqSets map[string]string, respAdds map[string]string, respHides []string) {
	hasStructured := len(ph.RequestHeaders) > 0 || len(ph.ResponseHeaders) > 0 || len(ph.HideHeaders) > 0
	if hasStructured {
		reqSets = map[string]string{}
		for _, h := range ph.RequestHeaders {
			name := strings.TrimSpace(h.Name)
			if name != "" {
				reqSets[name] = h.Value
			}
		}
		respAdds = map[string]string{}
		for _, h := range ph.ResponseHeaders {
			name := strings.TrimSpace(h.Name)
			if name != "" {
				respAdds[name] = h.Value
			}
		}
		for _, name := range ph.HideHeaders {
			name = strings.TrimSpace(name)
			if name != "" {
				respHides = append(respHides, name)
			}
		}
		return reqSets, respAdds, respHides
	}
	if ph.AdvancedConfig != "" {
		return parseAdvancedConfig(ph.AdvancedConfig)
	}
	return map[string]string{}, map[string]string{}, nil
}

// parseAdvancedConfig does a fuller (but still best-effort) parse of the advanced_config string
// (in the spirit of advanced config / snippets) to support common cases:
// - proxy_set_header Foo Bar
// - add_header X-Resp baz;
// - proxy_hide_header X-Foo
// - proxy_redirect (parsed, no-op for now as complex)
// - comments, ; or \n sep, set $var ignored.
// Returns maps for application via Director/ModifyResponse so headers survive proxy.
func parseAdvancedConfig(adv string) (reqSets map[string]string, respAdds map[string]string, respHides []string) {
	reqSets = map[string]string{}
	respAdds = map[string]string{}
	respHides = []string{}
	if adv == "" {
		return
	}
	// normalize separators
	adv = strings.ReplaceAll(adv, ";", "\n")
	lines := strings.Split(adv, "\n")
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		lower := strings.ToLower(line)
		switch {
		case strings.HasPrefix(lower, "proxy_set_header "):
			// proxy_set_header X-Foo bar baz
			parts := strings.Fields(line)
			if len(parts) >= 3 {
				name := parts[1]
				val := strings.Join(parts[2:], " ")
				reqSets[name] = val
			}
		case strings.HasPrefix(lower, "add_header "):
			// add_header X-Response-Header "value" always;
			parts := strings.Fields(line)
			if len(parts) >= 3 {
				name := parts[1]
				// take until ; or end, strip quotes
				val := strings.Join(parts[2:], " ")
				val = strings.Trim(val, `"; `)
				respAdds[name] = val
			}
		case strings.HasPrefix(lower, "proxy_hide_header "):
			parts := strings.Fields(line)
			if len(parts) >= 2 {
				name := parts[1]
				respHides = append(respHides, name)
			}
		case strings.HasPrefix(lower, "proxy_redirect "):
			// best-effort: no-op here (would require Location rewrite in ModifyResponse for full)
		default:
			// ignore set $foo, etc.
		}
	}
	return
}


func accessListPasswordOK(stored, provided string) bool {
	if stored == "" {
		return false
	}
	if strings.HasPrefix(stored, "$2a$") || strings.HasPrefix(stored, "$2b$") || strings.HasPrefix(stored, "$2y$") {
		return bcrypt.CompareHashAndPassword([]byte(stored), []byte(provided)) == nil
	}
	// Legacy plaintext rows until re-saved via admin API.
	return stored == provided
}