Hojun-Cho/got
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all: cut bloated HTTP-client and tuning options

Browse Source 
This is a personal torrent client, not a web client or daemon, so the
rarely-used HTTP-client knobs and redundant tuning flags are gone — flag,
config field, implementation, and tests removed for each. The option
surface drops from 54 to 28.

Web-client cluster removed: --all-proxy (HTTP_PROXY env still works),
load/save-cookies, --referer, -U/--user-agent, --http-user/--http-passwd,
--conditional-get, --remote-time, --check-certificate, --ca-certificate,
--lowest-speed-limit. Pass --header for a one-off auth/cookie/UA header.

Redundant knobs removed: -x and -k (folded into -s), --connect-timeout,
--retry-wait, per-item --max-download-limit/-u, --bt-max-peers,
--bt-stop-timeout, --disable-ipv6, --auto-save-interval, --human-readable,
--stop, -T (a positional .torrent works). follow-torrent and human-readable
are now unconditional defaults.
This commit is contained in:
Hojun-Cho
2026-06-22 08:30:12 +09:00
parent 2afcb861e4
commit cf9082ff72
15 changed files with 157 additions and 1053 deletions
Download Patch File Download Diff File Expand all files Collapse all files

406
httpdl/httpdl.go
View File

@@ -9,8 +9,6 @@ package httpdl
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"hash"
@@ -18,7 +16,6 @@ import (
"mime"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"os"
"path"
@@ -35,6 +32,15 @@ import (
const readBuf = 32 * 1024
// minSplitSize is the smallest piece a file is split into: a segment shorter
// than this is not worth its own connection, so makeSegments stops dividing once
// the pieces would fall below it.
const minSplitSize = 20 << 20
// userAgent is sent on every request; there is no flag to change it. A site that
// needs a different one can be given a User-Agent header via --header.
const userAgent = "got/1.0"
// ErrOutputExists reports that the resolved output file already exists and
// neither --allow-overwrite nor -c was given. main maps it to a distinct exit
// code via errors.Is, so it must stay on the error chain (CONTRACT).
@@ -45,60 +51,21 @@ var ErrOutputExists = errors.New("output file already exists")
// (CONTRACT).
var ErrNotFound = errors.New("not found")
// ErrTooSlow reports a transfer aborted by --lowest-speed-limit. main maps it to
// exit code 5 via errors.Is, so it must stay on the error chain (CONTRACT).
var ErrTooSlow = errors.New("download speed too low")
// Config holds everything one HTTP download needs. It is assembled by main from
// the resolved CLI options.
type Config struct {
Dir string
Out string
Split int // --split: total connections across all mirrors
MaxConnPerServer int // --max-connection-per-server: per-host connection cap
MinSplit int64
Tries int // 0 = unlimited
Timeout time.Duration
FileAlloc string // none | prealloc | trunc | falloc
Continue bool
AllowOverwrite bool
AutoRename bool
Headers []string
UserAgent string
Referer string
Proxy string
CheckCert bool
Limit int64 // per-download bytes/s, 0 = unlimited
OverallLimiter *rate.Limiter // shared across all downloads, may be nil
// RetryWait is the pause between retries of a transient failure
// (--retry-wait). 0 means retry immediately with no sleep.
RetryWait time.Duration
// AutoSaveInterval controls how often the .got control file is rewritten
// (--auto-save-interval). <=0 falls back to 60s.
AutoSaveInterval time.Duration
// ConnectTimeout is the time allowed to establish the TCP/TLS connection
// (--connect-timeout, default 60s), distinct from Timeout which
// covers stalls once data is flowing. <=0 falls back to Timeout.
ConnectTimeout time.Duration
// LoadCookies / SaveCookies are paths to a Netscape cookies.txt read into
// the client's jar before the download and written back on exit.
LoadCookies string
SaveCookies string
// ConditionalGet sends If-Modified-Since (from the existing local file's
// mtime) so a 304 reports success without re-downloading
// (--conditional-get).
ConditionalGet bool
// RemoteTime sets the finished file's mtime from the server's Last-Modified
// header (-R / --remote-time).
RemoteTime bool
// HTTPUser / HTTPPasswd enable HTTP Basic auth (--http-user/passwd).
HTTPUser string
HTTPPasswd string
// LowestSpeedLimit aborts a download whose speed stays at or below this many
// bytes/sec for a sustained window (--lowest-speed-limit). 0 = off.
LowestSpeedLimit int64
Dir string
Out string
Split int // --split: total connections across all mirrors
MinSplit int64 // smallest piece to split into; 0 = minSplitSize default (internal, no flag)
Tries int // --max-tries: 0 = unlimited
Timeout time.Duration
FileAlloc string // none | prealloc | trunc | falloc
Continue bool
AllowOverwrite bool
AutoRename bool
Headers []string // --header: extra request headers
OverallLimiter *rate.Limiter // shared across all downloads, may be nil
// Checksum is a "<type>=<digest>" spec (e.g. "sha-256=ab…"); when set,
// the finished file is hashed and compared, failing with ErrChecksum on a
// mismatch (--checksum). Empty means no verification.
@@ -106,11 +73,6 @@ type Config struct {
// DryRun probes the resource — proving it exists and its size — but downloads
// nothing, then reports success (--dry-run).
DryRun bool
// CACert is a path to a PEM bundle of CA certificates used to verify HTTPS
// servers (--ca-certificate). Empty uses the system roots.
CACert string
// DisableIPv6 forces IPv4-only dialing (--disable-ipv6).
DisableIPv6 bool
}
// Download implements download.Download for one HTTP(S) file, fetched from one
@@ -119,11 +81,10 @@ type Config struct {
// connections.
type Download struct {
uris []string // mirror list; uris[0] is the primary (naming + resume key)
maxConns int // segment-worker cap = min(split, len(uris)*max-connection-per-server)
maxConns int // segment-worker cap = min(split, len(uris))
cfg Config
client *http.Client
limit []*rate.Limiter
jar *cookiejar.Jar // non-nil when cookies are in use, for save-cookies
// name and out are resolved in Run after the probe response is known (the
// final post-redirect URL and any Content-Disposition can change them). The
@@ -135,11 +96,6 @@ type Download struct {
out atomic.Pointer[string]
initErr error
// mu guards seenURLs, which records every URL we sent a request to so
// SaveCookies can ask the jar for the cookies of each contacted host.
mu sync.Mutex
seenURLs []*url.URL
// live counters, read by Stat from any goroutine
total int64 // -1 until known
completed int64
@@ -147,21 +103,6 @@ type Download struct {
conns int32 // active connections
}
// loadCAPool reads a PEM bundle of CA certificates for verifying HTTPS servers
// (--ca-certificate). An unreadable file or one containing no
// certificates is an error rather than a silent fall-back to the system roots.
func loadCAPool(path string) (*x509.CertPool, error) {
pem, err := os.ReadFile(path)
if err != nil {
return nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("ca-certificate %s: no certificates found", path)
}
return pool, nil
}
// primary is the canonical URL — used for naming and the resume sidecar so they
// stay stable regardless of which mirror served the bytes.
func (d *Download) primary() string { return d.uris[0] }
@@ -174,96 +115,27 @@ func (d *Download) mirror(n int) string { return d.uris[n%len(d.uris)] }
// New builds an HTTP download for uris, which must all serve identical content
// (mirrors). uris[0] is the primary.
func New(uris []string, cfg Config) *Download {
// --connect-timeout bounds only connection establishment; once data flows
// --timeout (the idle guard) takes over. The dial and the TLS handshake each
// get the full connectTimeout (Go applies it per phase), so an HTTPS connect
// can take up to ~2x connectTimeout in the worst case — there is no single
// shared connect budget.
connectTimeout := cfg.ConnectTimeout
if connectTimeout <= 0 {
connectTimeout = cfg.Timeout
}
var initErr error
// --ca-certificate: verify HTTPS servers against a custom CA bundle instead of
// the system roots. A bad bundle becomes an initErr so Run fails loudly.
tlsCfg := &tls.Config{InsecureSkipVerify: !cfg.CheckCert}
if cfg.CACert != "" {
if pool, err := loadCAPool(cfg.CACert); err != nil {
initErr = err
} else {
tlsCfg.RootCAs = pool
}
if cfg.MinSplit <= 0 {
cfg.MinSplit = minSplitSize
}
// --disable-ipv6: force IPv4 by dialing "tcp4"; a plain "tcp" would let the
// resolver hand back a v6 address.
dialer := &net.Dialer{Timeout: connectTimeout}
dialContext := dialer.DialContext
if cfg.DisableIPv6 {
dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
if network == "tcp" {
network = "tcp4"
}
return dialer.DialContext(ctx, network, addr)
}
}
// MaxConnsPerHost is the per-host cap (--max-connection-per-server): with
// mirrors the total worker count can exceed it, but no single host does.
perHost := cfg.MaxConnPerServer
if perHost < 1 {
perHost = 1
}
// One connect budget: --timeout bounds both establishing the connection and
// idle stalls once data flows. Proxies still come from the standard
// HTTP_PROXY/HTTPS_PROXY environment, and TLS uses the system roots.
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
MaxConnsPerHost: perHost,
MaxIdleConnsPerHost: perHost,
MaxConnsPerHost: cfg.Split,
MaxIdleConnsPerHost: cfg.Split,
ResponseHeaderTimeout: cfg.Timeout,
TLSHandshakeTimeout: connectTimeout,
DialContext: dialContext,
TLSClientConfig: tlsCfg,
}
if cfg.Proxy != "" {
pu, err := url.Parse(cfg.Proxy)
switch {
case err != nil:
// A malformed proxy must fail loudly, not silently fall back to the
// environment proxy (or a direct connection) and leak the real IP.
if initErr == nil {
initErr = fmt.Errorf("proxy %q: %w", cfg.Proxy, err)
}
case pu.Host == "":
// "host:port" with no scheme parses to an opaque URL with no Host, which
// http.ProxyURL would route nowhere; require an explicit scheme.
if initErr == nil {
initErr = fmt.Errorf("proxy %q: missing scheme (use scheme://host:port)", cfg.Proxy)
}
default:
tr.Proxy = http.ProxyURL(pu)
}
}
// Seed the cookie jar from --load-cookies; a load error becomes an initErr so
// Run surfaces it rather than silently downloading uncredentialed.
var jar *cookiejar.Jar
if cfg.LoadCookies != "" || cfg.SaveCookies != "" {
if j, err := loadCookieJar(cfg.LoadCookies); err != nil {
if initErr == nil {
initErr = err
}
} else {
jar = j
}
TLSHandshakeTimeout: cfg.Timeout,
DialContext: (&net.Dialer{Timeout: cfg.Timeout}).DialContext,
}
var limit []*rate.Limiter
if cfg.OverallLimiter != nil {
limit = append(limit, cfg.OverallLimiter)
}
if cfg.Limit > 0 {
limit = append(limit, rate.NewLimiter(rate.Limit(cfg.Limit), download.LimiterBurst(cfg.Limit)))
}
// Seed a best-effort name from -o or the primary URL. The real name is
// finalised in Run after probe(), where Content-Disposition and the
@@ -283,31 +155,19 @@ func New(uris []string, cfg Config) *Download {
if len(uris) == 0 && initErr == nil {
initErr = errors.New("no URL to download")
}
// Worker count: --split connections are spread across the mirrors, capped
// at --max-connection-per-server per host, so the ceiling is
// min(split, M*max-conn-per-server). One mirror reduces to min(split, x).
// Worker count is --split: the file is cut into that many byte-range segments,
// each its own connection, whether served by one host or spread across mirrors.
maxConns := cfg.Split
if m := max(1, len(uris)) * perHost; maxConns > m {
maxConns = m
}
if maxConns < 1 {
maxConns = 1
}
// Only attach the jar when one was built: a typed-nil *cookiejar.Jar stored
// in the http.Client.Jar interface is non-nil and would panic on use.
client := &http.Client{Transport: tr}
if jar != nil {
client.Jar = jar
}
d := &Download{
uris: uris,
maxConns: maxConns,
cfg: cfg,
client: client,
client: &http.Client{Transport: tr},
limit: limit,
jar: jar,
total: -1,
initErr: initErr,
}
@@ -318,7 +178,8 @@ func New(uris []string, cfg Config) *Download {
func (d *Download) Name() string { return d.loadName() }
// Path returns the resolved output file path (used by --follow-torrent).
// Path returns the resolved output file path, so a fetched .torrent can be
// followed into a BitTorrent download of its content.
func (d *Download) Path() string { return d.loadOut() }
// name and out are read by Stat from the reporter goroutine while Run resolves
@@ -359,13 +220,12 @@ func (d *Download) setStatus(s download.Status) {
// probeResult carries everything probe() learned, so name finalisation can run
// after the request (the final URL and Content-Disposition are only known then).
type probeResult struct {
total int64
ranged bool
etag string
lastmod string
cdisp string // raw Content-Disposition header
finalURL *url.URL
notModified bool // server answered 304 to a conditional GET
total int64
ranged bool
etag string
lastmod string
cdisp string // raw Content-Disposition header
finalURL *url.URL
}
// Run probes the resource, finalises the output name, decides between a single
@@ -388,12 +248,6 @@ func (d *Download) Run(ctx context.Context) (err error) {
if err != nil {
return err
}
// --conditional-get: the server said our local copy is current, so there is
// nothing to download. Report success without touching the file.
if pr.notModified {
d.setStatus(download.Complete)
return nil
}
atomic.StoreInt64(&d.total, pr.total)
// Finalise name/out now that the post-redirect URL and Content-Disposition
@@ -421,25 +275,13 @@ func (d *Download) Run(ctx context.Context) (err error) {
return nil
}
// --lowest-speed-limit: cancel the transfer if its speed stays at or below
// the limit past a startup grace. The watcher cancels xferCtx and sets a
// flag so we can return the distinct "too slow" error rather than a bare
// cancellation.
out := d.loadOut()
xferCtx, stopGuard, tooSlow := d.speedGuard(ctx)
if !pr.ranged || pr.total <= 0 {
err = d.single(xferCtx, out)
err = d.single(ctx, out)
} else {
err = d.segmented(xferCtx, out, pr.total, pr.etag, pr.lastmod)
err = d.segmented(ctx, out, pr.total, pr.etag, pr.lastmod)
}
stopGuard()
if err != nil {
// Only translate to ErrTooSlow when the transfer actually ended in the
// guard's cancellation; a write error or mirror exhaustion that merely
// raced the guard firing keeps its own (real) cause and exit code.
if tooSlow() && errors.Is(err, context.Canceled) {
return fmt.Errorf("%s: %w (<= %d bytes/sec)", d.primary(), ErrTooSlow, d.cfg.LowestSpeedLimit)
}
return err
}
// --checksum: verify the finished file before declaring success, so a
@@ -449,12 +291,6 @@ func (d *Download) Run(ctx context.Context) (err error) {
return err
}
}
// -R / --remote-time: stamp the finished file with the server's mtime.
if d.cfg.RemoteTime && pr.lastmod != "" {
if t, err := http.ParseTime(pr.lastmod); err == nil {
os.Chtimes(out, t, t)
}
}
d.setStatus(download.Complete)
return nil
}
@@ -507,20 +343,6 @@ func (d *Download) resolveName(pr probeResult) error {
return nil
}
// backoff waits RetryWait between attempts, but never after the final attempt,
// and returns ctx.Err() if cancelled while waiting (item [20]).
func backoff(ctx context.Context, wait time.Duration, tries, attempt int) error {
if wait <= 0 || (tries != 0 && attempt >= tries-1) {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(wait):
return nil
}
}
// retriesExhausted wraps the final cause once the retry budget is spent, so the
// caller — and the exit-code mapping in main — can still see whether it was DNS,
// a refused connection, or a timeout, rather than an opaque "too many retries".
@@ -538,12 +360,11 @@ func retriesExhausted(what string, tries int, cause error) error {
}
// withRetries runs attempt up to d.cfg.Tries times (0 = unlimited), the shared
// tries/backoff/fatal policy that probe, the single stream, and each segment all
// use so one transient DNS/connect/5xx does not fail the whole download (item
// [18]). It returns immediately on success (nil), on context cancellation, or on
// a fatal{} error (retrying cannot help), sleeps RetryWait between attempts, and
// wraps the final cause with retriesExhausted(what, ...) once the budget is
// spent so the underlying cause stays inspectable.
// tries/fatal policy that probe, the single stream, and each segment all use so
// one transient DNS/connect/5xx does not fail the whole download (item [18]). It
// returns immediately on success (nil), on context cancellation, or on a fatal{}
// error (retrying cannot help), and wraps the final cause with retriesExhausted
// once the budget is spent so the underlying cause stays inspectable.
func (d *Download) withRetries(ctx context.Context, what string, attempt func() error) error {
tries := d.cfg.Tries
var lastErr error
@@ -560,9 +381,6 @@ func (d *Download) withRetries(ctx context.Context, what string, attempt func()
if errors.As(err, &fe) {
return err // permanent: retrying cannot help
}
if err := backoff(ctx, d.cfg.RetryWait, tries, n); err != nil {
return err
}
}
return retriesExhausted(what, tries, lastErr)
}
@@ -614,15 +432,6 @@ func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) {
if err != nil {
return probeResult{}, err
}
// --conditional-get: ask the server to skip the transfer when our local file
// is already current. Only done when there is no .got control file (an
// in-progress resume must not be short-circuited), using the local file's
// mtime for If-Modified-Since.
if out := d.loadOut(); d.cfg.ConditionalGet && !fileExists(controlPath(out)) {
if fi, err := os.Stat(out); err == nil {
req.Header.Set("If-Modified-Since", fi.ModTime().UTC().Format(http.TimeFormat))
}
}
resp, err := d.client.Do(req)
if err != nil {
return probeResult{}, err
@@ -639,11 +448,6 @@ func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) {
pr.finalURL = resp.Request.URL
}
if resp.StatusCode == http.StatusNotModified {
pr.notModified = true
return pr, nil
}
switch resp.StatusCode {
case http.StatusPartialContent:
// Content-Range: bytes 0-0/12345
@@ -864,74 +668,6 @@ func (d *Download) idleGuard(body io.Reader, cancel context.CancelFunc) (io.Read
return ir, func() { close(stop) }
}
// speedStartupGrace is how long a download may stay slow before the
// lowest-speed-limit watchdog starts checking it: a startup idle time of
// 10 seconds.
const speedStartupGrace = 10 * time.Second
// belowSpeed reports whether the bytes transferred between two cumulative
// d.completed samples (prev -> now over one window) put the download at or
// below limit. A negative delta is not a slow second: when a resumed Range is
// answered by a 200 the transfer restarts from offset 0, so d.completed jumps
// backward on that retry. Treating that backward jump as "below limit" would
// false-abort a healthy download, so a reset (delta < 0) reports false; the
// caller has already advanced its baseline, so the next full window measures the
// real speed. Only a genuine non-negative delta at or under the limit aborts.
func belowSpeed(prev, now, limit int64) bool {
delta := now - prev
if delta < 0 {
return false // counter reset, not a slow window
}
return delta <= limit
}
// speedGuard implements --lowest-speed-limit. It returns a child context that
// the watcher cancels (setting a flag) once the measured download speed has
// stayed at or below the limit past the startup grace, plus a stop func and a
// predicate reporting whether the watcher fired. When the limit is 0 it is a
// no-op passthrough.
//
// Speed is sampled from d.completed over a one-second window: a single slow
// second after the grace aborts the download as too slow.
func (d *Download) speedGuard(ctx context.Context) (context.Context, func(), func() bool) {
if d.cfg.LowestSpeedLimit <= 0 {
return ctx, func() {}, func() bool { return false }
}
child, cancel := context.WithCancel(ctx)
var fired atomic.Bool
done := make(chan struct{})
go func() {
defer close(done)
start := time.Now()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
last := atomic.LoadInt64(&d.completed)
for {
select {
case <-child.Done():
return
case <-ticker.C:
now := atomic.LoadInt64(&d.completed)
prev := last
last = now
if time.Since(start) < speedStartupGrace {
continue
}
if belowSpeed(prev, now, d.cfg.LowestSpeedLimit) {
fired.Store(true)
cancel()
return
}
}
}
}()
stop := func() {
cancel()
<-done
}
return child, stop, func() bool { return fired.Load() }
}
// fatal marks an error that retrying cannot fix. Unwrap exposes the wrapped
// error so errors.Is/As traverse a fatal{} — main's exit-code mapping relies on
// errors.Is(err, ErrNotFound) seeing through it (CONTRACT).
@@ -1134,13 +870,13 @@ func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDi
}
}
// saveInterval is how often the .got control file is rewritten so a crash can
// resume; on a clean finish it is removed.
const saveInterval = 60 * time.Second
func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, segs []seg, done chan<- struct{}) {
defer close(done)
interval := d.cfg.AutoSaveInterval
if interval <= 0 {
interval = 60 * time.Second // --auto-save-interval default
}
t := time.NewTicker(interval)
t := time.NewTicker(saveInterval)
defer t.Stop()
for {
select {
@@ -1171,13 +907,9 @@ func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Req
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", d.cfg.UserAgent)
if d.cfg.Referer != "" {
req.Header.Set("Referer", d.cfg.Referer)
}
if d.cfg.HTTPUser != "" {
req.SetBasicAuth(d.cfg.HTTPUser, d.cfg.HTTPPasswd)
}
req.Header.Set("User-Agent", userAgent)
// --header sets arbitrary request headers, applied last so they can override
// the User-Agent (or add a Referer, Authorization, Cookie, ...) when needed.
for _, h := range d.cfg.Headers {
if k, v, ok := strings.Cut(h, ":"); ok {
req.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
@@ -1186,33 +918,9 @@ func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Req
if rangeHdr != "" {
req.Header.Set("Range", rangeHdr)
}
if d.jar != nil {
d.recordURL(req.URL)
}
return req, nil
}
// recordURL remembers a contacted URL so SaveCookies can later read back the
// jar's cookies for that host. Duplicates are tolerated; saveCookieJar dedups.
func (d *Download) recordURL(u *url.URL) {
d.mu.Lock()
d.seenURLs = append(d.seenURLs, u)
d.mu.Unlock()
}
// SaveCookies writes the client's cookie jar back to the --save-cookies file,
// if one was configured. It is a no-op when cookies are not in use. main calls
// it once per HTTP download on exit (cookies are saved at session end).
func (d *Download) SaveCookies() error {
if d.cfg.SaveCookies == "" || d.jar == nil {
return nil
}
d.mu.Lock()
urls := append([]*url.URL{}, d.seenURLs...)
d.mu.Unlock()
return saveCookieJar(d.jar, urls, d.cfg.SaveCookies)
}
// seedSegmentsFromPrefix marks the first prefix bytes of the file as already
// downloaded, segment by segment. A foreign partial file has no per-segment
// metadata, so the only thing we can trust is that bytes [0,prefix) are present;