From cf9082ff729484a20e684b00113fe7d096b22831 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Mon, 22 Jun 2026 08:30:12 +0900 Subject: [PATCH] all: cut bloated HTTP-client and tuning options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a personal torrent client, not a web client or daemon, so the rarely-used HTTP-client knobs and redundant tuning flags are gone — flag, config field, implementation, and tests removed for each. The option surface drops from 54 to 28. Web-client cluster removed: --all-proxy (HTTP_PROXY env still works), load/save-cookies, --referer, -U/--user-agent, --http-user/--http-passwd, --conditional-get, --remote-time, --check-certificate, --ca-certificate, --lowest-speed-limit. Pass --header for a one-off auth/cookie/UA header. Redundant knobs removed: -x and -k (folded into -s), --connect-timeout, --retry-wait, per-item --max-download-limit/-u, --bt-max-peers, --bt-stop-timeout, --disable-ipv6, --auto-save-interval, --human-readable, --stop, -T (a positional .torrent works). follow-torrent and human-readable are now unconditional defaults. --- README.md | 16 +- bt/bt.go | 55 +---- cli/option.go | 36 +--- cli/parse.go | 19 +- cli/parse_test.go | 29 +-- httpdl/ca_test.go | 22 -- httpdl/conditional_test.go | 97 --------- httpdl/cookies.go | 177 ---------------- httpdl/cookies_test.go | 126 ------------ httpdl/httpdl.go | 406 ++++++------------------------------- httpdl/mirror_test.go | 27 ++- httpdl/single_test.go | 16 +- httpdl/speed_test.go | 32 --- main.go | 136 ++++--------- main_test.go | 16 +- 15 files changed, 157 insertions(+), 1053 deletions(-) delete mode 100644 httpdl/ca_test.go delete mode 100644 httpdl/conditional_test.go delete mode 100644 httpdl/cookies.go delete mode 100644 httpdl/cookies_test.go delete mode 100644 httpdl/speed_test.go diff --git a/README.md b/README.md index aec3c7e..348c057 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ A command-line download tool in Go. It fetches HTTP(S) files over multiple connections and downloads BitTorrent magnets and `.torrent` files — one binary, -familiar flags. +a small set of flags. It is a downloader, not a web client: there are no +cookie/proxy/auth flags — pass `--header` for a one-off header (auth token, +cookie, user-agent) when a download needs one. ## Build @@ -22,10 +24,9 @@ got -s16 https://example.com/big.iso # resume an interrupted download got -c https://example.com/big.iso -# one file from several mirrors (connections spread across servers, +# one file from several mirrors (segments spread across the servers, # a dead mirror falls over); -Z downloads them as separate files instead. -# -x caps connections per server when mirroring. -got -s16 -x4 https://a.example/big.iso https://b.example/big.iso +got -s16 https://a.example/big.iso https://b.example/big.iso # a torrent or magnet; seed for 30 minutes after finishing got --seed-time=30 ubuntu.torrent @@ -45,7 +46,6 @@ Run `got --help` (or `--help=all`) for every option. | `-o, --out` | output filename | from URL | | `-c, --continue` | resume a partial download | false | | `-s, --split` | connections per download (the speed dial) | 5 | -| `-x, --max-connection-per-server` | cap connections per server (1–16) | 1 (= split when unset) | | `-j, --max-concurrent-downloads` | files downloaded at once | 5 | | `--max-overall-download-limit` | global speed cap | 0 (off) | | `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, …) | | @@ -69,6 +69,6 @@ got -c --save-session=got.session -i got.session - BitTorrent uses [`anacrolix/torrent`](https://github.com/anacrolix/torrent); HTTP uses the standard library. - Not implemented: FTP/SFTP, Metalink, and the JSON-RPC server. -- When several torrents run at once, `--max-download-limit` / - `--max-upload-limit` apply per run rather than per torrent (`--max-overall-*` - are exact). +- Speed caps are global: `--max-overall-download-limit` / + `--max-overall-upload-limit` apply across the whole run. +- Standard `HTTP_PROXY` / `HTTPS_PROXY` environment variables are honored. diff --git a/bt/bt.go b/bt/bt.go index d5b4fda..a8aa541 100644 --- a/bt/bt.go +++ b/bt/bt.go @@ -28,15 +28,10 @@ import ( // ClientConfig holds the run-wide settings used to build the shared client. type ClientConfig struct { Dir string - ListenPortSpec string // raw --listen-port spec ("6881-6999,7000"); empty lets the OS choose. - DHT bool - MaxPeers int // max peer connections per torrent (0 = library default) + ListenPortSpec string // raw --listen-port spec ("6881-6999,7000"); empty lets the OS choose. + DHT bool // --enable-dht OverallDown *rate.Limiter // global down limit (may be nil) OverallUp *rate.Limiter // global up limit (may be nil) - DownLimit int64 // per-run bytes/s, 0 = unlimited (used only without a global limiter) - UpLimit int64 - UserAgent string - DisableIPv6 bool // force IPv4-only (--disable-ipv6) } // ParsePorts expands a --listen-port spec into a flat list of candidate ports. @@ -94,19 +89,9 @@ func quietSlogger(w io.Writer) *slog.Logger { // NewClient builds the shared anacrolix client from the run-wide config. When a // listen-port spec is given it tries each candidate port in turn, advancing past // any that is already in use, and falls back to an OS-chosen port if none bind, -// so a single busy port does not disable BitTorrent entirely. -// -// Two per-torrent features have no equivalent in anacrolix/torrent v1.61.0, -// which exposes only client-wide knobs; both must therefore be configured run-wide -// here rather than attached to an individual Torrent: -// -// 1. No per-Torrent rate limiter. Throttles are read from the client-wide -// DownloadRateLimiter/UploadRateLimiter (no TorrentSpec rate field exists), so -// per-torrent --max-download-limit / --max-upload-limit cannot be -// honored individually; the effective cap is the shared limiter set below. -// 2. No per-Torrent DHT/PEX/LSD toggle for BEP27 private torrents. Those are -// client-wide only (NoDHT, DisablePEX, etc.), so a private torrent's leak -// prevention must come from a run-wide setting (e.g. --enable-dht=false). +// so a single busy port does not disable BitTorrent entirely. The rate limits +// and the DHT toggle are client-wide (anacrolix has no per-torrent equivalent), +// which is why they live here on the shared client rather than per download. func NewClient(cfg ClientConfig) (*torrent.Client, error) { c := torrent.NewDefaultClientConfig() // Keep the library's own logging out of the readout. Webseed/peer warnings go @@ -120,24 +105,11 @@ func NewClient(cfg ClientConfig) (*torrent.Client, error) { } c.Seed = true c.NoDHT = !cfg.DHT - if cfg.MaxPeers > 0 { - c.EstablishedConnsPerTorrent = cfg.MaxPeers - } - if cfg.UserAgent != "" { - c.HTTPUserAgent = cfg.UserAgent - } - c.DisableIPv6 = cfg.DisableIPv6 - switch { - case cfg.OverallDown != nil: + if cfg.OverallDown != nil { c.DownloadRateLimiter = cfg.OverallDown - case cfg.DownLimit > 0: - c.DownloadRateLimiter = rate.NewLimiter(rate.Limit(cfg.DownLimit), download.LimiterBurst(cfg.DownLimit)) } - switch { - case cfg.OverallUp != nil: + if cfg.OverallUp != nil { c.UploadRateLimiter = cfg.OverallUp - case cfg.UpLimit > 0: - c.UploadRateLimiter = rate.NewLimiter(rate.Limit(cfg.UpLimit), download.LimiterBurst(cfg.UpLimit)) } // Candidate ports from the --listen-port spec. @@ -183,7 +155,6 @@ type Options struct { SeedTimeSet bool // was --seed-time given? SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding) SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime - StopTimeout time.Duration // abort if no download progress for this long (0 = off) SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all CheckIntegrity bool // re-verify data against piece hashes before downloading DryRun bool // fetch metadata only, then stop without downloading (--dry-run) @@ -409,14 +380,11 @@ func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error { } } -// wait blocks until the wanted data is complete, ctx is cancelled, or the -// download stalls past --bt-stop-timeout. +// wait blocks until the wanted data is complete or ctx is cancelled, polling +// completion a couple of times a second. func (d *Download) wait(ctx context.Context, t *torrent.Torrent) error { tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() - - last := t.BytesCompleted() - lastChange := time.Now() for { if d.complete(t) { return nil @@ -425,11 +393,6 @@ func (d *Download) wait(ctx context.Context, t *torrent.Torrent) error { case <-ctx.Done(): return ctx.Err() case <-tick.C: - if n := t.BytesCompleted(); n != last { - last, lastChange = n, time.Now() - } else if d.opts.StopTimeout > 0 && time.Since(lastChange) > d.opts.StopTimeout { - return fmt.Errorf("no progress for %s", d.opts.StopTimeout) - } } } } diff --git a/cli/option.go b/cli/option.go index c6355c8..4138a6d 100644 --- a/cli/option.go +++ b/cli/option.go @@ -45,7 +45,7 @@ func (t Tag) String() string { // Opt describes one option: its names, value kind, default and help text. type Opt struct { - Long string // long name without dashes, e.g. "max-connection-per-server" + Long string // long name without dashes, e.g. "max-concurrent-downloads" Short byte // short flag byte, e.g. 'x'; 0 if none Kind Kind Default string @@ -82,52 +82,26 @@ var options = []Opt{ {Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic}, // --- http --- - {Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "cap connections to any single server (1-16); limits --split per host, mainly for mirrors", Tag: HTTP}, - {Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: HTTP}, - {Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: HTTP}, {Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP}, {Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP}, - {Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection timeout in seconds", Tag: HTTP}, - {Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP}, - {Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP}, - {Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable)", Tag: HTTP}, - {Long: "user-agent", Short: 'U', Kind: Str, Default: "got/1.0", Help: "HTTP User-Agent", Tag: HTTP}, - {Long: "referer", Kind: Str, Help: "HTTP Referer header", Tag: HTTP}, - {Long: "all-proxy", Kind: Str, Help: "proxy for all protocols (host:port)", Tag: HTTP}, - {Long: "check-certificate", Kind: Bool, Default: "true", Help: "verify the server's TLS certificate", Tag: HTTP}, - {Long: "ca-certificate", Kind: Str, Help: "verify HTTPS servers against the CA certificates in FILE (PEM)", Tag: HTTP}, - {Long: "load-cookies", Kind: Str, Help: "load Cookies from FILE (Netscape/Mozilla format)", Tag: HTTP}, - {Long: "save-cookies", Kind: Str, Help: "save Cookies to FILE on exit", Tag: HTTP}, - {Long: "conditional-get", Kind: Bool, Default: "false", Help: "download only if the remote file is newer than the local one", Tag: HTTP}, - {Long: "remote-time", Kind: Bool, Default: "false", Help: "set the local file's mtime from the server", Tag: HTTP}, - {Long: "http-user", Kind: Str, Help: "HTTP basic-auth user", Tag: HTTP}, - {Long: "http-passwd", Kind: Str, Help: "HTTP basic-auth password", Tag: HTTP}, - {Long: "lowest-speed-limit", Kind: Size, Default: "0", Help: "abort if speed stays below SIZE bytes/sec (0 = off)", Tag: HTTP}, + {Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection/idle timeout in seconds", Tag: HTTP}, + {Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable), e.g. 'Authorization: Bearer …'", Tag: HTTP}, // --- bittorrent --- - {Long: "torrent-file", Short: 'T', Kind: Str, Help: "path to a .torrent file", Tag: BitTorrent}, {Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes", Tag: BitTorrent}, {Long: "seed-time", Kind: Float, Metavar: "MIN", Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent}, {Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent}, - {Long: "bt-stop-timeout", Kind: Int, Default: "0", Min: 0, Help: "stop a torrent with no download progress for N seconds (0 = off)", Tag: BitTorrent}, {Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent}, - {Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT", Tag: BitTorrent}, - {Long: "bt-max-peers", Kind: Int, Default: "55", Min: 0, Help: "max peers per torrent (0 = unlimited)", Tag: BitTorrent}, + {Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT (turn off for private torrents)", Tag: BitTorrent}, {Long: "max-overall-upload-limit", Kind: Size, Default: "0", Help: "global upload speed limit (0 = unlimited)", Tag: BitTorrent}, - {Long: "max-upload-limit", Short: 'u', Kind: Size, Default: "0", Help: "per-torrent upload speed limit", Tag: BitTorrent}, {Long: "select-file", Kind: Str, Help: "download only these file indexes, e.g. 1,3-5", Tag: BitTorrent}, - {Long: "follow-torrent", Kind: Bool, Default: "true", Help: "after fetching a .torrent over HTTP, download its content", Tag: BitTorrent}, // --- advanced --- - {Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Advanced}, + {Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file instead of renaming the new one", Tag: Advanced}, {Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Advanced}, - {Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Advanced}, {Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to reserve disk space (prealloc and falloc are identical here)", Tag: Advanced}, {Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced}, - {Long: "stop", Kind: Int, Default: "0", Min: 0, Help: "stop the program after N seconds (0 = off)", Tag: Advanced}, - {Long: "disable-ipv6", Kind: Bool, Default: "false", Help: "disable IPv6 (force IPv4-only connections)", Tag: Advanced}, {Long: "save-session", Kind: Str, Help: "on exit, write unfinished downloads to FILE (reload with -i)", Tag: Advanced}, - {Long: "auto-save-interval", Kind: Int, Default: "60", Min: 0, Max: 600, Help: "save the session file every N seconds (0 = only on exit)", Tag: Advanced}, {Long: "conf-path", Kind: Str, Help: "path to the config file", Tag: Advanced}, {Long: "no-conf", Kind: Bool, Default: "false", Help: "do not read a config file", Tag: Advanced}, } diff --git a/cli/parse.go b/cli/parse.go index 0bccfe3..4911789 100644 --- a/cli/parse.go +++ b/cli/parse.go @@ -27,8 +27,8 @@ type Result struct { } // Parse resolves args into options and URIs, applying the layered override -// order: built-in defaults, then the config file, then proxy -// environment variables, then the command line (last wins). +// order: built-in defaults, then the config file, then the command line +// (last wins). func Parse(args []string) (*Result, error) { cmdVals, uris, action, helpTag, err := parseArgs(args) if err != nil { @@ -48,7 +48,6 @@ func Parse(args []string) (*Result, error) { return nil, err } } - applyEnv(opts) for name, val := range cmdVals { if err := set(opts, name, val); err != nil { @@ -225,20 +224,6 @@ func applyConfig(o *Options, path string, explicit bool) error { return sc.Err() } -// applyEnv folds the conventional proxy environment variables into all-proxy -// unless it was already set explicitly. -func applyEnv(o *Options) { - if o.set["all-proxy"] { - return - } - for _, key := range []string{"all_proxy", "https_proxy", "http_proxy"} { - if v := os.Getenv(key); v != "" { - o.vals["all-proxy"] = v - return - } - } -} - // set stores a raw value, marking the option as explicitly set. Validation and // conversion happen later, once, in finalize(); set only records the string. func set(o *Options, name, val string) error { diff --git a/cli/parse_test.go b/cli/parse_test.go index a14a75d..c60a021 100644 --- a/cli/parse_test.go +++ b/cli/parse_test.go @@ -13,7 +13,7 @@ func TestParseArgs(t *testing.T) { wantVal map[string]string uris []string }{ - {"short attached", []string{"-x16", "https://e.com"}, Run, map[string]string{"max-connection-per-server": "16"}, []string{"https://e.com"}}, + {"short attached", []string{"-m3", "https://e.com"}, Run, map[string]string{"max-tries": "3"}, []string{"https://e.com"}}, {"long equals + bool", []string{"--split=8", "-c", "u"}, Run, map[string]string{"split": "8", "continue": "true"}, []string{"u"}}, {"long separate value", []string{"--out", "f.bin", "u"}, Run, map[string]string{"out": "f.bin"}, []string{"u"}}, {"bool bundle", []string{"-cq", "u"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"u"}}, @@ -22,7 +22,7 @@ func TestParseArgs(t *testing.T) { {"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}}, {"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}}, {"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}}, - {"long int + magnet uri", []string{"--bt-stop-timeout=120", "magnet:x"}, Run, map[string]string{"bt-stop-timeout": "120"}, []string{"magnet:x"}}, + {"long int + magnet uri", []string{"--timeout=120", "magnet:x"}, Run, map[string]string{"timeout": "120"}, []string{"magnet:x"}}, {"help", []string{"--help"}, ShowHelp, nil, nil}, {"help short", []string{"-h"}, ShowHelp, nil, nil}, {"version", []string{"-v"}, ShowVersion, nil, nil}, @@ -95,13 +95,18 @@ func TestParseSize(t *testing.T) { } func TestParseValueRange(t *testing.T) { - x := byLong["max-connection-per-server"] - if _, err := parseValue(x, "16"); err != nil { + // A synthetic bounded Int exercises the Min/Max range check independent of + // which options happen to carry bounds. + o := &Opt{Long: "n", Kind: Int, Min: 1, Max: 16} + if _, err := parseValue(o, "16"); err != nil { t.Errorf("parseValue 16: %v", err) } - if _, err := parseValue(x, "99"); err == nil { + if _, err := parseValue(o, "99"); err == nil { t.Errorf("parseValue 99: want out-of-range error") } + if _, err := parseValue(o, "0"); err == nil { + t.Errorf("parseValue 0: want below-minimum error") + } a := byLong["file-allocation"] if _, err := parseValue(a, "bogus"); err == nil { t.Errorf("parseValue enum bogus: want error") @@ -156,17 +161,17 @@ func TestParseValueBool(t *testing.T) { } func TestParseValueSizeBounds(t *testing.T) { - k := byLong["min-split-size"] // 1M..1024M (no G unit) + k := &Opt{Long: "sz", Kind: Size, Min: 1 << 20, Max: 1 << 30} // 1M..1024M if _, err := parseValue(k, "20M"); err != nil { - t.Errorf("parseValue min-split-size 20M: %v", err) + t.Errorf("parseValue 20M: %v", err) } if _, err := parseValue(k, "512K"); err == nil { - t.Errorf("parseValue min-split-size 512K: want below-minimum error") + t.Errorf("parseValue 512K: want below-minimum error") } // 1025M exceeds the 1<<30 cap and still parses, so it tests the bounds path // (where "2G" would have failed earlier as an unknown unit). if _, err := parseValue(k, "1025M"); err == nil { - t.Errorf("parseValue min-split-size 1025M: want above-maximum error") + t.Errorf("parseValue 1025M: want above-maximum error") } } @@ -204,7 +209,7 @@ func TestExplicitConfMissingFatal(t *testing.T) { func TestOptionsGetters(t *testing.T) { o := newOptions() o.vals["split"] = "8" - o.vals["min-split-size"] = "20M" + o.vals["max-overall-download-limit"] = "20M" o.vals["continue"] = "true" o.vals["seed-ratio"] = "1.5" if err := o.finalize(); err != nil { // parse the raw values once, as Parse does @@ -213,8 +218,8 @@ func TestOptionsGetters(t *testing.T) { if o.Int("split") != 8 { t.Errorf("Int split = %d", o.Int("split")) } - if o.Size("min-split-size") != 20<<20 { - t.Errorf("Size min-split-size = %d", o.Size("min-split-size")) + if o.Size("max-overall-download-limit") != 20<<20 { + t.Errorf("Size max-overall-download-limit = %d", o.Size("max-overall-download-limit")) } if !o.Bool("continue") { t.Errorf("Bool continue = false") diff --git a/httpdl/ca_test.go b/httpdl/ca_test.go deleted file mode 100644 index 8ebd5b0..0000000 --- a/httpdl/ca_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package httpdl - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLoadCAPool(t *testing.T) { - // A missing file is an error, not a silent fall-back to the system roots. - if _, err := loadCAPool(filepath.Join(t.TempDir(), "absent.pem")); err == nil { - t.Errorf("loadCAPool(missing): want error") - } - // A file with no PEM certificates is an error (AppendCertsFromPEM found none). - bad := filepath.Join(t.TempDir(), "bad.pem") - if err := os.WriteFile(bad, []byte("not a certificate\n"), 0o644); err != nil { - t.Fatal(err) - } - if _, err := loadCAPool(bad); err == nil { - t.Errorf("loadCAPool(garbage): want error (no certificates)") - } -} diff --git a/httpdl/conditional_test.go b/httpdl/conditional_test.go deleted file mode 100644 index c89341c..0000000 --- a/httpdl/conditional_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package httpdl - -import ( - "context" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "github.com/hanbok/got/download" -) - -// TestConditionalGet304 checks that --conditional-get sends If-Modified-Since -// for an existing local file and treats a 304 as success without rewriting the -// file. -func TestConditionalGet304(t *testing.T) { - dir := t.TempDir() - out := filepath.Join(dir, "file.bin") - const existing = "already here" - if err := os.WriteFile(out, []byte(existing), 0o644); err != nil { - t.Fatal(err) - } - - gotIfModSince := false - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("If-Modified-Since") != "" { - gotIfModSince = true - } - w.WriteHeader(http.StatusNotModified) - })) - defer srv.Close() - - d := New([]string{srv.URL + "/file.bin"}, Config{ - Dir: dir, - Out: "file.bin", - MaxConnPerServer: 1, - ConditionalGet: true, - AllowOverwrite: true, - UserAgent: "got-test", - }) - if err := d.Run(context.Background()); err != nil { - t.Fatalf("conditional-get 304 should succeed, got %v", err) - } - if !gotIfModSince { - t.Fatal("expected an If-Modified-Since header on the request") - } - if d.Stat().Status != download.Complete { - t.Fatalf("status = %v, want Complete", d.Stat().Status) - } - // The file must be untouched: 304 means "up to date". - b, err := os.ReadFile(out) - if err != nil { - t.Fatal(err) - } - if string(b) != existing { - t.Fatalf("file was rewritten: %q, want %q", b, existing) - } -} - -// TestConditionalGetSkippedWithControlFile verifies the rule that -// --conditional-get is ignored when a .got control file exists, so an -// in-progress resume is never short-circuited by a 304. -func TestConditionalGetSkippedWithControlFile(t *testing.T) { - dir := t.TempDir() - out := filepath.Join(dir, "file.bin") - if err := os.WriteFile(out, []byte("partial"), 0o644); err != nil { - t.Fatal(err) - } - // A control file present means a resume is in flight. - if err := os.WriteFile(controlPath(out), []byte("{}"), 0o644); err != nil { - t.Fatal(err) - } - - sawIfModSince := false - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("If-Modified-Since") != "" { - sawIfModSince = true - } - // Serve a small body so the run can complete. - http.Error(w, "no range", http.StatusOK) - })) - defer srv.Close() - - d := New([]string{srv.URL + "/file.bin"}, Config{ - Dir: dir, - Out: "file.bin", - MaxConnPerServer: 1, - ConditionalGet: true, - AllowOverwrite: true, - UserAgent: "got-test", - }) - _ = d.Run(context.Background()) - if sawIfModSince { - t.Fatal("If-Modified-Since must not be sent when a control file exists") - } -} diff --git a/httpdl/cookies.go b/httpdl/cookies.go deleted file mode 100644 index 66db54b..0000000 --- a/httpdl/cookies.go +++ /dev/null @@ -1,177 +0,0 @@ -package httpdl - -import ( - "bufio" - "fmt" - "math" - "net/http" - "net/http/cookiejar" - "net/url" - "os" - "strconv" - "strings" - "time" -) - -// Cookie handling implements the --load-cookies / --save-cookies options, which -// read and write the Netscape/Mozilla cookies.txt format (one tab-separated -// cookie per line). We back it with the standard net/http/cookiejar so the -// http.Client applies and tracks Set-Cookie headers automatically; the file -// format is just the on-disk representation of that jar. - -// loadCookieJar builds a cookie jar seeded from a Netscape cookies.txt file. A -// missing file yields an empty jar (not an error): there are simply no cookies -// to load. A malformed line is skipped, giving per-line tolerance. -func loadCookieJar(path string) (*cookiejar.Jar, error) { - jar, err := cookiejar.New(nil) - if err != nil { - return nil, err - } - if path == "" { - return jar, nil - } - f, err := os.Open(path) - if err != nil { - if os.IsNotExist(err) { - return jar, nil // nothing to load yet - } - return nil, fmt.Errorf("load-cookies %s: %w", path, err) - } - defer f.Close() - - // Group cookies by the URL their domain implies, since SetCookies keys on a - // URL. We synthesise a representative URL per (domain, secure, path). - byURL := map[string][]*http.Cookie{} - urls := map[string]*url.URL{} - sc := bufio.NewScanner(f) - for sc.Scan() { - line := sc.Text() - c, u := parseNetscapeCookie(line) - if c == nil { - continue - } - key := u.String() - if _, ok := urls[key]; !ok { - urls[key] = u - } - byURL[key] = append(byURL[key], c) - } - if err := sc.Err(); err != nil { - return nil, fmt.Errorf("load-cookies %s: %w", path, err) - } - for key, cs := range byURL { - jar.SetCookies(urls[key], cs) - } - return jar, nil -} - -// parseNetscapeCookie parses one cookies.txt line into a cookie plus the URL it -// belongs to, or (nil, nil) for a comment, blank or malformed line. The format -// is seven tab-separated fields: -// -// domain includeSubdomains path secure expiry name value -// -// Curl marks HttpOnly cookies with a "#HttpOnly_" line prefix rather than a -// real comment, so we recognise that prefix instead of dropping the line. -// Honouring HttpOnly_ rather than skipping every "#" line never loses a cookie. -func parseNetscapeCookie(line string) (*http.Cookie, *url.URL) { - httpOnly := false - if strings.HasPrefix(line, "#HttpOnly_") { - httpOnly = true - line = strings.TrimPrefix(line, "#HttpOnly_") - } else if strings.HasPrefix(line, "#") { - return nil, nil - } - if strings.TrimSpace(line) == "" { - return nil, nil - } - f := strings.Split(line, "\t") - if len(f) < 6 { - return nil, nil - } - domain := f[0] - hostOnly := !strings.EqualFold(f[1], "TRUE") - path := f[2] - secure := strings.EqualFold(f[3], "TRUE") - name := f[5] - value := "" - if len(f) >= 7 { - value = f[6] - } - if name == "" || domain == "" { - return nil, nil - } - - c := &http.Cookie{ - Name: name, - Value: value, - Path: path, - Secure: secure, - HttpOnly: httpOnly, - } - // A non-host-only cookie (includeSubdomains TRUE) carries a Domain attribute - // so the jar applies it to subdomains; a host-only one leaves Domain empty so - // the jar binds it to the exact request host. - if !hostOnly { - c.Domain = domain - } - // expiry 0 means a session cookie; otherwise it is a Unix timestamp. The jar - // drops already-expired cookies on load. - if exp, err := strconv.ParseInt(strings.TrimSpace(f[4]), 10, 64); err == nil && exp > 0 { - c.Expires = time.Unix(exp, 0) - } - - // The jar needs a URL whose host and scheme match the cookie. A leading dot - // denotes a domain cookie; strip it for the host but keep the cookie Domain - // so the jar treats it as domain-wide. - host := strings.TrimPrefix(domain, ".") - scheme := "http" - if secure { - scheme = "https" - } - u := &url.URL{Scheme: scheme, Host: host, Path: path} - return c, u -} - -// saveCookieJar writes every cookie the jar holds (for the hosts we have -// contacted) back to path in Netscape format, implementing --save-cookies on -// exit. Because net/http/cookiejar exposes cookies only per URL, we ask it for -// the cookies of each host we recorded a request against. -func saveCookieJar(jar *cookiejar.Jar, urls []*url.URL, path string) error { - if path == "" || jar == nil { - return nil - } - var b strings.Builder - b.WriteString("# Netscape HTTP Cookie File\n") - seen := map[string]bool{} - for _, u := range urls { - for _, c := range jar.Cookies(u) { - // Cookies() drops the domain/path/expiry, so reconstruct sensible - // values: host-only domain, root path, far-future expiry. This is the - // best a host-only readback can do; it round-trips the name/value pair - // that authenticated downloads actually need. - key := u.Host + "\x00" + c.Name - if seen[key] { - continue - } - seen[key] = true - expiry := int64(math.MaxInt32) // session-stable far future - line := strings.Join([]string{ - u.Host, "FALSE", "/", - boolToTRUE(u.Scheme == "https"), - strconv.FormatInt(expiry, 10), - c.Name, c.Value, - }, "\t") - b.WriteString(line) - b.WriteByte('\n') - } - } - return os.WriteFile(path, []byte(b.String()), 0o600) -} - -func boolToTRUE(b bool) string { - if b { - return "TRUE" - } - return "FALSE" -} diff --git a/httpdl/cookies_test.go b/httpdl/cookies_test.go deleted file mode 100644 index 30602b9..0000000 --- a/httpdl/cookies_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package httpdl - -import ( - "net/http" - "os" - "path/filepath" - "strconv" - "testing" - "time" -) - -func TestParseNetscapeCookie(t *testing.T) { - future := time.Now().Add(24 * time.Hour).Unix() - tests := []struct { - name string - line string - wantNil bool - wantName string - wantValue string - wantDom string // expected http.Cookie.Domain (empty for host-only) - wantHTTP bool - }{ - { - name: "domain cookie", - line: ".example.com\tTRUE\t/\tFALSE\t" + itoa(future) + "\tSID\tabc123", - wantName: "SID", - wantValue: "abc123", - wantDom: ".example.com", - }, - { - name: "host-only cookie", - line: "example.com\tFALSE\t/\tTRUE\t" + itoa(future) + "\ttoken\txyz", - wantName: "token", - wantValue: "xyz", - wantDom: "", // host-only: no Domain attribute - }, - { - name: "httponly prefix kept", - line: "#HttpOnly_example.com\tFALSE\t/\tFALSE\t" + itoa(future) + "\tHO\tsecret", - wantName: "HO", - wantValue: "secret", - wantHTTP: true, - }, - { - name: "empty value field", - line: "example.com\tFALSE\t/\tFALSE\t0\tflag\t", - wantName: "flag", - wantValue: "", - }, - { - name: "comment line", - line: "# this is a comment", - wantNil: true, - }, - { - name: "blank line", - line: " ", - wantNil: true, - }, - { - name: "too few fields", - line: "example.com\tFALSE\t/\tFALSE\t0", - wantNil: true, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - c, u := parseNetscapeCookie(tc.line) - if tc.wantNil { - if c != nil { - t.Fatalf("expected nil cookie, got %+v", c) - } - return - } - if c == nil { - t.Fatal("expected a cookie, got nil") - } - if c.Name != tc.wantName || c.Value != tc.wantValue { - t.Fatalf("name/value = %q/%q, want %q/%q", c.Name, c.Value, tc.wantName, tc.wantValue) - } - if c.Domain != tc.wantDom { - t.Fatalf("domain = %q, want %q", c.Domain, tc.wantDom) - } - if c.HttpOnly != tc.wantHTTP { - t.Fatalf("httponly = %v, want %v", c.HttpOnly, tc.wantHTTP) - } - if u == nil { - t.Fatal("expected a URL, got nil") - } - }) - } -} - -func itoa(n int64) string { return strconv.FormatInt(n, 10) } -func itoa64(n int64) string { return strconv.FormatInt(n, 10) } - -func TestLoadCookieJarMissingFile(t *testing.T) { - // A missing cookies file is not an error: there are simply no cookies to load. - jar, err := loadCookieJar(filepath.Join(t.TempDir(), "nope.txt")) - if err != nil { - t.Fatalf("missing file should not error: %v", err) - } - if jar == nil { - t.Fatal("expected an empty jar, got nil") - } -} - -func TestLoadCookieJarAppliesCookie(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "cookies.txt") - future := time.Now().Add(24 * time.Hour).Unix() - data := "# Netscape HTTP Cookie File\n" + - ".example.com\tTRUE\t/\tFALSE\t" + itoa64(future) + "\tSID\tabc123\n" - if err := os.WriteFile(path, []byte(data), 0o600); err != nil { - t.Fatal(err) - } - jar, err := loadCookieJar(path) - if err != nil { - t.Fatal(err) - } - req, _ := http.NewRequest(http.MethodGet, "http://www.example.com/file", nil) - got := jar.Cookies(req.URL) - if len(got) != 1 || got[0].Name != "SID" || got[0].Value != "abc123" { - t.Fatalf("jar did not apply the domain cookie to a subdomain: %+v", got) - } -} diff --git a/httpdl/httpdl.go b/httpdl/httpdl.go index fc2df30..6b9c195 100644 --- a/httpdl/httpdl.go +++ b/httpdl/httpdl.go @@ -9,8 +9,6 @@ package httpdl import ( "context" - "crypto/tls" - "crypto/x509" "errors" "fmt" "hash" @@ -18,7 +16,6 @@ import ( "mime" "net" "net/http" - "net/http/cookiejar" "net/url" "os" "path" @@ -35,6 +32,15 @@ import ( const readBuf = 32 * 1024 +// minSplitSize is the smallest piece a file is split into: a segment shorter +// than this is not worth its own connection, so makeSegments stops dividing once +// the pieces would fall below it. +const minSplitSize = 20 << 20 + +// userAgent is sent on every request; there is no flag to change it. A site that +// needs a different one can be given a User-Agent header via --header. +const userAgent = "got/1.0" + // ErrOutputExists reports that the resolved output file already exists and // neither --allow-overwrite nor -c was given. main maps it to a distinct exit // code via errors.Is, so it must stay on the error chain (CONTRACT). @@ -45,60 +51,21 @@ var ErrOutputExists = errors.New("output file already exists") // (CONTRACT). var ErrNotFound = errors.New("not found") -// ErrTooSlow reports a transfer aborted by --lowest-speed-limit. main maps it to -// exit code 5 via errors.Is, so it must stay on the error chain (CONTRACT). -var ErrTooSlow = errors.New("download speed too low") - // Config holds everything one HTTP download needs. It is assembled by main from // the resolved CLI options. type Config struct { - Dir string - Out string - Split int // --split: total connections across all mirrors - MaxConnPerServer int // --max-connection-per-server: per-host connection cap - MinSplit int64 - Tries int // 0 = unlimited - Timeout time.Duration - FileAlloc string // none | prealloc | trunc | falloc - Continue bool - AllowOverwrite bool - AutoRename bool - Headers []string - UserAgent string - Referer string - Proxy string - CheckCert bool - Limit int64 // per-download bytes/s, 0 = unlimited - OverallLimiter *rate.Limiter // shared across all downloads, may be nil - - // RetryWait is the pause between retries of a transient failure - // (--retry-wait). 0 means retry immediately with no sleep. - RetryWait time.Duration - // AutoSaveInterval controls how often the .got control file is rewritten - // (--auto-save-interval). <=0 falls back to 60s. - AutoSaveInterval time.Duration - - // ConnectTimeout is the time allowed to establish the TCP/TLS connection - // (--connect-timeout, default 60s), distinct from Timeout which - // covers stalls once data is flowing. <=0 falls back to Timeout. - ConnectTimeout time.Duration - // LoadCookies / SaveCookies are paths to a Netscape cookies.txt read into - // the client's jar before the download and written back on exit. - LoadCookies string - SaveCookies string - // ConditionalGet sends If-Modified-Since (from the existing local file's - // mtime) so a 304 reports success without re-downloading - // (--conditional-get). - ConditionalGet bool - // RemoteTime sets the finished file's mtime from the server's Last-Modified - // header (-R / --remote-time). - RemoteTime bool - // HTTPUser / HTTPPasswd enable HTTP Basic auth (--http-user/passwd). - HTTPUser string - HTTPPasswd string - // LowestSpeedLimit aborts a download whose speed stays at or below this many - // bytes/sec for a sustained window (--lowest-speed-limit). 0 = off. - LowestSpeedLimit int64 + Dir string + Out string + Split int // --split: total connections across all mirrors + MinSplit int64 // smallest piece to split into; 0 = minSplitSize default (internal, no flag) + Tries int // --max-tries: 0 = unlimited + Timeout time.Duration + FileAlloc string // none | prealloc | trunc | falloc + Continue bool + AllowOverwrite bool + AutoRename bool + Headers []string // --header: extra request headers + OverallLimiter *rate.Limiter // shared across all downloads, may be nil // Checksum is a "=" spec (e.g. "sha-256=ab…"); when set, // the finished file is hashed and compared, failing with ErrChecksum on a // mismatch (--checksum). Empty means no verification. @@ -106,11 +73,6 @@ type Config struct { // DryRun probes the resource — proving it exists and its size — but downloads // nothing, then reports success (--dry-run). DryRun bool - // CACert is a path to a PEM bundle of CA certificates used to verify HTTPS - // servers (--ca-certificate). Empty uses the system roots. - CACert string - // DisableIPv6 forces IPv4-only dialing (--disable-ipv6). - DisableIPv6 bool } // Download implements download.Download for one HTTP(S) file, fetched from one @@ -119,11 +81,10 @@ type Config struct { // connections. type Download struct { uris []string // mirror list; uris[0] is the primary (naming + resume key) - maxConns int // segment-worker cap = min(split, len(uris)*max-connection-per-server) + maxConns int // segment-worker cap = min(split, len(uris)) cfg Config client *http.Client limit []*rate.Limiter - jar *cookiejar.Jar // non-nil when cookies are in use, for save-cookies // name and out are resolved in Run after the probe response is known (the // final post-redirect URL and any Content-Disposition can change them). The @@ -135,11 +96,6 @@ type Download struct { out atomic.Pointer[string] initErr error - // mu guards seenURLs, which records every URL we sent a request to so - // SaveCookies can ask the jar for the cookies of each contacted host. - mu sync.Mutex - seenURLs []*url.URL - // live counters, read by Stat from any goroutine total int64 // -1 until known completed int64 @@ -147,21 +103,6 @@ type Download struct { conns int32 // active connections } -// loadCAPool reads a PEM bundle of CA certificates for verifying HTTPS servers -// (--ca-certificate). An unreadable file or one containing no -// certificates is an error rather than a silent fall-back to the system roots. -func loadCAPool(path string) (*x509.CertPool, error) { - pem, err := os.ReadFile(path) - if err != nil { - return nil, err - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(pem) { - return nil, fmt.Errorf("ca-certificate %s: no certificates found", path) - } - return pool, nil -} - // primary is the canonical URL — used for naming and the resume sidecar so they // stay stable regardless of which mirror served the bytes. func (d *Download) primary() string { return d.uris[0] } @@ -174,96 +115,27 @@ func (d *Download) mirror(n int) string { return d.uris[n%len(d.uris)] } // New builds an HTTP download for uris, which must all serve identical content // (mirrors). uris[0] is the primary. func New(uris []string, cfg Config) *Download { - // --connect-timeout bounds only connection establishment; once data flows - // --timeout (the idle guard) takes over. The dial and the TLS handshake each - // get the full connectTimeout (Go applies it per phase), so an HTTPS connect - // can take up to ~2x connectTimeout in the worst case — there is no single - // shared connect budget. - connectTimeout := cfg.ConnectTimeout - if connectTimeout <= 0 { - connectTimeout = cfg.Timeout - } var initErr error - - // --ca-certificate: verify HTTPS servers against a custom CA bundle instead of - // the system roots. A bad bundle becomes an initErr so Run fails loudly. - tlsCfg := &tls.Config{InsecureSkipVerify: !cfg.CheckCert} - if cfg.CACert != "" { - if pool, err := loadCAPool(cfg.CACert); err != nil { - initErr = err - } else { - tlsCfg.RootCAs = pool - } + if cfg.MinSplit <= 0 { + cfg.MinSplit = minSplitSize } - // --disable-ipv6: force IPv4 by dialing "tcp4"; a plain "tcp" would let the - // resolver hand back a v6 address. - dialer := &net.Dialer{Timeout: connectTimeout} - dialContext := dialer.DialContext - if cfg.DisableIPv6 { - dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - if network == "tcp" { - network = "tcp4" - } - return dialer.DialContext(ctx, network, addr) - } - } - - // MaxConnsPerHost is the per-host cap (--max-connection-per-server): with - // mirrors the total worker count can exceed it, but no single host does. - perHost := cfg.MaxConnPerServer - if perHost < 1 { - perHost = 1 - } + // One connect budget: --timeout bounds both establishing the connection and + // idle stalls once data flows. Proxies still come from the standard + // HTTP_PROXY/HTTPS_PROXY environment, and TLS uses the system roots. tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, - MaxConnsPerHost: perHost, - MaxIdleConnsPerHost: perHost, + MaxConnsPerHost: cfg.Split, + MaxIdleConnsPerHost: cfg.Split, ResponseHeaderTimeout: cfg.Timeout, - TLSHandshakeTimeout: connectTimeout, - DialContext: dialContext, - TLSClientConfig: tlsCfg, - } - if cfg.Proxy != "" { - pu, err := url.Parse(cfg.Proxy) - switch { - case err != nil: - // A malformed proxy must fail loudly, not silently fall back to the - // environment proxy (or a direct connection) and leak the real IP. - if initErr == nil { - initErr = fmt.Errorf("proxy %q: %w", cfg.Proxy, err) - } - case pu.Host == "": - // "host:port" with no scheme parses to an opaque URL with no Host, which - // http.ProxyURL would route nowhere; require an explicit scheme. - if initErr == nil { - initErr = fmt.Errorf("proxy %q: missing scheme (use scheme://host:port)", cfg.Proxy) - } - default: - tr.Proxy = http.ProxyURL(pu) - } - } - - // Seed the cookie jar from --load-cookies; a load error becomes an initErr so - // Run surfaces it rather than silently downloading uncredentialed. - var jar *cookiejar.Jar - if cfg.LoadCookies != "" || cfg.SaveCookies != "" { - if j, err := loadCookieJar(cfg.LoadCookies); err != nil { - if initErr == nil { - initErr = err - } - } else { - jar = j - } + TLSHandshakeTimeout: cfg.Timeout, + DialContext: (&net.Dialer{Timeout: cfg.Timeout}).DialContext, } var limit []*rate.Limiter if cfg.OverallLimiter != nil { limit = append(limit, cfg.OverallLimiter) } - if cfg.Limit > 0 { - limit = append(limit, rate.NewLimiter(rate.Limit(cfg.Limit), download.LimiterBurst(cfg.Limit))) - } // Seed a best-effort name from -o or the primary URL. The real name is // finalised in Run after probe(), where Content-Disposition and the @@ -283,31 +155,19 @@ func New(uris []string, cfg Config) *Download { if len(uris) == 0 && initErr == nil { initErr = errors.New("no URL to download") } - // Worker count: --split connections are spread across the mirrors, capped - // at --max-connection-per-server per host, so the ceiling is - // min(split, M*max-conn-per-server). One mirror reduces to min(split, x). + // Worker count is --split: the file is cut into that many byte-range segments, + // each its own connection, whether served by one host or spread across mirrors. maxConns := cfg.Split - if m := max(1, len(uris)) * perHost; maxConns > m { - maxConns = m - } if maxConns < 1 { maxConns = 1 } - // Only attach the jar when one was built: a typed-nil *cookiejar.Jar stored - // in the http.Client.Jar interface is non-nil and would panic on use. - client := &http.Client{Transport: tr} - if jar != nil { - client.Jar = jar - } - d := &Download{ uris: uris, maxConns: maxConns, cfg: cfg, - client: client, + client: &http.Client{Transport: tr}, limit: limit, - jar: jar, total: -1, initErr: initErr, } @@ -318,7 +178,8 @@ func New(uris []string, cfg Config) *Download { func (d *Download) Name() string { return d.loadName() } -// Path returns the resolved output file path (used by --follow-torrent). +// Path returns the resolved output file path, so a fetched .torrent can be +// followed into a BitTorrent download of its content. func (d *Download) Path() string { return d.loadOut() } // name and out are read by Stat from the reporter goroutine while Run resolves @@ -359,13 +220,12 @@ func (d *Download) setStatus(s download.Status) { // probeResult carries everything probe() learned, so name finalisation can run // after the request (the final URL and Content-Disposition are only known then). type probeResult struct { - total int64 - ranged bool - etag string - lastmod string - cdisp string // raw Content-Disposition header - finalURL *url.URL - notModified bool // server answered 304 to a conditional GET + total int64 + ranged bool + etag string + lastmod string + cdisp string // raw Content-Disposition header + finalURL *url.URL } // Run probes the resource, finalises the output name, decides between a single @@ -388,12 +248,6 @@ func (d *Download) Run(ctx context.Context) (err error) { if err != nil { return err } - // --conditional-get: the server said our local copy is current, so there is - // nothing to download. Report success without touching the file. - if pr.notModified { - d.setStatus(download.Complete) - return nil - } atomic.StoreInt64(&d.total, pr.total) // Finalise name/out now that the post-redirect URL and Content-Disposition @@ -421,25 +275,13 @@ func (d *Download) Run(ctx context.Context) (err error) { return nil } - // --lowest-speed-limit: cancel the transfer if its speed stays at or below - // the limit past a startup grace. The watcher cancels xferCtx and sets a - // flag so we can return the distinct "too slow" error rather than a bare - // cancellation. out := d.loadOut() - xferCtx, stopGuard, tooSlow := d.speedGuard(ctx) if !pr.ranged || pr.total <= 0 { - err = d.single(xferCtx, out) + err = d.single(ctx, out) } else { - err = d.segmented(xferCtx, out, pr.total, pr.etag, pr.lastmod) + err = d.segmented(ctx, out, pr.total, pr.etag, pr.lastmod) } - stopGuard() if err != nil { - // Only translate to ErrTooSlow when the transfer actually ended in the - // guard's cancellation; a write error or mirror exhaustion that merely - // raced the guard firing keeps its own (real) cause and exit code. - if tooSlow() && errors.Is(err, context.Canceled) { - return fmt.Errorf("%s: %w (<= %d bytes/sec)", d.primary(), ErrTooSlow, d.cfg.LowestSpeedLimit) - } return err } // --checksum: verify the finished file before declaring success, so a @@ -449,12 +291,6 @@ func (d *Download) Run(ctx context.Context) (err error) { return err } } - // -R / --remote-time: stamp the finished file with the server's mtime. - if d.cfg.RemoteTime && pr.lastmod != "" { - if t, err := http.ParseTime(pr.lastmod); err == nil { - os.Chtimes(out, t, t) - } - } d.setStatus(download.Complete) return nil } @@ -507,20 +343,6 @@ func (d *Download) resolveName(pr probeResult) error { return nil } -// backoff waits RetryWait between attempts, but never after the final attempt, -// and returns ctx.Err() if cancelled while waiting (item [20]). -func backoff(ctx context.Context, wait time.Duration, tries, attempt int) error { - if wait <= 0 || (tries != 0 && attempt >= tries-1) { - return nil - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(wait): - return nil - } -} - // retriesExhausted wraps the final cause once the retry budget is spent, so the // caller — and the exit-code mapping in main — can still see whether it was DNS, // a refused connection, or a timeout, rather than an opaque "too many retries". @@ -538,12 +360,11 @@ func retriesExhausted(what string, tries int, cause error) error { } // withRetries runs attempt up to d.cfg.Tries times (0 = unlimited), the shared -// tries/backoff/fatal policy that probe, the single stream, and each segment all -// use so one transient DNS/connect/5xx does not fail the whole download (item -// [18]). It returns immediately on success (nil), on context cancellation, or on -// a fatal{} error (retrying cannot help), sleeps RetryWait between attempts, and -// wraps the final cause with retriesExhausted(what, ...) once the budget is -// spent so the underlying cause stays inspectable. +// tries/fatal policy that probe, the single stream, and each segment all use so +// one transient DNS/connect/5xx does not fail the whole download (item [18]). It +// returns immediately on success (nil), on context cancellation, or on a fatal{} +// error (retrying cannot help), and wraps the final cause with retriesExhausted +// once the budget is spent so the underlying cause stays inspectable. func (d *Download) withRetries(ctx context.Context, what string, attempt func() error) error { tries := d.cfg.Tries var lastErr error @@ -560,9 +381,6 @@ func (d *Download) withRetries(ctx context.Context, what string, attempt func() if errors.As(err, &fe) { return err // permanent: retrying cannot help } - if err := backoff(ctx, d.cfg.RetryWait, tries, n); err != nil { - return err - } } return retriesExhausted(what, tries, lastErr) } @@ -614,15 +432,6 @@ func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) { if err != nil { return probeResult{}, err } - // --conditional-get: ask the server to skip the transfer when our local file - // is already current. Only done when there is no .got control file (an - // in-progress resume must not be short-circuited), using the local file's - // mtime for If-Modified-Since. - if out := d.loadOut(); d.cfg.ConditionalGet && !fileExists(controlPath(out)) { - if fi, err := os.Stat(out); err == nil { - req.Header.Set("If-Modified-Since", fi.ModTime().UTC().Format(http.TimeFormat)) - } - } resp, err := d.client.Do(req) if err != nil { return probeResult{}, err @@ -639,11 +448,6 @@ func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) { pr.finalURL = resp.Request.URL } - if resp.StatusCode == http.StatusNotModified { - pr.notModified = true - return pr, nil - } - switch resp.StatusCode { case http.StatusPartialContent: // Content-Range: bytes 0-0/12345 @@ -864,74 +668,6 @@ func (d *Download) idleGuard(body io.Reader, cancel context.CancelFunc) (io.Read return ir, func() { close(stop) } } -// speedStartupGrace is how long a download may stay slow before the -// lowest-speed-limit watchdog starts checking it: a startup idle time of -// 10 seconds. -const speedStartupGrace = 10 * time.Second - -// belowSpeed reports whether the bytes transferred between two cumulative -// d.completed samples (prev -> now over one window) put the download at or -// below limit. A negative delta is not a slow second: when a resumed Range is -// answered by a 200 the transfer restarts from offset 0, so d.completed jumps -// backward on that retry. Treating that backward jump as "below limit" would -// false-abort a healthy download, so a reset (delta < 0) reports false; the -// caller has already advanced its baseline, so the next full window measures the -// real speed. Only a genuine non-negative delta at or under the limit aborts. -func belowSpeed(prev, now, limit int64) bool { - delta := now - prev - if delta < 0 { - return false // counter reset, not a slow window - } - return delta <= limit -} - -// speedGuard implements --lowest-speed-limit. It returns a child context that -// the watcher cancels (setting a flag) once the measured download speed has -// stayed at or below the limit past the startup grace, plus a stop func and a -// predicate reporting whether the watcher fired. When the limit is 0 it is a -// no-op passthrough. -// -// Speed is sampled from d.completed over a one-second window: a single slow -// second after the grace aborts the download as too slow. -func (d *Download) speedGuard(ctx context.Context) (context.Context, func(), func() bool) { - if d.cfg.LowestSpeedLimit <= 0 { - return ctx, func() {}, func() bool { return false } - } - child, cancel := context.WithCancel(ctx) - var fired atomic.Bool - done := make(chan struct{}) - go func() { - defer close(done) - start := time.Now() - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - last := atomic.LoadInt64(&d.completed) - for { - select { - case <-child.Done(): - return - case <-ticker.C: - now := atomic.LoadInt64(&d.completed) - prev := last - last = now - if time.Since(start) < speedStartupGrace { - continue - } - if belowSpeed(prev, now, d.cfg.LowestSpeedLimit) { - fired.Store(true) - cancel() - return - } - } - } - }() - stop := func() { - cancel() - <-done - } - return child, stop, func() bool { return fired.Load() } -} - // fatal marks an error that retrying cannot fix. Unwrap exposes the wrapped // error so errors.Is/As traverse a fatal{} — main's exit-code mapping relies on // errors.Is(err, ErrNotFound) seeing through it (CONTRACT). @@ -1134,13 +870,13 @@ func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDi } } +// saveInterval is how often the .got control file is rewritten so a crash can +// resume; on a clean finish it is removed. +const saveInterval = 60 * time.Second + func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, segs []seg, done chan<- struct{}) { defer close(done) - interval := d.cfg.AutoSaveInterval - if interval <= 0 { - interval = 60 * time.Second // --auto-save-interval default - } - t := time.NewTicker(interval) + t := time.NewTicker(saveInterval) defer t.Stop() for { select { @@ -1171,13 +907,9 @@ func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Req if err != nil { return nil, err } - req.Header.Set("User-Agent", d.cfg.UserAgent) - if d.cfg.Referer != "" { - req.Header.Set("Referer", d.cfg.Referer) - } - if d.cfg.HTTPUser != "" { - req.SetBasicAuth(d.cfg.HTTPUser, d.cfg.HTTPPasswd) - } + req.Header.Set("User-Agent", userAgent) + // --header sets arbitrary request headers, applied last so they can override + // the User-Agent (or add a Referer, Authorization, Cookie, ...) when needed. for _, h := range d.cfg.Headers { if k, v, ok := strings.Cut(h, ":"); ok { req.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v)) @@ -1186,33 +918,9 @@ func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Req if rangeHdr != "" { req.Header.Set("Range", rangeHdr) } - if d.jar != nil { - d.recordURL(req.URL) - } return req, nil } -// recordURL remembers a contacted URL so SaveCookies can later read back the -// jar's cookies for that host. Duplicates are tolerated; saveCookieJar dedups. -func (d *Download) recordURL(u *url.URL) { - d.mu.Lock() - d.seenURLs = append(d.seenURLs, u) - d.mu.Unlock() -} - -// SaveCookies writes the client's cookie jar back to the --save-cookies file, -// if one was configured. It is a no-op when cookies are not in use. main calls -// it once per HTTP download on exit (cookies are saved at session end). -func (d *Download) SaveCookies() error { - if d.cfg.SaveCookies == "" || d.jar == nil { - return nil - } - d.mu.Lock() - urls := append([]*url.URL{}, d.seenURLs...) - d.mu.Unlock() - return saveCookieJar(d.jar, urls, d.cfg.SaveCookies) -} - // seedSegmentsFromPrefix marks the first prefix bytes of the file as already // downloaded, segment by segment. A foreign partial file has no per-segment // metadata, so the only thing we can trust is that bytes [0,prefix) are present; diff --git a/httpdl/mirror_test.go b/httpdl/mirror_test.go index 1882e51..619ea2e 100644 --- a/httpdl/mirror_test.go +++ b/httpdl/mirror_test.go @@ -30,25 +30,24 @@ func TestMirrorPick(t *testing.T) { } } -// TestMaxConns checks the connection ceiling min(split, mirrors*x), floored -// at 1, since that decides how many segment workers fan out. +// TestMaxConns checks the connection ceiling: --split segment workers, floored +// at 1. Mirrors are alternate sources, not extra connections, so they do not +// raise the count. func TestMaxConns(t *testing.T) { - cases := []struct{ split, x, mirrors, want int }{ - {5, 1, 1, 1}, // default -x1 -s5: one connection - {5, 16, 1, 5}, // split is the ceiling on one server - {16, 16, 1, 16}, - {5, 1, 3, 3}, // 3 mirrors * 1 per host < split - {16, 2, 3, 6}, // 3 mirrors * 2 per host < split - {0, 1, 1, 1}, // floor at 1 + cases := []struct{ split, mirrors, want int }{ + {5, 1, 5}, // split is the connection count on one server + {16, 1, 16}, + {5, 3, 5}, // 3 mirrors, still split connections + {0, 1, 1}, // floor at 1 } for _, c := range cases { uris := make([]string, c.mirrors) for i := range uris { uris[i] = "http://h/x" } - d := New(uris, Config{Split: c.split, MaxConnPerServer: c.x}) + d := New(uris, Config{Split: c.split}) if d.maxConns != c.want { - t.Errorf("split=%d x=%d mirrors=%d: maxConns=%d, want %d", c.split, c.x, c.mirrors, d.maxConns, c.want) + t.Errorf("split=%d mirrors=%d: maxConns=%d, want %d", c.split, c.mirrors, d.maxConns, c.want) } } } @@ -77,8 +76,7 @@ func TestMirrorFallback(t *testing.T) { // Primary is healthy (so the probe anchors size/validators), bad is a mirror // that 404s every segment it's handed. d := New([]string{good.URL + "/f.bin", bad.URL + "/f.bin"}, Config{ - Dir: dir, Out: "f.bin", Split: 4, MaxConnPerServer: 2, MinSplit: 1, - Tries: 1, UserAgent: "got-test", + Dir: dir, Out: "f.bin", Split: 4, MinSplit: 1, Tries: 1, }) if err := d.Run(context.Background()); err != nil { t.Fatalf("Run with a 404 mirror should still complete via the healthy one: %v", err) @@ -121,8 +119,7 @@ func TestMirrorDivergentRejected(t *testing.T) { dir := t.TempDir() out := filepath.Join(dir, "f.bin") d := New([]string{primary.URL + "/f.bin", div.URL + "/f.bin"}, Config{ - Dir: dir, Out: "f.bin", Split: 4, MaxConnPerServer: 2, MinSplit: 1, - Tries: 1, UserAgent: "got-test", + Dir: dir, Out: "f.bin", Split: 4, MinSplit: 1, Tries: 1, }) if err := d.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) diff --git a/httpdl/single_test.go b/httpdl/single_test.go index 8d38bc7..6fede36 100644 --- a/httpdl/single_test.go +++ b/httpdl/single_test.go @@ -43,11 +43,9 @@ func TestSingleShortBody(t *testing.T) { out := filepath.Join(dir, "file.bin") d := New([]string{srv.URL + "/file.bin"}, Config{ - Dir: dir, - Out: "file.bin", - MaxConnPerServer: 1, - Tries: 1, // a single attempt so the retryable short read surfaces, not loops - UserAgent: "got-test", + Dir: dir, + Out: "file.bin", + Tries: 1, // a single attempt so the retryable short read surfaces, not loops }) err := d.single(context.Background(), out) @@ -75,11 +73,9 @@ func TestSingleCompleteFullBody(t *testing.T) { out := filepath.Join(dir, "file.bin") d := New([]string{srv.URL + "/file.bin"}, Config{ - Dir: dir, - Out: "file.bin", - MaxConnPerServer: 1, - Tries: 1, - UserAgent: "got-test", + Dir: dir, + Out: "file.bin", + Tries: 1, }) if err := d.single(context.Background(), out); err != nil { diff --git a/httpdl/speed_test.go b/httpdl/speed_test.go deleted file mode 100644 index fb0f172..0000000 --- a/httpdl/speed_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package httpdl - -import "testing" - -// TestBelowSpeed covers the speedGuard decision, including the counter-reset -// case (item [1]): d.completed can jump backward when singleOnce rewrites it -// with an absolute StoreInt64 and a resumed Range is answered by a 200 that -// resets the offset to 0. A backward jump must NOT be read as a slow window, or -// a healthy download would be false-aborted. -func TestBelowSpeed(t *testing.T) { - const limit = 1000 - cases := []struct { - name string - prev, now int64 - want bool - }{ - {"fast", 0, 5000, false}, - {"exactly at limit aborts", 1000, 2000, true}, - {"just under limit aborts", 1000, 1999, true}, - {"just over limit ok", 1000, 2001, false}, - {"stalled aborts", 4242, 4242, true}, - {"counter reset is not slow", 1_000_000, 0, false}, - {"small backward jump is not slow", 5000, 4500, false}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - if got := belowSpeed(c.prev, c.now, limit); got != c.want { - t.Errorf("belowSpeed(%d, %d, %d) = %v; want %v", c.prev, c.now, limit, got, c.want) - } - }) - } -} diff --git a/main.go b/main.go index 63e2d7b..8fe12eb 100644 --- a/main.go +++ b/main.go @@ -88,12 +88,6 @@ func run(args []string) int { sessionFile := opts.Str("save-session") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // --stop: cancel everything after N seconds, exactly like a Ctrl-C, so an - // unattended run can't hang forever (0 = off). - if stop := opts.Int("stop"); stop > 0 { - t := time.AfterFunc(time.Duration(stop)*time.Second, cancel) - defer t.Stop() - } var jobsRef atomic.Pointer[[]job] // publish snapshots the slice header (j is a by-value copy) and stores a // pointer to it, so the signal handler can read the current jobs lock-free @@ -106,7 +100,7 @@ func run(args []string) int { uiCtx, uiCancel := context.WithCancel(context.Background()) uiDone := make(chan struct{}) if !opts.Bool("quiet") && !opts.Bool("dry-run") { - rep := progress.New(eng.Snapshot, opts.Bool("human-readable")) + rep := progress.New(eng.Snapshot, true) go func() { rep.Run(uiCtx) close(uiDone) @@ -118,10 +112,9 @@ func run(args []string) int { publish(jobs) results := runJobs(ctx, eng, jobs) - // --follow-torrent: a .torrent fetched over HTTP becomes a BitTorrent - // download of its content (on by default). Done as a second pass so the - // engine and scheduler stay simple. - if opts.Bool("follow-torrent") && ctx.Err() == nil && !opts.Bool("dry-run") { + // A .torrent fetched over HTTP becomes a BitTorrent download of its content, + // done as a second pass so the engine and scheduler stay simple. + if ctx.Err() == nil && !opts.Bool("dry-run") { if follow := followUps(jobs, btClient, btOptions(opts)); len(follow) > 0 { // The engine numbers each Result by its position in the slice it was // passed, so the follow pass restarts at Index 0. Offset those by the @@ -141,8 +134,6 @@ func run(args []string) int { uiCancel() <-uiDone // wait for the renderer's final frame before printing the OK/FAIL lines - saveCookies(jobs) - if sessionFile != "" { if err := saveSession(sessionFile, jobs); err != nil { fmt.Fprintln(os.Stderr, "got:", err) @@ -208,11 +199,11 @@ func seenInfoHash(seen map[metainfo.Hash]string, source string, isFile bool) (fi // dedupeTorrents finds torrent/magnet sources that repeat an earlier source's // infohash, returning each duplicate URI mapped to the source it repeats. -func dedupeTorrents(uris []string, follow bool) map[string]string { +func dedupeTorrents(uris []string) map[string]string { seen := map[metainfo.Hash]string{} dup := map[string]string{} for _, u := range uris { - k := uriKind(u, follow) + k := uriKind(u) if k != kindMagnet && k != kindTorrent { continue } @@ -230,7 +221,7 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) // finish would Drop it out from under the other. The duplicate is reported as // a failed download (duplicate infohash, exit 12); the loop below does this, // so the duplicate never obtains or Drops a shared torrent. - dups := dedupeTorrents(flat, opts.Bool("follow-torrent")) + dups := dedupeTorrents(flat) overallDL := makeLimiter(opts.Size("max-overall-download-limit")) overallUL := makeLimiter(opts.Size("max-overall-upload-limit")) @@ -248,7 +239,7 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) // it (e.g. the listen port is taken) disables BitTorrent but still lets HTTP // downloads run; those torrents then report the error. var client *torrent.Client - if needsBT(opts, flat) { + if needsBT(flat) { c, err := bt.NewClient(btClientConfig(opts, overallDL, overallUL)) if err != nil { fmt.Fprintln(os.Stderr, "got: bittorrent disabled:", err) @@ -269,7 +260,7 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) continue } var dl download.Download - switch uriKind(u, opts.Bool("follow-torrent")) { + switch uriKind(u) { case kindMagnet: dl = bt.New(client, u, false, bo) case kindHTTP, kindFollow: @@ -288,16 +279,15 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) } // uriKind classifies a URI by scheme into the taxonomy build() and needsBT() -// both switch on. Scheme wins over suffix: an http URL ending in .torrent is a -// file to fetch over HTTP, and with --follow-torrent it becomes a torrent only -// after the fetch (kindFollow), so it still downloads over HTTP first. -func uriKind(u string, follow bool) kind { +// both switch on. Scheme wins over suffix: an http URL ending in .torrent is +// fetched over HTTP first, then followed as a torrent (kindFollow). +func uriKind(u string) kind { lu := strings.ToLower(u) switch { case strings.HasPrefix(u, "magnet:"): return kindMagnet case strings.HasPrefix(u, "http://"), strings.HasPrefix(u, "https://"): - if follow && strings.HasSuffix(lu, ".torrent") { + if strings.HasSuffix(lu, ".torrent") { return kindFollow } return kindHTTP @@ -319,10 +309,9 @@ const ( // needsBT reports whether the run will involve any torrent, so we only start a // client when one is actually needed. -func needsBT(opts *cli.Options, uris []string) bool { - follow := opts.Bool("follow-torrent") +func needsBT(uris []string) bool { for _, u := range uris { - switch uriKind(u, follow) { + switch uriKind(u) { case kindMagnet, kindTorrent, kindFollow: return true } @@ -387,19 +376,6 @@ func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job { return out } -// saveCookies asks each HTTP download to write its cookie jar back to the -// --save-cookies file, persisting cookies at session end. A download with no -// --save-cookies configured is a no-op. -func saveCookies(jobs []job) { - for _, j := range jobs { - if h, ok := j.dl.(*httpdl.Download); ok { - if err := h.SaveCookies(); err != nil { - fmt.Fprintln(os.Stderr, "got:", err) - } - } - } -} - // target is one download: a single torrent/magnet/URL, or several HTTP mirror // URLs that serve identical bytes (uris[0] is the primary). type target struct{ uris []string } @@ -414,16 +390,14 @@ func allURIs(ts []target) []string { return u } -// gatherURIs collects download targets from the command line, --torrent-file, -// and --input-file. Plain command-line http/https URIs are grouped into ONE -// mirrored download by default (every command-line URI names the same file); -// -Z/--force-sequential makes each its own download. Torrents, magnets, and -// .torrent URLs (fetched over HTTP then followed) are always one target each, -// since distinct torrents are not mirrors of one another; --input-file groups -// TAB-separated URIs per line. +// gatherURIs collects download targets from the command line and --input-file. +// Plain command-line http/https URIs are grouped into ONE mirrored download by +// default (every command-line URI names the same file); -Z/--force-sequential +// makes each its own download. Torrents, magnets, and .torrent URLs (fetched +// over HTTP then followed) are always one target each, since distinct torrents +// are not mirrors of one another; --input-file groups TAB-separated URIs per line. func gatherURIs(opts *cli.Options, cmdURIs []string) []target { var ts []target - follow := opts.Bool("follow-torrent") seq := opts.Bool("force-sequential") var httpGroup []string for _, u := range cmdURIs { @@ -431,7 +405,7 @@ func gatherURIs(opts *cli.Options, cmdURIs []string) []target { // for the same file); -Z opts out. A .torrent URL (kindFollow), like a // torrent or magnet, is its own download — distinct torrents are never // mirrors of one another. - if uriKind(u, follow) == kindHTTP && !seq { + if uriKind(u) == kindHTTP && !seq { httpGroup = append(httpGroup, u) } else { ts = append(ts, target{uris: []string{u}}) @@ -440,9 +414,6 @@ func gatherURIs(opts *cli.Options, cmdURIs []string) []target { if len(httpGroup) > 0 { ts = append(ts, target{uris: httpGroup}) } - if tf := opts.Str("torrent-file"); tf != "" { - ts = append(ts, target{uris: []string{tf}}) - } if in := opts.Str("input-file"); in != "" { ts = append(ts, readInputFile(in)...) } @@ -497,49 +468,20 @@ func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl. if single { out = opts.Str("out") } - // max-connection-per-server caps connections per host. Its default is 1, - // which on its own would clamp --split to a single connection, so a bare - // `got URL` would download single-threaded. Honor the literal default of 1 - // only when the user actually set -x; otherwise let --split drive per-host - // parallelism, capped at the -x ceiling of 16. -x explicitly set keeps the - // exact min(split, mirrors*x) behavior. - perHost := opts.Int("max-connection-per-server") - if !opts.IsSet("max-connection-per-server") { - perHost = min(opts.Int("split"), 16) - } return httpdl.Config{ - Dir: opts.Str("dir"), - Out: out, - Split: opts.Int("split"), - MaxConnPerServer: perHost, - MinSplit: opts.Size("min-split-size"), - Tries: opts.Int("max-tries"), - Timeout: time.Duration(opts.Int("timeout")) * time.Second, - FileAlloc: opts.Str("file-allocation"), - Continue: opts.Bool("continue"), - AllowOverwrite: opts.Bool("allow-overwrite"), - AutoRename: opts.Bool("auto-file-renaming"), - Headers: opts.List("header"), - UserAgent: opts.Str("user-agent"), - Referer: opts.Str("referer"), - Proxy: opts.Str("all-proxy"), - CheckCert: opts.Bool("check-certificate"), - Limit: opts.Size("max-download-limit"), - OverallLimiter: overallDL, - RetryWait: time.Duration(opts.Int("retry-wait")) * time.Second, - AutoSaveInterval: time.Duration(opts.Int("auto-save-interval")) * time.Second, - ConnectTimeout: time.Duration(opts.Int("connect-timeout")) * time.Second, - LoadCookies: opts.Str("load-cookies"), - SaveCookies: opts.Str("save-cookies"), - ConditionalGet: opts.Bool("conditional-get"), - RemoteTime: opts.Bool("remote-time"), - HTTPUser: opts.Str("http-user"), - HTTPPasswd: opts.Str("http-passwd"), - LowestSpeedLimit: opts.Size("lowest-speed-limit"), - Checksum: opts.Str("checksum"), - DryRun: opts.Bool("dry-run"), - CACert: opts.Str("ca-certificate"), - DisableIPv6: opts.Bool("disable-ipv6"), + Dir: opts.Str("dir"), + Out: out, + Split: opts.Int("split"), + Tries: opts.Int("max-tries"), + Timeout: time.Duration(opts.Int("timeout")) * time.Second, + FileAlloc: opts.Str("file-allocation"), + Continue: opts.Bool("continue"), + AllowOverwrite: opts.Bool("allow-overwrite"), + AutoRename: opts.Bool("auto-file-renaming"), + Headers: opts.List("header"), + OverallLimiter: overallDL, + Checksum: opts.Str("checksum"), + DryRun: opts.Bool("dry-run"), } } @@ -549,13 +491,8 @@ func btClientConfig(opts *cli.Options, overallDL, overallUL *rate.Limiter) bt.Cl Dir: opts.Str("dir"), ListenPortSpec: opts.Str("listen-port"), DHT: opts.Bool("enable-dht"), - MaxPeers: opts.Int("bt-max-peers"), OverallDown: overallDL, OverallUp: overallUL, - DownLimit: opts.Size("max-download-limit"), - UpLimit: opts.Size("max-upload-limit"), - UserAgent: opts.Str("user-agent"), - DisableIPv6: opts.Bool("disable-ipv6"), } } @@ -565,7 +502,6 @@ func btOptions(opts *cli.Options) bt.Options { SeedTimeSet: opts.IsSet("seed-time"), SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)), SeedRatio: opts.Float("seed-ratio"), - StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second, SelectFiles: parseSelect(opts.Str("select-file")), CheckIntegrity: opts.Bool("check-integrity"), DryRun: opts.Bool("dry-run"), @@ -686,8 +622,6 @@ func exitCode(err error) int { return 6 case errors.Is(err, httpdl.ErrNotFound): return 3 - case errors.Is(err, httpdl.ErrTooSlow): - return 5 case errors.Is(err, httpdl.ErrChecksum): return 32 case errors.Is(err, errDuplicate): diff --git a/main_test.go b/main_test.go index ce656bd..73cc113 100644 --- a/main_test.go +++ b/main_test.go @@ -78,7 +78,6 @@ func TestExitCode(t *testing.T) { {"refused typed", refused, 6}, {"refused wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", refused), 6}, {"not found", fmt.Errorf("%w: http://x/y: 404 Not Found", httpdl.ErrNotFound), 3}, - {"too slow", fmt.Errorf("http://x: %w (<= %d bytes/sec)", httpdl.ErrTooSlow, 1024), 5}, {"checksum", fmt.Errorf("/tmp/x: %w (want a, got b)", httpdl.ErrChecksum), 32}, {"duplicate", fmt.Errorf("%w: same torrent as y", errDuplicate), 12}, {"unknown", errors.New("disk full"), 1}, @@ -104,7 +103,7 @@ func TestDedupeTorrents(t *testing.T) { c := "magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=other" web := "http://example.com/file.iso" - dup := dedupeTorrents([]string{a, web, b, c}, true) + dup := dedupeTorrents([]string{a, web, b, c}) // b is the only duplicate (of a); a, web and c are not duplicates. if len(dup) != 1 || dup[b] != a { @@ -145,9 +144,8 @@ func TestGatherURIsGrouping(t *testing.T) { } // TestHTTPConnDefault locks the parallelism default: a bare `got URL` must use -// several connections, not one. With -x unset, --split drives the per-host -// connection count (capped at the -x ceiling of 16); an explicit -x is honored -// verbatim, including -x1 for a single connection. +// several connections, not one. --split is the single speed dial — it is the +// connection count directly, defaulting to 5. func TestHTTPConnDefault(t *testing.T) { cfg := func(args ...string) httpdl.Config { res, err := cli.Parse(append([]string{"--no-conf"}, args...)) @@ -163,13 +161,11 @@ func TestHTTPConnDefault(t *testing.T) { }{ {"bare URL uses split default", []string{"http://a/x"}, 5}, {"-s16 means 16", []string{"-s16", "http://a/x"}, 16}, - {"-s100 capped at 16 per host", []string{"-s100", "http://a/x"}, 16}, - {"explicit -x4 honored", []string{"-x4", "http://a/x"}, 4}, - {"explicit -x1 stays single", []string{"-x1", "http://a/x"}, 1}, + {"-s1 stays single", []string{"-s1", "http://a/x"}, 1}, } for _, tc := range tests { - if c := cfg(tc.args...); c.MaxConnPerServer != tc.want { - t.Errorf("%s: MaxConnPerServer = %d, want %d", tc.name, c.MaxConnPerServer, tc.want) + if c := cfg(tc.args...); c.Split != tc.want { + t.Errorf("%s: Split = %d, want %d", tc.name, c.Split, tc.want) } } }