diff --git a/README.md b/README.md index b80e16c..aec3c7e 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,17 @@ Needs Go 1.25+. ## Usage ```sh -# HTTP download with 16 connections -got -x16 -s16 https://example.com/big.iso +# HTTP download: parallel out of the box (5 connections); -s turns it up +got https://example.com/big.iso +got -s16 https://example.com/big.iso # resume an interrupted download got -c https://example.com/big.iso # one file from several mirrors (connections spread across servers, -# a dead mirror falls over); -Z downloads them as separate files instead -got -x4 https://a.example/big.iso https://b.example/big.iso +# a dead mirror falls over); -Z downloads them as separate files instead. +# -x caps connections per server when mirroring. +got -s16 -x4 https://a.example/big.iso https://b.example/big.iso # a torrent or magnet; seed for 30 minutes after finishing got --seed-time=30 ubuntu.torrent @@ -42,9 +44,9 @@ Run `got --help` (or `--help=all`) for every option. | `-d, --dir` | output directory | `.` | | `-o, --out` | output filename | from URL | | `-c, --continue` | resume a partial download | false | -| `-x, --max-connection-per-server` | connections to one server (1–16) | 1 | -| `-s, --split` | split a download into N connections | 5 | -| `-j, --max-concurrent-downloads` | downloads at once | 5 | +| `-s, --split` | connections per download (the speed dial) | 5 | +| `-x, --max-connection-per-server` | cap connections per server (1–16) | 1 (= split when unset) | +| `-j, --max-concurrent-downloads` | files downloaded at once | 5 | | `--max-overall-download-limit` | global speed cap | 0 (off) | | `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, …) | | | `--seed-time` | minutes to seed after a torrent finishes | by ratio | diff --git a/bt/bt.go b/bt/bt.go index 390131f..4ee8eb4 100644 --- a/bt/bt.go +++ b/bt/bt.go @@ -184,7 +184,6 @@ type Options struct { SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding) SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime StopTimeout time.Duration // abort if no download progress for this long (0 = off) - MetaTimeout time.Duration // abort a magnet that can't fetch metadata in this long (0 = off) SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all CheckIntegrity bool // re-verify data against piece hashes before downloading DryRun bool // fetch metadata only, then stop without downloading (--dry-run) @@ -374,27 +373,27 @@ func (d *Download) choose(t *torrent.Torrent) error { return nil } -// awaitInfo blocks until the torrent metadata arrives. It gives up after -// --bt-metadata-timeout (default 60s) so a magnet with no reachable peers — a -// dead link, or UDP trackers behind a firewall with no DHT — fails fast instead -// of hanging forever. Setting --bt-metadata-timeout=0 disables the timeout and -// waits indefinitely. +// metadataTimeout bounds the magnet metadata-fetch phase. A magnet with no +// reachable peers — a dead link, or UDP trackers behind a firewall with no DHT — +// would otherwise hang forever, so awaitInfo gives up after this fixed window. +// It is not a user-facing flag: aria2 has no --bt-metadata-timeout, and 60s is a +// generous ceiling that a healthy swarm clears in well under a second. +const metadataTimeout = 60 * time.Second + +// awaitInfo blocks until the torrent metadata arrives, giving up after +// metadataTimeout so a peerless magnet fails fast instead of hanging forever. func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error { - var deadline <-chan time.Time - if d.opts.MetaTimeout > 0 { - tm := time.NewTimer(d.opts.MetaTimeout) - defer tm.Stop() - deadline = tm.C - } + tm := time.NewTimer(metadataTimeout) + defer tm.Stop() select { case <-t.GotInfo(): return nil case <-ctx.Done(): return ctx.Err() - case <-deadline: + case <-tm.C: // Wrap DeadlineExceeded so the exit-code mapping classifies it as a timeout // (2) via errors.Is, rather than matching on the message text. - return fmt.Errorf("timed out fetching metadata after %s: %w", d.opts.MetaTimeout, context.DeadlineExceeded) + return fmt.Errorf("timed out fetching metadata after %s: %w", metadataTimeout, context.DeadlineExceeded) } } diff --git a/cli/option.go b/cli/option.go index 1a1d744..76e7d24 100644 --- a/cli/option.go +++ b/cli/option.go @@ -64,38 +64,38 @@ type Opt struct { Max int64 } -// options is the single source of truth: a focused, practical subset. +// options is the single source of truth: a focused, practical subset. The Tag +// decides which --help group an option shows under; the Basic group is the bare +// `got --help` and is kept to the handful of flags that change WHAT happens, not +// how fast. Every other flag still works — it just lives behind --help=. var options = []Opt{ - // --- basic --- + // --- basic --- the outcome-changing core shown by a bare `got --help`. {Long: "dir", Short: 'd', Kind: Str, Help: "directory to store downloaded files", Tag: Basic}, {Long: "out", Short: 'o', Kind: Str, Help: "output filename for the download", Tag: Basic}, {Long: "input-file", Short: 'i', Kind: Str, Help: "read URIs line by line from FILE (- for stdin)", Tag: Basic}, + {Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "how many files to download at once (not connections within one file)", Tag: Basic}, {Long: "continue", Short: 'c', Kind: Bool, Default: "false", Help: "resume a partially downloaded file", Tag: Basic}, - {Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "max number of parallel downloads", Tag: Basic}, - {Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes (BitTorrent)", Tag: Basic}, - {Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic}, - {Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Basic}, - {Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Basic}, + {Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "connections per download — the speed dial (default 5; e.g. -s16 for 16)", Tag: Basic}, {Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic}, - {Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: Basic}, + {Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256= (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: Basic}, + {Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic}, {Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic}, - {Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Basic}, // --- http --- - {Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "max connections to one server (1-16)", Tag: Basic}, - {Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "split a download into N connections; actual connections are min(max-connection-per-server, split), and -x defaults to 1", Tag: Basic}, - {Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: Basic}, + {Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "cap connections to any single server (1-16); limits --split per host, mainly for mirrors", Tag: HTTP}, + {Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: HTTP}, + {Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: HTTP}, {Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP}, {Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP}, {Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection timeout in seconds", Tag: HTTP}, + {Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP}, + {Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP}, {Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable)", Tag: HTTP}, {Long: "user-agent", Short: 'U', Kind: Str, Default: "got/1.0", Help: "HTTP User-Agent", Tag: HTTP}, {Long: "referer", Kind: Str, Help: "HTTP Referer header", Tag: HTTP}, {Long: "all-proxy", Kind: Str, Help: "proxy for all protocols (host:port)", Tag: HTTP}, {Long: "check-certificate", Kind: Bool, Default: "true", Help: "verify the server's TLS certificate", Tag: HTTP}, {Long: "ca-certificate", Kind: Str, Help: "verify HTTPS servers against the CA certificates in FILE (PEM)", Tag: HTTP}, - {Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP}, - {Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP}, {Long: "load-cookies", Kind: Str, Help: "load Cookies from FILE (Netscape/Mozilla format)", Tag: HTTP}, {Long: "save-cookies", Kind: Str, Help: "save Cookies to FILE on exit", Tag: HTTP}, {Long: "conditional-get", Kind: Bool, Default: "false", Help: "download only if the remote file is newer than the local one", Tag: HTTP}, @@ -103,14 +103,13 @@ var options = []Opt{ {Long: "http-user", Kind: Str, Help: "HTTP basic-auth user", Tag: HTTP}, {Long: "http-passwd", Kind: Str, Help: "HTTP basic-auth password", Tag: HTTP}, {Long: "lowest-speed-limit", Kind: Size, Default: "0", Help: "abort if speed stays below SIZE bytes/sec (0 = off)", Tag: HTTP}, - {Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256= (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: HTTP}, // --- bittorrent --- {Long: "torrent-file", Short: 'T', Kind: Str, Help: "path to a .torrent file", Tag: BitTorrent}, + {Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes", Tag: BitTorrent}, {Long: "seed-time", Kind: Float, Metavar: "MIN", Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent}, {Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent}, {Long: "bt-stop-timeout", Kind: Int, Default: "0", Min: 0, Help: "stop a torrent with no download progress for N seconds (0 = off)", Tag: BitTorrent}, - {Long: "bt-metadata-timeout", Kind: Int, Default: "60", Min: 0, Help: "stop a magnet that can't fetch metadata in N seconds (0 = off)", Tag: BitTorrent}, {Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent}, {Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT", Tag: BitTorrent}, {Long: "bt-max-peers", Kind: Int, Default: "55", Min: 0, Help: "max peers per torrent (0 = unlimited)", Tag: BitTorrent}, @@ -120,6 +119,9 @@ var options = []Opt{ {Long: "follow-torrent", Kind: Bool, Default: "true", Help: "after fetching a .torrent over HTTP, download its content", Tag: BitTorrent}, // --- advanced --- + {Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Advanced}, + {Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Advanced}, + {Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Advanced}, {Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to allocate disk space", Tag: Advanced}, {Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced}, {Long: "stop", Kind: Int, Default: "0", Min: 0, Help: "stop the program after N seconds (0 = off)", Tag: Advanced}, diff --git a/cli/parse_test.go b/cli/parse_test.go index da23605..83be192 100644 --- a/cli/parse_test.go +++ b/cli/parse_test.go @@ -22,7 +22,7 @@ func TestParseArgs(t *testing.T) { {"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}}, {"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}}, {"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}}, - {"bt metadata timeout", []string{"--bt-metadata-timeout=120", "magnet:x"}, Run, map[string]string{"bt-metadata-timeout": "120"}, []string{"magnet:x"}}, + {"long int + magnet uri", []string{"--bt-stop-timeout=120", "magnet:x"}, Run, map[string]string{"bt-stop-timeout": "120"}, []string{"magnet:x"}}, {"help", []string{"--help"}, ShowHelp, nil, nil}, {"help short", []string{"-h"}, ShowHelp, nil, nil}, {"version", []string{"-v"}, ShowVersion, nil, nil}, @@ -68,21 +68,6 @@ func TestParseArgsErrors(t *testing.T) { } } -// got bounds the magnet metadata fetch by default rather than waiting forever; -// guard the default value and its zero floor. -func TestBTMetadataTimeoutDefault(t *testing.T) { - o, ok := byLong["bt-metadata-timeout"] - if !ok { - t.Fatal("bt-metadata-timeout option is missing") - } - if o.Default != "60" { - t.Errorf("bt-metadata-timeout default = %q, want %q", o.Default, "60") - } - if o.Min != 0 { - t.Errorf("bt-metadata-timeout Min = %d, want 0 (reject negatives, allow 0=off)", o.Min) - } -} - func TestParseSize(t *testing.T) { tests := []struct { in string diff --git a/main.go b/main.go index b37e321..5ffb3a5 100644 --- a/main.go +++ b/main.go @@ -524,11 +524,21 @@ func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl. if single { out = opts.Str("out") } + // max-connection-per-server caps connections per host. aria2's default is 1, + // which on its own would clamp --split to a single connection, so a bare + // `got URL` would download single-threaded. Honor the literal default of 1 + // only when the user actually set -x; otherwise let --split drive per-host + // parallelism, capped at the -x ceiling of 16. -x explicitly set keeps the + // exact min(split, mirrors*x) behavior. + perHost := opts.Int("max-connection-per-server") + if !opts.IsSet("max-connection-per-server") { + perHost = min(opts.Int("split"), 16) + } return httpdl.Config{ Dir: opts.Str("dir"), Out: out, Split: opts.Int("split"), - MaxConnPerServer: opts.Int("max-connection-per-server"), + MaxConnPerServer: perHost, MinSplit: opts.Size("min-split-size"), Tries: opts.Int("max-tries"), Timeout: time.Duration(opts.Int("timeout")) * time.Second, @@ -583,7 +593,6 @@ func btOptions(opts *cli.Options) bt.Options { SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)), SeedRatio: opts.Float("seed-ratio"), StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second, - MetaTimeout: time.Duration(opts.Int("bt-metadata-timeout")) * time.Second, SelectFiles: parseSelect(opts.Str("select-file")), CheckIntegrity: opts.Bool("check-integrity"), DryRun: opts.Bool("dry-run"), diff --git a/main_test.go b/main_test.go index a064daf..e6df4ae 100644 --- a/main_test.go +++ b/main_test.go @@ -144,6 +144,36 @@ func TestGatherURIsGrouping(t *testing.T) { } } +// TestHTTPConnDefault locks the parallelism default: a bare `got URL` must use +// several connections, not one. With -x unset, --split drives the per-host +// connection count (capped at the -x ceiling of 16); an explicit -x is honored +// verbatim, including the aria2-faithful -x1 single connection. +func TestHTTPConnDefault(t *testing.T) { + cfg := func(args ...string) httpdl.Config { + res, err := cli.Parse(append([]string{"--no-conf"}, args...)) + if err != nil { + t.Fatalf("parse %v: %v", args, err) + } + return httpConfig(res.Options, true, nil) + } + tests := []struct { + name string + args []string + want int + }{ + {"bare URL uses split default", []string{"http://a/x"}, 5}, + {"-s16 means 16", []string{"-s16", "http://a/x"}, 16}, + {"-s100 capped at 16 per host", []string{"-s100", "http://a/x"}, 16}, + {"explicit -x4 honored", []string{"-x4", "http://a/x"}, 4}, + {"explicit -x1 stays single", []string{"-x1", "http://a/x"}, 1}, + } + for _, tc := range tests { + if c := cfg(tc.args...); c.MaxConnPerServer != tc.want { + t.Errorf("%s: MaxConnPerServer = %d, want %d", tc.name, c.MaxConnPerServer, tc.want) + } + } +} + // TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on // a line are mirrors of one download; a plain line is one download. func TestReadInputFileTAB(t *testing.T) {