cli: trim basic help, drop invented flag, parallelize by default
Surface area, not the machinery, was the problem: 16 flags crowded the bare --help. Re-tag so basic shows only the 10 outcome-changing flags; every other flag still works behind --help=<group>. - cut --bt-metadata-timeout, an invented flag with no real counterpart. Keep its 60s magnet-metadata stall as a fixed internal default (bt.go), not a user knob. - fix the parallelism footgun: a bare `got URL` used min(split,x)=1 connection. When -x is unset, let --split drive per-host connections (capped at 16), so `got URL` gets 5 and `-s16` gets 16. An explicit -x is honored verbatim, including -x1. Locked by TestHTTPConnDefault. - -s is the per-download speed dial now, so it moves to basic; -x (a per-server cap) moves to the http group. -j help says "files at once, not connections within one file".
This commit is contained in:
16
README.md
16
README.md
@@ -15,15 +15,17 @@ Needs Go 1.25+.
|
||||
## Usage
|
||||
|
||||
```sh
|
||||
# HTTP download with 16 connections
|
||||
got -x16 -s16 https://example.com/big.iso
|
||||
# HTTP download: parallel out of the box (5 connections); -s turns it up
|
||||
got https://example.com/big.iso
|
||||
got -s16 https://example.com/big.iso
|
||||
|
||||
# resume an interrupted download
|
||||
got -c https://example.com/big.iso
|
||||
|
||||
# one file from several mirrors (connections spread across servers,
|
||||
# a dead mirror falls over); -Z downloads them as separate files instead
|
||||
got -x4 https://a.example/big.iso https://b.example/big.iso
|
||||
# a dead mirror falls over); -Z downloads them as separate files instead.
|
||||
# -x caps connections per server when mirroring.
|
||||
got -s16 -x4 https://a.example/big.iso https://b.example/big.iso
|
||||
|
||||
# a torrent or magnet; seed for 30 minutes after finishing
|
||||
got --seed-time=30 ubuntu.torrent
|
||||
@@ -42,9 +44,9 @@ Run `got --help` (or `--help=all`) for every option.
|
||||
| `-d, --dir` | output directory | `.` |
|
||||
| `-o, --out` | output filename | from URL |
|
||||
| `-c, --continue` | resume a partial download | false |
|
||||
| `-x, --max-connection-per-server` | connections to one server (1–16) | 1 |
|
||||
| `-s, --split` | split a download into N connections | 5 |
|
||||
| `-j, --max-concurrent-downloads` | downloads at once | 5 |
|
||||
| `-s, --split` | connections per download (the speed dial) | 5 |
|
||||
| `-x, --max-connection-per-server` | cap connections per server (1–16) | 1 (= split when unset) |
|
||||
| `-j, --max-concurrent-downloads` | files downloaded at once | 5 |
|
||||
| `--max-overall-download-limit` | global speed cap | 0 (off) |
|
||||
| `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, …) | |
|
||||
| `--seed-time` | minutes to seed after a torrent finishes | by ratio |
|
||||
|
||||
25
bt/bt.go
25
bt/bt.go
@@ -184,7 +184,6 @@ type Options struct {
|
||||
SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding)
|
||||
SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime
|
||||
StopTimeout time.Duration // abort if no download progress for this long (0 = off)
|
||||
MetaTimeout time.Duration // abort a magnet that can't fetch metadata in this long (0 = off)
|
||||
SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all
|
||||
CheckIntegrity bool // re-verify data against piece hashes before downloading
|
||||
DryRun bool // fetch metadata only, then stop without downloading (--dry-run)
|
||||
@@ -374,27 +373,27 @@ func (d *Download) choose(t *torrent.Torrent) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// awaitInfo blocks until the torrent metadata arrives. It gives up after
|
||||
// --bt-metadata-timeout (default 60s) so a magnet with no reachable peers — a
|
||||
// dead link, or UDP trackers behind a firewall with no DHT — fails fast instead
|
||||
// of hanging forever. Setting --bt-metadata-timeout=0 disables the timeout and
|
||||
// waits indefinitely.
|
||||
// metadataTimeout bounds the magnet metadata-fetch phase. A magnet with no
|
||||
// reachable peers — a dead link, or UDP trackers behind a firewall with no DHT —
|
||||
// would otherwise hang forever, so awaitInfo gives up after this fixed window.
|
||||
// It is not a user-facing flag: aria2 has no --bt-metadata-timeout, and 60s is a
|
||||
// generous ceiling that a healthy swarm clears in well under a second.
|
||||
const metadataTimeout = 60 * time.Second
|
||||
|
||||
// awaitInfo blocks until the torrent metadata arrives, giving up after
|
||||
// metadataTimeout so a peerless magnet fails fast instead of hanging forever.
|
||||
func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error {
|
||||
var deadline <-chan time.Time
|
||||
if d.opts.MetaTimeout > 0 {
|
||||
tm := time.NewTimer(d.opts.MetaTimeout)
|
||||
tm := time.NewTimer(metadataTimeout)
|
||||
defer tm.Stop()
|
||||
deadline = tm.C
|
||||
}
|
||||
select {
|
||||
case <-t.GotInfo():
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-deadline:
|
||||
case <-tm.C:
|
||||
// Wrap DeadlineExceeded so the exit-code mapping classifies it as a timeout
|
||||
// (2) via errors.Is, rather than matching on the message text.
|
||||
return fmt.Errorf("timed out fetching metadata after %s: %w", d.opts.MetaTimeout, context.DeadlineExceeded)
|
||||
return fmt.Errorf("timed out fetching metadata after %s: %w", metadataTimeout, context.DeadlineExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -64,38 +64,38 @@ type Opt struct {
|
||||
Max int64
|
||||
}
|
||||
|
||||
// options is the single source of truth: a focused, practical subset.
|
||||
// options is the single source of truth: a focused, practical subset. The Tag
|
||||
// decides which --help group an option shows under; the Basic group is the bare
|
||||
// `got --help` and is kept to the handful of flags that change WHAT happens, not
|
||||
// how fast. Every other flag still works — it just lives behind --help=<group>.
|
||||
var options = []Opt{
|
||||
// --- basic ---
|
||||
// --- basic --- the outcome-changing core shown by a bare `got --help`.
|
||||
{Long: "dir", Short: 'd', Kind: Str, Help: "directory to store downloaded files", Tag: Basic},
|
||||
{Long: "out", Short: 'o', Kind: Str, Help: "output filename for the download", Tag: Basic},
|
||||
{Long: "input-file", Short: 'i', Kind: Str, Help: "read URIs line by line from FILE (- for stdin)", Tag: Basic},
|
||||
{Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "how many files to download at once (not connections within one file)", Tag: Basic},
|
||||
{Long: "continue", Short: 'c', Kind: Bool, Default: "false", Help: "resume a partially downloaded file", Tag: Basic},
|
||||
{Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "max number of parallel downloads", Tag: Basic},
|
||||
{Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes (BitTorrent)", Tag: Basic},
|
||||
{Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic},
|
||||
{Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Basic},
|
||||
{Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Basic},
|
||||
{Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "connections per download — the speed dial (default 5; e.g. -s16 for 16)", Tag: Basic},
|
||||
{Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic},
|
||||
{Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: Basic},
|
||||
{Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256=<hex> (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: Basic},
|
||||
{Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic},
|
||||
{Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic},
|
||||
{Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Basic},
|
||||
|
||||
// --- http ---
|
||||
{Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "max connections to one server (1-16)", Tag: Basic},
|
||||
{Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "split a download into N connections; actual connections are min(max-connection-per-server, split), and -x defaults to 1", Tag: Basic},
|
||||
{Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: Basic},
|
||||
{Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "cap connections to any single server (1-16); limits --split per host, mainly for mirrors", Tag: HTTP},
|
||||
{Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: HTTP},
|
||||
{Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: HTTP},
|
||||
{Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP},
|
||||
{Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP},
|
||||
{Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection timeout in seconds", Tag: HTTP},
|
||||
{Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP},
|
||||
{Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP},
|
||||
{Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable)", Tag: HTTP},
|
||||
{Long: "user-agent", Short: 'U', Kind: Str, Default: "got/1.0", Help: "HTTP User-Agent", Tag: HTTP},
|
||||
{Long: "referer", Kind: Str, Help: "HTTP Referer header", Tag: HTTP},
|
||||
{Long: "all-proxy", Kind: Str, Help: "proxy for all protocols (host:port)", Tag: HTTP},
|
||||
{Long: "check-certificate", Kind: Bool, Default: "true", Help: "verify the server's TLS certificate", Tag: HTTP},
|
||||
{Long: "ca-certificate", Kind: Str, Help: "verify HTTPS servers against the CA certificates in FILE (PEM)", Tag: HTTP},
|
||||
{Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP},
|
||||
{Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP},
|
||||
{Long: "load-cookies", Kind: Str, Help: "load Cookies from FILE (Netscape/Mozilla format)", Tag: HTTP},
|
||||
{Long: "save-cookies", Kind: Str, Help: "save Cookies to FILE on exit", Tag: HTTP},
|
||||
{Long: "conditional-get", Kind: Bool, Default: "false", Help: "download only if the remote file is newer than the local one", Tag: HTTP},
|
||||
@@ -103,14 +103,13 @@ var options = []Opt{
|
||||
{Long: "http-user", Kind: Str, Help: "HTTP basic-auth user", Tag: HTTP},
|
||||
{Long: "http-passwd", Kind: Str, Help: "HTTP basic-auth password", Tag: HTTP},
|
||||
{Long: "lowest-speed-limit", Kind: Size, Default: "0", Help: "abort if speed stays below SIZE bytes/sec (0 = off)", Tag: HTTP},
|
||||
{Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256=<hex> (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: HTTP},
|
||||
|
||||
// --- bittorrent ---
|
||||
{Long: "torrent-file", Short: 'T', Kind: Str, Help: "path to a .torrent file", Tag: BitTorrent},
|
||||
{Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes", Tag: BitTorrent},
|
||||
{Long: "seed-time", Kind: Float, Metavar: "MIN", Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent},
|
||||
{Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent},
|
||||
{Long: "bt-stop-timeout", Kind: Int, Default: "0", Min: 0, Help: "stop a torrent with no download progress for N seconds (0 = off)", Tag: BitTorrent},
|
||||
{Long: "bt-metadata-timeout", Kind: Int, Default: "60", Min: 0, Help: "stop a magnet that can't fetch metadata in N seconds (0 = off)", Tag: BitTorrent},
|
||||
{Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent},
|
||||
{Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT", Tag: BitTorrent},
|
||||
{Long: "bt-max-peers", Kind: Int, Default: "55", Min: 0, Help: "max peers per torrent (0 = unlimited)", Tag: BitTorrent},
|
||||
@@ -120,6 +119,9 @@ var options = []Opt{
|
||||
{Long: "follow-torrent", Kind: Bool, Default: "true", Help: "after fetching a .torrent over HTTP, download its content", Tag: BitTorrent},
|
||||
|
||||
// --- advanced ---
|
||||
{Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Advanced},
|
||||
{Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Advanced},
|
||||
{Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Advanced},
|
||||
{Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to allocate disk space", Tag: Advanced},
|
||||
{Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced},
|
||||
{Long: "stop", Kind: Int, Default: "0", Min: 0, Help: "stop the program after N seconds (0 = off)", Tag: Advanced},
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestParseArgs(t *testing.T) {
|
||||
{"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}},
|
||||
{"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}},
|
||||
{"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}},
|
||||
{"bt metadata timeout", []string{"--bt-metadata-timeout=120", "magnet:x"}, Run, map[string]string{"bt-metadata-timeout": "120"}, []string{"magnet:x"}},
|
||||
{"long int + magnet uri", []string{"--bt-stop-timeout=120", "magnet:x"}, Run, map[string]string{"bt-stop-timeout": "120"}, []string{"magnet:x"}},
|
||||
{"help", []string{"--help"}, ShowHelp, nil, nil},
|
||||
{"help short", []string{"-h"}, ShowHelp, nil, nil},
|
||||
{"version", []string{"-v"}, ShowVersion, nil, nil},
|
||||
@@ -68,21 +68,6 @@ func TestParseArgsErrors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// got bounds the magnet metadata fetch by default rather than waiting forever;
|
||||
// guard the default value and its zero floor.
|
||||
func TestBTMetadataTimeoutDefault(t *testing.T) {
|
||||
o, ok := byLong["bt-metadata-timeout"]
|
||||
if !ok {
|
||||
t.Fatal("bt-metadata-timeout option is missing")
|
||||
}
|
||||
if o.Default != "60" {
|
||||
t.Errorf("bt-metadata-timeout default = %q, want %q", o.Default, "60")
|
||||
}
|
||||
if o.Min != 0 {
|
||||
t.Errorf("bt-metadata-timeout Min = %d, want 0 (reject negatives, allow 0=off)", o.Min)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
|
||||
13
main.go
13
main.go
@@ -524,11 +524,21 @@ func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.
|
||||
if single {
|
||||
out = opts.Str("out")
|
||||
}
|
||||
// max-connection-per-server caps connections per host. aria2's default is 1,
|
||||
// which on its own would clamp --split to a single connection, so a bare
|
||||
// `got URL` would download single-threaded. Honor the literal default of 1
|
||||
// only when the user actually set -x; otherwise let --split drive per-host
|
||||
// parallelism, capped at the -x ceiling of 16. -x explicitly set keeps the
|
||||
// exact min(split, mirrors*x) behavior.
|
||||
perHost := opts.Int("max-connection-per-server")
|
||||
if !opts.IsSet("max-connection-per-server") {
|
||||
perHost = min(opts.Int("split"), 16)
|
||||
}
|
||||
return httpdl.Config{
|
||||
Dir: opts.Str("dir"),
|
||||
Out: out,
|
||||
Split: opts.Int("split"),
|
||||
MaxConnPerServer: opts.Int("max-connection-per-server"),
|
||||
MaxConnPerServer: perHost,
|
||||
MinSplit: opts.Size("min-split-size"),
|
||||
Tries: opts.Int("max-tries"),
|
||||
Timeout: time.Duration(opts.Int("timeout")) * time.Second,
|
||||
@@ -583,7 +593,6 @@ func btOptions(opts *cli.Options) bt.Options {
|
||||
SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)),
|
||||
SeedRatio: opts.Float("seed-ratio"),
|
||||
StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second,
|
||||
MetaTimeout: time.Duration(opts.Int("bt-metadata-timeout")) * time.Second,
|
||||
SelectFiles: parseSelect(opts.Str("select-file")),
|
||||
CheckIntegrity: opts.Bool("check-integrity"),
|
||||
DryRun: opts.Bool("dry-run"),
|
||||
|
||||
30
main_test.go
30
main_test.go
@@ -144,6 +144,36 @@ func TestGatherURIsGrouping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHTTPConnDefault locks the parallelism default: a bare `got URL` must use
|
||||
// several connections, not one. With -x unset, --split drives the per-host
|
||||
// connection count (capped at the -x ceiling of 16); an explicit -x is honored
|
||||
// verbatim, including the aria2-faithful -x1 single connection.
|
||||
func TestHTTPConnDefault(t *testing.T) {
|
||||
cfg := func(args ...string) httpdl.Config {
|
||||
res, err := cli.Parse(append([]string{"--no-conf"}, args...))
|
||||
if err != nil {
|
||||
t.Fatalf("parse %v: %v", args, err)
|
||||
}
|
||||
return httpConfig(res.Options, true, nil)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{"bare URL uses split default", []string{"http://a/x"}, 5},
|
||||
{"-s16 means 16", []string{"-s16", "http://a/x"}, 16},
|
||||
{"-s100 capped at 16 per host", []string{"-s100", "http://a/x"}, 16},
|
||||
{"explicit -x4 honored", []string{"-x4", "http://a/x"}, 4},
|
||||
{"explicit -x1 stays single", []string{"-x1", "http://a/x"}, 1},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if c := cfg(tc.args...); c.MaxConnPerServer != tc.want {
|
||||
t.Errorf("%s: MaxConnPerServer = %d, want %d", tc.name, c.MaxConnPerServer, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on
|
||||
// a line are mirrors of one download; a plain line is one download.
|
||||
func TestReadInputFileTAB(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user