Compare commits
10 Commits
6e77cde4cc
...
81c44d9ec7
| Author | SHA1 | Date | |
|---|---|---|---|
| 81c44d9ec7 | |||
| 270812de3e | |||
| 65104ade92 | |||
| db73b51e0d | |||
| 27a5cece41 | |||
| d9de63f451 | |||
| 11734af7fb | |||
| 16cbb97748 | |||
| ee66235c36 | |||
| 3d9ea7ce8b |
16
README.md
16
README.md
@@ -15,17 +15,15 @@ Needs Go 1.25+.
|
||||
## Usage
|
||||
|
||||
```sh
|
||||
# HTTP download: parallel out of the box (5 connections); -s turns it up
|
||||
got https://example.com/big.iso
|
||||
got -s16 https://example.com/big.iso
|
||||
# HTTP download with 16 connections
|
||||
got -x16 -s16 https://example.com/big.iso
|
||||
|
||||
# resume an interrupted download
|
||||
got -c https://example.com/big.iso
|
||||
|
||||
# one file from several mirrors (connections spread across servers,
|
||||
# a dead mirror falls over); -Z downloads them as separate files instead.
|
||||
# -x caps connections per server when mirroring.
|
||||
got -s16 -x4 https://a.example/big.iso https://b.example/big.iso
|
||||
# a dead mirror falls over); -Z downloads them as separate files instead
|
||||
got -x4 https://a.example/big.iso https://b.example/big.iso
|
||||
|
||||
# a torrent or magnet; seed for 30 minutes after finishing
|
||||
got --seed-time=30 ubuntu.torrent
|
||||
@@ -44,9 +42,9 @@ Run `got --help` (or `--help=all`) for every option.
|
||||
| `-d, --dir` | output directory | `.` |
|
||||
| `-o, --out` | output filename | from URL |
|
||||
| `-c, --continue` | resume a partial download | false |
|
||||
| `-s, --split` | connections per download (the speed dial) | 5 |
|
||||
| `-x, --max-connection-per-server` | cap connections per server (1–16) | 1 (= split when unset) |
|
||||
| `-j, --max-concurrent-downloads` | files downloaded at once | 5 |
|
||||
| `-x, --max-connection-per-server` | connections to one server (1–16) | 1 |
|
||||
| `-s, --split` | split a download into N connections | 5 |
|
||||
| `-j, --max-concurrent-downloads` | downloads at once | 5 |
|
||||
| `--max-overall-download-limit` | global speed cap | 0 (off) |
|
||||
| `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, …) | |
|
||||
| `--seed-time` | minutes to seed after a torrent finishes | by ratio |
|
||||
|
||||
37
bt/bt.go
37
bt/bt.go
@@ -19,6 +19,7 @@ import (
|
||||
"time"
|
||||
|
||||
alog "github.com/anacrolix/log"
|
||||
missinggo "github.com/anacrolix/missinggo/v2"
|
||||
"github.com/anacrolix/torrent"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/hanbok/got/download"
|
||||
@@ -171,11 +172,10 @@ func NewClient(cfg ClientConfig) (*torrent.Client, error) {
|
||||
return cl, nil
|
||||
}
|
||||
|
||||
// isAddrInUse reports whether err is an "address already in use" bind failure.
|
||||
// (anacrolix/missinggo's IsAddrInUse is this exact string match, so we just do
|
||||
// it directly rather than carry the import for it.)
|
||||
// isAddrInUse reports whether err is an "address already in use" bind failure,
|
||||
// matching either missinggo's helper or the raw string the library wraps.
|
||||
func isAddrInUse(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "address already in use")
|
||||
return err != nil && (missinggo.IsAddrInUse(err) || strings.Contains(err.Error(), "address already in use"))
|
||||
}
|
||||
|
||||
// Options are the per-download settings.
|
||||
@@ -184,6 +184,7 @@ type Options struct {
|
||||
SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding)
|
||||
SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime
|
||||
StopTimeout time.Duration // abort if no download progress for this long (0 = off)
|
||||
MetaTimeout time.Duration // abort a magnet that can't fetch metadata in this long (0 = off)
|
||||
SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all
|
||||
CheckIntegrity bool // re-verify data against piece hashes before downloading
|
||||
DryRun bool // fetch metadata only, then stop without downloading (--dry-run)
|
||||
@@ -313,8 +314,8 @@ func (d *Download) Run(ctx context.Context) (err error) {
|
||||
}
|
||||
|
||||
// --check-integrity: re-hash the existing on-disk data BEFORE arming the
|
||||
// request loop, so already-good pieces are not re-requested from peers
|
||||
// (verify first, then fetch only what is missing).
|
||||
// request loop, so already-good pieces are not re-requested from peers (aria2
|
||||
// verifies first, then fetches only what is missing).
|
||||
if d.opts.CheckIntegrity {
|
||||
if err := t.VerifyDataContext(ctx); err != nil {
|
||||
return err
|
||||
@@ -373,27 +374,27 @@ func (d *Download) choose(t *torrent.Torrent) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// metadataTimeout bounds the magnet metadata-fetch phase. A magnet with no
|
||||
// reachable peers — a dead link, or UDP trackers behind a firewall with no DHT —
|
||||
// would otherwise hang forever, so awaitInfo gives up after this fixed window.
|
||||
// It is not a user-facing flag — 60s is a generous ceiling that a healthy swarm
|
||||
// clears in well under a second.
|
||||
const metadataTimeout = 60 * time.Second
|
||||
|
||||
// awaitInfo blocks until the torrent metadata arrives, giving up after
|
||||
// metadataTimeout so a peerless magnet fails fast instead of hanging forever.
|
||||
// awaitInfo blocks until the torrent metadata arrives. It gives up after
|
||||
// --bt-metadata-timeout (default 60s) so a magnet with no reachable peers — a
|
||||
// dead link, or UDP trackers behind a firewall with no DHT — fails fast instead
|
||||
// of hanging forever. Setting --bt-metadata-timeout=0 disables the timeout and
|
||||
// waits indefinitely.
|
||||
func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error {
|
||||
tm := time.NewTimer(metadataTimeout)
|
||||
var deadline <-chan time.Time
|
||||
if d.opts.MetaTimeout > 0 {
|
||||
tm := time.NewTimer(d.opts.MetaTimeout)
|
||||
defer tm.Stop()
|
||||
deadline = tm.C
|
||||
}
|
||||
select {
|
||||
case <-t.GotInfo():
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-tm.C:
|
||||
case <-deadline:
|
||||
// Wrap DeadlineExceeded so the exit-code mapping classifies it as a timeout
|
||||
// (2) via errors.Is, rather than matching on the message text.
|
||||
return fmt.Errorf("timed out fetching metadata after %s: %w", metadataTimeout, context.DeadlineExceeded)
|
||||
return fmt.Errorf("timed out fetching metadata after %s: %w", d.opts.MetaTimeout, context.DeadlineExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
43
cli/help.go
43
cli/help.go
@@ -89,31 +89,24 @@ func flagLabel(o *Opt) string {
|
||||
b.WriteString(" ")
|
||||
}
|
||||
b.WriteString("--" + o.Long)
|
||||
if m := metavar(o); m != "" {
|
||||
b.WriteString("=" + m)
|
||||
switch o.Kind {
|
||||
case Bool:
|
||||
// no argument shown
|
||||
case Size:
|
||||
b.WriteString("=SIZE")
|
||||
case Int:
|
||||
b.WriteString("=N")
|
||||
case Float:
|
||||
// seed-time is a count of minutes; only seed-ratio is an actual ratio.
|
||||
if o.Long == "seed-time" {
|
||||
b.WriteString("=MIN")
|
||||
} else {
|
||||
b.WriteString("=RATIO")
|
||||
}
|
||||
case Enum:
|
||||
b.WriteString("=" + strings.Join(o.Enum, "|"))
|
||||
default:
|
||||
b.WriteString("=VAL")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// metavar is the placeholder shown after a value-taking flag, e.g. the "N" in
|
||||
// "--split=N". An explicit Opt.Metavar wins; otherwise it derives from the Kind.
|
||||
// Bool takes no value, so its metavar is empty and no "=ARG" is printed.
|
||||
func metavar(o *Opt) string {
|
||||
if o.Metavar != "" {
|
||||
return o.Metavar
|
||||
}
|
||||
switch o.Kind {
|
||||
case Bool:
|
||||
return ""
|
||||
case Size:
|
||||
return "SIZE"
|
||||
case Int:
|
||||
return "N"
|
||||
case Float:
|
||||
return "RATIO"
|
||||
case Enum:
|
||||
return strings.Join(o.Enum, "|")
|
||||
default:
|
||||
return "VAL"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,10 +50,6 @@ type Opt struct {
|
||||
Kind Kind
|
||||
Default string
|
||||
Help string
|
||||
// Metavar overrides the "=PLACEHOLDER" shown after the flag in --help. Empty
|
||||
// means derive it from Kind (N, SIZE, RATIO, ...). Set it only when the
|
||||
// Kind-derived default would mislead, e.g. seed-time is minutes, not a ratio.
|
||||
Metavar string
|
||||
Tag Tag
|
||||
Enum []string // valid values when Kind == Enum
|
||||
// Min is the inclusive lower Int/Size bound. The zero value enforces a
|
||||
@@ -64,38 +60,38 @@ type Opt struct {
|
||||
Max int64
|
||||
}
|
||||
|
||||
// options is the single source of truth: a focused, practical subset. The Tag
|
||||
// decides which --help group an option shows under; the Basic group is the bare
|
||||
// `got --help` and is kept to the handful of flags that change WHAT happens, not
|
||||
// how fast. Every other flag still works — it just lives behind --help=<group>.
|
||||
// options is the single source of truth: a focused, practical subset.
|
||||
var options = []Opt{
|
||||
// --- basic --- the outcome-changing core shown by a bare `got --help`.
|
||||
// --- basic ---
|
||||
{Long: "dir", Short: 'd', Kind: Str, Help: "directory to store downloaded files", Tag: Basic},
|
||||
{Long: "out", Short: 'o', Kind: Str, Help: "output filename for the download", Tag: Basic},
|
||||
{Long: "input-file", Short: 'i', Kind: Str, Help: "read URIs line by line from FILE (- for stdin)", Tag: Basic},
|
||||
{Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "how many files to download at once (not connections within one file)", Tag: Basic},
|
||||
{Long: "continue", Short: 'c', Kind: Bool, Default: "false", Help: "resume a partially downloaded file", Tag: Basic},
|
||||
{Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "connections per download — the speed dial (default 5; e.g. -s16 for 16)", Tag: Basic},
|
||||
{Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic},
|
||||
{Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256=<hex> (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: Basic},
|
||||
{Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "max number of parallel downloads", Tag: Basic},
|
||||
{Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes (BitTorrent)", Tag: Basic},
|
||||
{Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic},
|
||||
{Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Basic},
|
||||
{Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Basic},
|
||||
{Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic},
|
||||
{Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: Basic},
|
||||
{Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic},
|
||||
{Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Basic},
|
||||
|
||||
// --- http ---
|
||||
{Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "cap connections to any single server (1-16); limits --split per host, mainly for mirrors", Tag: HTTP},
|
||||
{Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: HTTP},
|
||||
{Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: HTTP},
|
||||
{Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "max connections to one server (1-16)", Tag: Basic},
|
||||
{Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "split a download into N connections; actual connections are min(max-connection-per-server, split), and -x defaults to 1", Tag: Basic},
|
||||
{Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: Basic},
|
||||
{Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP},
|
||||
{Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP},
|
||||
{Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection timeout in seconds", Tag: HTTP},
|
||||
{Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP},
|
||||
{Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP},
|
||||
{Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable)", Tag: HTTP},
|
||||
{Long: "user-agent", Short: 'U', Kind: Str, Default: "got/1.0", Help: "HTTP User-Agent", Tag: HTTP},
|
||||
{Long: "referer", Kind: Str, Help: "HTTP Referer header", Tag: HTTP},
|
||||
{Long: "all-proxy", Kind: Str, Help: "proxy for all protocols (host:port)", Tag: HTTP},
|
||||
{Long: "check-certificate", Kind: Bool, Default: "true", Help: "verify the server's TLS certificate", Tag: HTTP},
|
||||
{Long: "ca-certificate", Kind: Str, Help: "verify HTTPS servers against the CA certificates in FILE (PEM)", Tag: HTTP},
|
||||
{Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP},
|
||||
{Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP},
|
||||
{Long: "load-cookies", Kind: Str, Help: "load Cookies from FILE (Netscape/Mozilla format)", Tag: HTTP},
|
||||
{Long: "save-cookies", Kind: Str, Help: "save Cookies to FILE on exit", Tag: HTTP},
|
||||
{Long: "conditional-get", Kind: Bool, Default: "false", Help: "download only if the remote file is newer than the local one", Tag: HTTP},
|
||||
@@ -103,13 +99,14 @@ var options = []Opt{
|
||||
{Long: "http-user", Kind: Str, Help: "HTTP basic-auth user", Tag: HTTP},
|
||||
{Long: "http-passwd", Kind: Str, Help: "HTTP basic-auth password", Tag: HTTP},
|
||||
{Long: "lowest-speed-limit", Kind: Size, Default: "0", Help: "abort if speed stays below SIZE bytes/sec (0 = off)", Tag: HTTP},
|
||||
{Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256=<hex> (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: HTTP},
|
||||
|
||||
// --- bittorrent ---
|
||||
{Long: "torrent-file", Short: 'T', Kind: Str, Help: "path to a .torrent file", Tag: BitTorrent},
|
||||
{Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes", Tag: BitTorrent},
|
||||
{Long: "seed-time", Kind: Float, Metavar: "MIN", Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent},
|
||||
{Long: "seed-time", Kind: Float, Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent},
|
||||
{Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent},
|
||||
{Long: "bt-stop-timeout", Kind: Int, Default: "0", Min: 0, Help: "stop a torrent with no download progress for N seconds (0 = off)", Tag: BitTorrent},
|
||||
{Long: "bt-metadata-timeout", Kind: Int, Default: "60", Min: 0, Help: "stop a magnet that can't fetch metadata in N seconds (0 = off)", Tag: BitTorrent},
|
||||
{Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent},
|
||||
{Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT", Tag: BitTorrent},
|
||||
{Long: "bt-max-peers", Kind: Int, Default: "55", Min: 0, Help: "max peers per torrent (0 = unlimited)", Tag: BitTorrent},
|
||||
@@ -119,10 +116,7 @@ var options = []Opt{
|
||||
{Long: "follow-torrent", Kind: Bool, Default: "true", Help: "after fetching a .torrent over HTTP, download its content", Tag: BitTorrent},
|
||||
|
||||
// --- advanced ---
|
||||
{Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Advanced},
|
||||
{Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Advanced},
|
||||
{Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Advanced},
|
||||
{Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to reserve disk space (prealloc and falloc are identical here)", Tag: Advanced},
|
||||
{Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to allocate disk space", Tag: Advanced},
|
||||
{Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced},
|
||||
{Long: "stop", Kind: Int, Default: "0", Min: 0, Help: "stop the program after N seconds (0 = off)", Tag: Advanced},
|
||||
{Long: "disable-ipv6", Kind: Bool, Default: "false", Help: "disable IPv6 (force IPv4-only connections)", Tag: Advanced},
|
||||
|
||||
@@ -27,20 +27,18 @@ func (o *Options) IsSet(name string) bool { return o.set[name] }
|
||||
// Str returns the raw string value (empty if unset and no default).
|
||||
func (o *Options) Str(name string) string { return o.vals[name] }
|
||||
|
||||
// boolWords is the accepted vocabulary for boolean options: exactly true and
|
||||
// false, nothing else. boolWord is the single consult point — the Bool reader,
|
||||
// validate(), and the no-conf bootstrap all go through it — so exactly the words
|
||||
// that validate are honoured.
|
||||
// boolWords is the single accepted vocabulary for boolean options, mapping each
|
||||
// recognised spelling to its truth value. The Bool reader, truthy, and
|
||||
// validate() all consult it, so exactly the words that validate are honoured
|
||||
// (no "accepted by the reader but rejected by validate" surprises like --x=on).
|
||||
var boolWords = map[string]bool{
|
||||
"true": true,
|
||||
"false": false,
|
||||
"true": true, "yes": true, "1": true, "on": true,
|
||||
"false": false, "no": false, "0": false, "off": false,
|
||||
}
|
||||
|
||||
// boolWord reports a value's truth and whether it is a recognised boolean word.
|
||||
// Whitespace is trimmed, but case is significant: "True"/"TRUE" are rejected, so
|
||||
// the match is against exactly "true"/"false".
|
||||
func boolWord(s string) (val, ok bool) {
|
||||
val, ok = boolWords[strings.TrimSpace(s)]
|
||||
val, ok = boolWords[strings.ToLower(strings.TrimSpace(s))]
|
||||
return val, ok
|
||||
}
|
||||
|
||||
@@ -50,11 +48,13 @@ func (o *Options) Bool(name string) bool {
|
||||
return val
|
||||
}
|
||||
|
||||
// Int returns the value as an int, or 0 if empty/invalid. The value has already
|
||||
// passed validate(), so a parse failure here is unreachable in normal flow.
|
||||
func (o *Options) Int(name string) int {
|
||||
// Int returns the value as an int, or 0 if empty/invalid.
|
||||
func (o *Options) Int(name string) int { return int(o.Int64(name)) }
|
||||
|
||||
// Int64 returns the value as an int64, or 0 if empty/invalid.
|
||||
func (o *Options) Int64(name string) int64 {
|
||||
n, _ := strconv.ParseInt(strings.TrimSpace(o.vals[name]), 10, 64)
|
||||
return int(n)
|
||||
return n
|
||||
}
|
||||
|
||||
// Float returns the value as a float64, or 0 if empty/invalid.
|
||||
|
||||
28
cli/parse.go
28
cli/parse.go
@@ -41,7 +41,7 @@ func Parse(args []string) (*Result, error) {
|
||||
opts := newOptions()
|
||||
applyDefaults(opts)
|
||||
|
||||
if noConf, _ := boolWord(cmdVals["no-conf"]); !noConf {
|
||||
if !truthy(cmdVals["no-conf"]) {
|
||||
explicit := cmdVals["conf-path"]
|
||||
path, fromUser := confPath(explicit)
|
||||
if err := applyConfig(opts, path, fromUser); err != nil {
|
||||
@@ -138,23 +138,14 @@ func parseArgs(args []string) (vals map[string]string, uris []string, action Act
|
||||
// accumulate stores a value, joining repeatable List options with newlines.
|
||||
func accumulate(vals map[string]string, o *Opt, val string) {
|
||||
if o.Kind == List {
|
||||
vals[o.Long] = appendList(vals[o.Long], val)
|
||||
if prev, ok := vals[o.Long]; ok {
|
||||
vals[o.Long] = prev + "\n" + val
|
||||
return
|
||||
}
|
||||
}
|
||||
vals[o.Long] = val
|
||||
}
|
||||
|
||||
// appendList is the single home of the List newline-join rule, shared by the
|
||||
// raw-parse layer (accumulate) and the resolved layer (set). An empty prev (the
|
||||
// option not yet seen) yields val unchanged, so the first value carries no
|
||||
// leading newline.
|
||||
func appendList(prev, val string) string {
|
||||
if prev == "" {
|
||||
return val
|
||||
}
|
||||
return prev + "\n" + val
|
||||
}
|
||||
|
||||
func applyDefaults(o *Options) {
|
||||
for i := range options {
|
||||
opt := &options[i]
|
||||
@@ -224,8 +215,10 @@ func set(o *Options, name, val string) error {
|
||||
if err := validate(opt, val); err != nil {
|
||||
return err
|
||||
}
|
||||
if opt.Kind == List && o.set[name] {
|
||||
val = appendList(o.vals[name], val)
|
||||
if opt.Kind == List {
|
||||
if prev, ok := o.vals[name]; ok && o.set[name] {
|
||||
val = prev + "\n" + val
|
||||
}
|
||||
}
|
||||
o.vals[name] = val
|
||||
o.set[name] = true
|
||||
@@ -311,3 +304,8 @@ func confPath(explicit string) (path string, fromUser bool) {
|
||||
}
|
||||
return filepath.Join(home, ".config", "got", "got.conf"), false
|
||||
}
|
||||
|
||||
func truthy(s string) bool {
|
||||
val, _ := boolWord(s)
|
||||
return val
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestParseArgs(t *testing.T) {
|
||||
{"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}},
|
||||
{"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}},
|
||||
{"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}},
|
||||
{"long int + magnet uri", []string{"--bt-stop-timeout=120", "magnet:x"}, Run, map[string]string{"bt-stop-timeout": "120"}, []string{"magnet:x"}},
|
||||
{"bt metadata timeout", []string{"--bt-metadata-timeout=120", "magnet:x"}, Run, map[string]string{"bt-metadata-timeout": "120"}, []string{"magnet:x"}},
|
||||
{"help", []string{"--help"}, ShowHelp, nil, nil},
|
||||
{"help short", []string{"-h"}, ShowHelp, nil, nil},
|
||||
{"version", []string{"-v"}, ShowVersion, nil, nil},
|
||||
@@ -68,6 +68,21 @@ func TestParseArgsErrors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// got bounds the magnet metadata fetch by default rather than waiting forever;
|
||||
// guard the default value and its zero floor.
|
||||
func TestBTMetadataTimeoutDefault(t *testing.T) {
|
||||
o, ok := byLong["bt-metadata-timeout"]
|
||||
if !ok {
|
||||
t.Fatal("bt-metadata-timeout option is missing")
|
||||
}
|
||||
if o.Default != "60" {
|
||||
t.Errorf("bt-metadata-timeout default = %q, want %q", o.Default, "60")
|
||||
}
|
||||
if o.Min != 0 {
|
||||
t.Errorf("bt-metadata-timeout Min = %d, want 0 (reject negatives, allow 0=off)", o.Min)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
@@ -139,19 +154,13 @@ func TestValidateFloatNonNegative(t *testing.T) {
|
||||
|
||||
func TestValidateBool(t *testing.T) {
|
||||
b := byLong["enable-dht"]
|
||||
// Only literal true/false are accepted (case-sensitive), with surrounding
|
||||
// whitespace trimmed.
|
||||
for _, ok := range []string{"true", "false", " true ", "false\t"} {
|
||||
for _, ok := range []string{"true", "false", "yes", "no", "1", "0", "on", "off"} {
|
||||
if err := validate(b, ok); err != nil {
|
||||
t.Errorf("validate bool %q: %v", ok, err)
|
||||
}
|
||||
}
|
||||
// The invented spellings and any case variant must now fail.
|
||||
for _, bad := range []string{"yes", "no", "1", "0", "on", "off", "flase",
|
||||
"True", "TRUE", "tRuE", "False", "FALSE"} {
|
||||
if err := validate(b, bad); err == nil {
|
||||
t.Errorf("validate bool %q: want error (only true/false accepted)", bad)
|
||||
}
|
||||
if err := validate(b, "flase"); err == nil {
|
||||
t.Errorf("validate bool flase: want error")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,9 @@ type Stat struct {
|
||||
Seeders int // connected seeders (BitTorrent only)
|
||||
}
|
||||
|
||||
// Done reports whether the download has reached a terminal state.
|
||||
func (s Stat) Done() bool { return s.Status == Complete || s.Status == Errored }
|
||||
|
||||
// A Download is one logical job: a URL, a torrent, a magnet. Run blocks in its
|
||||
// own goroutine until the work finishes, fails, or ctx is cancelled. Stat may
|
||||
// be called concurrently at any time and must not block; implementations back
|
||||
|
||||
2
go.mod
2
go.mod
@@ -4,6 +4,7 @@ go 1.25
|
||||
|
||||
require (
|
||||
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb
|
||||
github.com/anacrolix/missinggo/v2 v2.10.0
|
||||
github.com/anacrolix/torrent v1.61.0
|
||||
golang.org/x/sys v0.38.0
|
||||
golang.org/x/term v0.37.0
|
||||
@@ -21,7 +22,6 @@ require (
|
||||
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
||||
github.com/anacrolix/missinggo/v2 v2.10.0 // indirect
|
||||
github.com/anacrolix/mmsg v1.0.1 // indirect
|
||||
github.com/anacrolix/multiless v0.4.0 // indirect
|
||||
github.com/anacrolix/stm v0.5.0 // indirect
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
@@ -33,7 +34,7 @@ func controlPath(out string) string { return out + ".got" }
|
||||
func snapshot(url string, total int64, etag, lastmod string, segs []seg) control {
|
||||
c := control{URL: url, Total: total, ETag: etag, LastModified: lastmod, Segs: make([]segState, len(segs))}
|
||||
for i := range segs {
|
||||
c.Segs[i] = segState{segs[i].start, segs[i].end, atomic.LoadInt64(&segs[i].written)}
|
||||
c.Segs[i] = segState{segs[i].start, segs[i].endOff(), atomic.LoadInt64(&segs[i].written)}
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -87,15 +88,16 @@ func loadControl(out, url string, total int64, etag, lastmod string) *control {
|
||||
}
|
||||
|
||||
// validSegs reports whether segs cover [0,total) with no gap or overlap and a
|
||||
// sane written count for each. got always writes segments in ascending start
|
||||
// order, so coverage is checked in place; an out-of-order sidecar (only possible
|
||||
// from a hand-edited or corrupted file) is rejected, which restarts cleanly.
|
||||
// sane written count for each. The recorded order is not sorted (a steal appends
|
||||
// a tail), so we check coverage on a sorted copy.
|
||||
func validSegs(segs []segState, total int64) bool {
|
||||
if total <= 0 || len(segs) == 0 {
|
||||
return false
|
||||
}
|
||||
sorted := append([]segState(nil), segs...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
|
||||
var next int64
|
||||
for _, s := range segs {
|
||||
for _, s := range sorted {
|
||||
length := s.End - s.Start + 1
|
||||
if s.Start != next || s.End < s.Start || s.Written < 0 || s.Written > length {
|
||||
return false
|
||||
|
||||
@@ -758,7 +758,7 @@ func (d *Download) fetchSeg(ctx context.Context, f *os.File, s *seg, total int64
|
||||
func (d *Download) fetchOnce(ctx context.Context, f *os.File, s *seg, uri string, total int64) error {
|
||||
reqCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
req, err := d.request(reqCtx, uri, fmt.Sprintf("bytes=%d-%d", s.offset(), s.end))
|
||||
req, err := d.request(reqCtx, uri, fmt.Sprintf("bytes=%d-%d", s.offset(), s.endOff()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ type seg struct {
|
||||
}
|
||||
|
||||
func (s *seg) length() int64 { return s.end - s.start + 1 }
|
||||
func (s *seg) endOff() int64 { return s.end }
|
||||
func (s *seg) done() bool { return atomic.LoadInt64(&s.written) >= s.length() }
|
||||
func (s *seg) addWritten(n int64) { atomic.AddInt64(&s.written, n) }
|
||||
func (s *seg) progress() int64 { return atomic.LoadInt64(&s.written) }
|
||||
|
||||
@@ -201,7 +201,7 @@ func tilesCover(t *testing.T, segs []seg, total int64) {
|
||||
if s.start != next {
|
||||
t.Fatalf("segment gap/overlap: next byte %d, got start %d", next, s.start)
|
||||
}
|
||||
next = s.end + 1
|
||||
next = s.endOff() + 1
|
||||
}
|
||||
if next != total {
|
||||
t.Fatalf("segments cover %d bytes, want %d", next, total)
|
||||
|
||||
54
main.go
54
main.go
@@ -17,7 +17,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -94,12 +94,8 @@ func run(args []string) int {
|
||||
t := time.AfterFunc(time.Duration(stop)*time.Second, cancel)
|
||||
defer t.Stop()
|
||||
}
|
||||
var jobsRef atomic.Pointer[[]job]
|
||||
// publish snapshots the slice header (j is a by-value copy) and stores a
|
||||
// pointer to it, so the signal handler can read the current jobs lock-free
|
||||
// without racing the `jobs = append(...)` reassignment in the follow pass.
|
||||
publish := func(j []job) { jobsRef.Store(&j) }
|
||||
go handleSignals(cancel, sessionFile, &jobsRef)
|
||||
jobsRef := &jobsHolder{}
|
||||
go handleSignals(cancel, sessionFile, jobsRef)
|
||||
|
||||
eng := download.NewEngine(opts.Int("max-concurrent-downloads"))
|
||||
|
||||
@@ -115,7 +111,7 @@ func run(args []string) int {
|
||||
close(uiDone)
|
||||
}
|
||||
|
||||
publish(jobs)
|
||||
jobsRef.set(jobs)
|
||||
results := runJobs(ctx, eng, jobs)
|
||||
|
||||
// --follow-torrent: a .torrent fetched over HTTP becomes a BitTorrent
|
||||
@@ -129,7 +125,7 @@ func run(args []string) int {
|
||||
// Index space, so the sort below restores input order deterministically.
|
||||
nInitial := len(jobs)
|
||||
jobs = append(jobs, follow...)
|
||||
publish(jobs)
|
||||
jobsRef.set(jobs)
|
||||
followResults := runJobs(ctx, eng, follow)
|
||||
for i := range followResults {
|
||||
followResults[i].Index += nInitial
|
||||
@@ -157,6 +153,25 @@ func run(args []string) int {
|
||||
return report(results)
|
||||
}
|
||||
|
||||
// jobsHolder lets the signal handler reach the latest job list for a
|
||||
// best-effort session save on a forced (second-Ctrl-C) exit.
|
||||
type jobsHolder struct {
|
||||
mu sync.Mutex
|
||||
jobs []job
|
||||
}
|
||||
|
||||
func (h *jobsHolder) set(jobs []job) {
|
||||
h.mu.Lock()
|
||||
h.jobs = jobs
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *jobsHolder) get() []job {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.jobs
|
||||
}
|
||||
|
||||
// job pairs a download with the source string that produced it, so we can write
|
||||
// the still-unfinished ones back to a session file.
|
||||
type job struct {
|
||||
@@ -509,21 +524,11 @@ func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.
|
||||
if single {
|
||||
out = opts.Str("out")
|
||||
}
|
||||
// max-connection-per-server caps connections per host. Its default is 1,
|
||||
// which on its own would clamp --split to a single connection, so a bare
|
||||
// `got URL` would download single-threaded. Honor the literal default of 1
|
||||
// only when the user actually set -x; otherwise let --split drive per-host
|
||||
// parallelism, capped at the -x ceiling of 16. -x explicitly set keeps the
|
||||
// exact min(split, mirrors*x) behavior.
|
||||
perHost := opts.Int("max-connection-per-server")
|
||||
if !opts.IsSet("max-connection-per-server") {
|
||||
perHost = min(opts.Int("split"), 16)
|
||||
}
|
||||
return httpdl.Config{
|
||||
Dir: opts.Str("dir"),
|
||||
Out: out,
|
||||
Split: opts.Int("split"),
|
||||
MaxConnPerServer: perHost,
|
||||
MaxConnPerServer: opts.Int("max-connection-per-server"),
|
||||
MinSplit: opts.Size("min-split-size"),
|
||||
Tries: opts.Int("max-tries"),
|
||||
Timeout: time.Duration(opts.Int("timeout")) * time.Second,
|
||||
@@ -578,6 +583,7 @@ func btOptions(opts *cli.Options) bt.Options {
|
||||
SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)),
|
||||
SeedRatio: opts.Float("seed-ratio"),
|
||||
StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second,
|
||||
MetaTimeout: time.Duration(opts.Int("bt-metadata-timeout")) * time.Second,
|
||||
SelectFiles: parseSelect(opts.Str("select-file")),
|
||||
CheckIntegrity: opts.Bool("check-integrity"),
|
||||
DryRun: opts.Bool("dry-run"),
|
||||
@@ -795,18 +801,14 @@ func progressNote(s download.Stat) string {
|
||||
// TERM) cancels the graceful context so run() can wind down and save normally;
|
||||
// a second forces exit, but first writes the session best-effort so the resume
|
||||
// list survives a force-quit.
|
||||
func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *atomic.Pointer[[]job]) {
|
||||
func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *jobsHolder) {
|
||||
sig := make(chan os.Signal, 2)
|
||||
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
||||
<-sig // first: cancel the graceful context
|
||||
cancel()
|
||||
<-sig // second: force exit
|
||||
if sessionFile != "" {
|
||||
var js []job // nil until the first publish; saveSession handles an empty list
|
||||
if p := jobs.Load(); p != nil {
|
||||
js = *p
|
||||
}
|
||||
if err := saveSession(sessionFile, js); err != nil {
|
||||
if err := saveSession(sessionFile, jobs.get()); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
}
|
||||
}
|
||||
|
||||
30
main_test.go
30
main_test.go
@@ -144,36 +144,6 @@ func TestGatherURIsGrouping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHTTPConnDefault locks the parallelism default: a bare `got URL` must use
|
||||
// several connections, not one. With -x unset, --split drives the per-host
|
||||
// connection count (capped at the -x ceiling of 16); an explicit -x is honored
|
||||
// verbatim, including -x1 for a single connection.
|
||||
func TestHTTPConnDefault(t *testing.T) {
|
||||
cfg := func(args ...string) httpdl.Config {
|
||||
res, err := cli.Parse(append([]string{"--no-conf"}, args...))
|
||||
if err != nil {
|
||||
t.Fatalf("parse %v: %v", args, err)
|
||||
}
|
||||
return httpConfig(res.Options, true, nil)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{"bare URL uses split default", []string{"http://a/x"}, 5},
|
||||
{"-s16 means 16", []string{"-s16", "http://a/x"}, 16},
|
||||
{"-s100 capped at 16 per host", []string{"-s100", "http://a/x"}, 16},
|
||||
{"explicit -x4 honored", []string{"-x4", "http://a/x"}, 4},
|
||||
{"explicit -x1 stays single", []string{"-x1", "http://a/x"}, 1},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if c := cfg(tc.args...); c.MaxConnPerServer != tc.want {
|
||||
t.Errorf("%s: MaxConnPerServer = %d, want %d", tc.name, c.MaxConnPerServer, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on
|
||||
// a line are mirrors of one download; a plain line is one download.
|
||||
func TestReadInputFileTAB(t *testing.T) {
|
||||
|
||||
@@ -41,6 +41,12 @@ func humanSize(n int64, human bool) string {
|
||||
return fmt.Sprintf("%.0f%ciB", val, suffix)
|
||||
}
|
||||
|
||||
// speed formats a bytes-per-second rate. The DL:/UL: fields show the bare
|
||||
// abbreviated size with no "/s" suffix.
|
||||
func speed(bytesPerSec int64, human bool) string {
|
||||
return humanSize(bytesPerSec, human)
|
||||
}
|
||||
|
||||
// secfmt renders a duration as "1h2m3s", appending each unit only when it is
|
||||
// nonzero (but still showing seconds when the whole input is 0):
|
||||
// 3600s->"1h", 120s->"2m", 3720s->"1h2m". A non-positive or absurd
|
||||
|
||||
@@ -53,6 +53,16 @@ func TestSecfmt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeedNoSuffix(t *testing.T) {
|
||||
// The DL:/UL: fields show the bare abbreviated size with no "/s".
|
||||
if got := speed(1536, true); got != "1.5KiB" {
|
||||
t.Errorf("speed(1536,true) = %q, want %q", got, "1.5KiB")
|
||||
}
|
||||
if got := speed(2048, false); got != "2048B" {
|
||||
t.Errorf("speed(2048,false) = %q, want %q", got, "2048B")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercent(t *testing.T) {
|
||||
if got := percent(50, 100); got != 50 {
|
||||
t.Errorf("percent(50,100) = %d, want 50", got)
|
||||
|
||||
@@ -147,10 +147,10 @@ func (r *Reporter) lineOne(s download.Stat) string {
|
||||
fmt.Fprintf(&b, " SD:%d", s.Seeders)
|
||||
}
|
||||
if !seeding {
|
||||
fmt.Fprintf(&b, " DL:%s", humanSize(dl, r.human))
|
||||
fmt.Fprintf(&b, " DL:%s", speed(dl, r.human))
|
||||
}
|
||||
if seeding || ul > 0 {
|
||||
fmt.Fprintf(&b, " UL:%s", humanSize(ul, r.human))
|
||||
fmt.Fprintf(&b, " UL:%s", speed(ul, r.human))
|
||||
}
|
||||
if !seeding && s.Total > 0 && dl > 0 {
|
||||
fmt.Fprintf(&b, " ETA:%s", secfmt(etaDuration(s.Total-s.Completed, dl)))
|
||||
@@ -227,7 +227,7 @@ func (r *Reporter) summary(stats []download.Stat) string {
|
||||
stalled++
|
||||
}
|
||||
}
|
||||
line := fmt.Sprintf("%d active DL:%s UL:%s", len(stats)-waiting, humanSize(totDL, r.human), humanSize(totUL, r.human))
|
||||
line := fmt.Sprintf("%d active DL:%s UL:%s", len(stats)-waiting, speed(totDL, r.human), speed(totUL, r.human))
|
||||
if waiting > 0 {
|
||||
line += fmt.Sprintf(" (%d waiting)", waiting)
|
||||
}
|
||||
@@ -254,13 +254,13 @@ func (r *Reporter) rowMetric(s download.Stat, dl, ul int64) (metric, tail string
|
||||
case download.Waiting:
|
||||
return "queued", ""
|
||||
case download.Seeding:
|
||||
return "seeding", "UL:" + humanSize(ul, r.human)
|
||||
return "seeding", "UL:" + speed(ul, r.human)
|
||||
case download.Complete:
|
||||
return "done", ""
|
||||
case download.Errored:
|
||||
return "failed", ""
|
||||
default:
|
||||
metric = "DL:" + humanSize(dl, r.human)
|
||||
metric = "DL:" + speed(dl, r.human)
|
||||
switch {
|
||||
case dl > 0 && s.Total > 0:
|
||||
tail = secfmt(etaDuration(s.Total-s.Completed, dl))
|
||||
|
||||
Reference in New Issue
Block a user