commit eb8d9b94602ac3e3be848f8f5622481ac15be97f Author: Hojun-Cho Date: Fri Jun 19 12:32:14 2026 +0900 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..269047f --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.claude +ref/ +*.torrent +*.got +/got +/ria +bin/ +.torrent.db* +*.part diff --git a/README.md b/README.md new file mode 100644 index 0000000..e0995c5 --- /dev/null +++ b/README.md @@ -0,0 +1,162 @@ +# got + +A small, Rob-Pike-style command-line download tool in Go: one program that +downloads HTTP(S) URLs over several connections and BitTorrent +magnets/torrents, with a focused set of options. + +The design leans on Go's runtime instead of a hand-rolled reactor. There is no +single-threaded event-poll loop: one goroutine per download blocks on real I/O, +a buffered channel of slots bounds concurrency, and `context` carries shutdown. +What remains is the data — and, per Pike, the data is the design. + +## Build + +```sh +go build -o got . +``` + +Requires Go 1.25+. The BitTorrent engine is +[`anacrolix/torrent`](https://github.com/anacrolix/torrent); everything else is +the standard library plus `golang.org/x/{term,time}`. + +## Use + +```sh +# segmented HTTP download, 16 connections, into ./downloads +got -x16 -s16 -d downloads https://example.com/big.iso + +# resume an interrupted download +got -c -x16 https://example.com/big.iso + +# the same file from several mirrors at once (one file, connections spread +# across servers, a dead mirror falls over); -Z downloads them separately instead +got -x4 https://a.example/big.iso https://b.example/big.iso + +# a torrent or magnet; seed for 30 minutes after finishing +got --seed-time=30 ubuntu.torrent +got 'magnet:?xt=urn:btih:...' + +# list the files in a torrent without downloading +got -S -T some.torrent + +# several downloads at once, two running in parallel +got -j2 -i urls.txt +``` + +Run `got --help` (or `--help=http`, `--help=bittorrent`, ...) for the full +option list. + +### Options + +A focused subset of options, with familiar short flags and sensible defaults. + +| flag | meaning | default | +|------|---------|---------| +| `-d, --dir` | output directory | `.` | +| `-o, --out` | output filename (single download) | from URL | +| `-i, --input-file` | read downloads line by line (TAB-separated URLs = mirrors; `-` = stdin) | | +| `-c, --continue` | resume a partial download | false | +| `-x, --max-connection-per-server` | connections to one server (1–16) | 1 | +| `-s, --split` | split a download into N connections | 5 | +| `-Z, --force-sequential` | download each command-line URL separately, not as mirrors | false | +| `-k, --min-split-size` | do not split below this size | 20M | +| `-j, --max-concurrent-downloads` | parallel downloads | 5 | +| `--max-overall-download-limit` | global speed cap | 0 (off) | +| `--max-download-limit` | per-download speed cap | 0 (off) | +| `--retry-wait` | seconds to wait between retries | 0 | +| `--load-cookies` / `--save-cookies` | read/write a Netscape `cookies.txt` | | +| `--conditional-get` | skip the download if the local file is up to date | false | +| `--remote-time` | set the file's mtime from the server | false | +| `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, md5, …) | | +| `--dry-run` | check the file is available but do not download it | false | +| `--ca-certificate` | verify HTTPS against the CA certificates in FILE (PEM) | | +| `--stop` | stop the program after N seconds | 0 (off) | +| `--disable-ipv6` | force IPv4-only connections | false | +| `-T, --torrent-file` | a `.torrent` file | | +| `--seed-time` | minutes to seed after finishing | seed by ratio | +| `--seed-ratio` | stop seeding at this ratio | 1.0 | +| `--listen-port` | port (range) for incoming peers | 6881-6999 | +| `--enable-dht` | use the BitTorrent DHT | true | +| `--select-file` | fetch only these file indexes (`1,3-5`) | all | +| `-S, --show-files` | list torrent files and exit | | +| `--bt-stop-timeout` | give up if no download progress for N s | 0 (off) | +| `--bt-metadata-timeout` | give up if a magnet can't fetch metadata in N s | 60 | +| `--save-session` | on exit, write unfinished downloads to FILE | | +| `-q, --quiet` | no progress readout | false | + +### Resuming across runs + +There is no daemon — resume is a plain file. Point +`--save-session` and `-i` at the same file and add `-c`: unfinished downloads +are loaded at start and the still-unfinished ones are written back on exit. + +```sh +got -c --save-session=got.session -i got.session +``` + +On the first run the session file need not exist. HTTP downloads resume from +their `.got` control file; torrents resume from the data already on disk +(re-checked by the engine). A magnet with no reachable peers no longer hangs: +`--bt-metadata-timeout` bounds the metadata fetch and defaults to 60s (set it to +0 to wait forever). + +## Design + +``` +main classify URIs, wire options, run, report exit code + └─ download the Download interface + the scheduler (-j semaphore, ctx) + ├─ httpdl []Segment + worker pool + WriteAt + JSON resume sidecar + └─ bt anacrolix/torrent behind the Download interface + ├─ cli one flat option table -> hand parser -> layered config + └─ progress one ticker goroutine, pull-snapshot, \r line redraw +``` + +The contract is one small interface: + +```go +type Download interface { + Name() string + Run(ctx context.Context) error // blocks in its own goroutine + Stat() Stat // a value snapshot, lock-free to read +} +``` + +- **HTTP** splits the file into byte-range `Segment`s handed to a pool of + workers. Each worker streams its range straight to one shared file with + `os.File.WriteAt` — safe for concurrent non-overlapping writes, so there is no + shared seek offset and no mutex. Resume is a small JSON sidecar + (`.got`) holding per-segment progress plus validators (length + + ETag/Last-Modified) so a stale file is never trusted. +- **BitTorrent** is `anacrolix/torrent` configured from the CLI options and + driven to completion, then seeded for `--seed-time` minutes or up to + `--seed-ratio`. +- **Options** are one flat `[]Opt` table — the single source of truth for + parsing, validation, defaults and `--help`. Layers apply in order: + built-in defaults, config file, proxy environment, command line. +- **Progress** is a single goroutine that ticks once a second, pulls a snapshot + of the running downloads, derives speeds from the change since the last tick, + and redraws one line with `\r` + erase-to-end-of-line. + +## Status + +Implemented: segmented HTTP(S) with resume — including a foreign/browser-started +partial file (`-c`) — multi-mirror downloads (several URLs, or TAB-grouped `-i` +lines, fetch one file with connections spread across servers and fallover; `-Z` +to download them separately), retries, rate limits, auto-renaming, +`Content-Disposition` naming, conditional GET, cookies +(`--load-cookies`/`--save-cookies`), HTTP basic auth, `--remote-time`, whole-file +checksum verification (`--checksum`), and a `--dry-run` availability check; +BitTorrent download + seeding (magnet and `.torrent`, +DHT, trackers, file selection, show-files, metadata-fetch timeout), seeding that +stops on whichever of `--seed-time`/`--seed-ratio` comes first, and a listen port +chosen from the whole range so a busy port no longer disables BitTorrent; +a command-line interface with documented exit codes, config file, live progress, +and session save/reload for resume across runs. + +Deferred (not implemented yet): FTP/SFTP, Metalink, and the +JSON-RPC server. The architecture leaves room for +each — a new protocol is just another `Download`. One known limitation: the +BitTorrent engine (`anacrolix/torrent`) applies download/upload rate limits +client-wide, so `--max-upload-limit`/`--max-download-limit` act per-run rather +than per-torrent when several torrents run at once (`--max-overall-*` are exact); +DHT/PEX likewise cannot yet be disabled per private torrent. diff --git a/bt/bt.go b/bt/bt.go new file mode 100644 index 0000000..dc42c9d --- /dev/null +++ b/bt/bt.go @@ -0,0 +1,521 @@ +// Package bt downloads a torrent or magnet link. The peer wire protocol, DHT, +// trackers, magnet metadata, piece selection and choking are all carried by +// github.com/anacrolix/torrent; this package's job is to present that engine as +// a download.Download. Within a single run, one shared torrent.Client serves +// every download, so they share one listen port, DHT node and piece-store +// rather than fighting over them. +package bt + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "time" + + missinggo "github.com/anacrolix/missinggo/v2" + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" + "github.com/hanbok/got/download" + "golang.org/x/time/rate" +) + +// ClientConfig holds the run-wide settings used to build the shared client. +type ClientConfig struct { + Dir string + ListenPortSpec string // raw --listen-port spec ("6881-6999,7000"); empty lets the OS choose. + DHT bool + MaxPeers int // max peer connections per torrent (0 = library default) + OverallDown *rate.Limiter // global down limit (may be nil) + OverallUp *rate.Limiter // global up limit (may be nil) + DownLimit int64 // per-run bytes/s, 0 = unlimited (used only without a global limiter) + UpLimit int64 + UserAgent string + DisableIPv6 bool // force IPv4-only (--disable-ipv6) +} + +// ParsePorts expands a --listen-port spec into a flat list of candidate ports. +// The spec is a comma-separated list of single ports and a-b ranges +// (e.g. "6881-6889,6999"). Out-of-range and unparsable entries are +// skipped; an empty or all-bad spec yields nil. +func ParsePorts(spec string) []int { + var ports []int + for _, part := range strings.Split(spec, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + lo, hi, isRange := strings.Cut(part, "-") + a, err := strconv.Atoi(strings.TrimSpace(lo)) + if err != nil { + // Unparsable low bound: drop the whole entry, symmetrically with the + // high bound below (an out-of-range a is still filtered per-port). + continue + } + if !isRange { + if a >= 1 && a <= 65535 { + ports = append(ports, a) + } + continue + } + b, err := strconv.Atoi(strings.TrimSpace(hi)) + if err != nil { + continue + } + for i := a; i <= b; i++ { + if i >= 1 && i <= 65535 { + ports = append(ports, i) + } + } + } + return ports +} + +// NewClient builds the shared anacrolix client from the run-wide config. When a +// listen-port spec is given it tries each candidate port in turn, advancing past +// any that is already in use, and falls back to an OS-chosen port if none bind, +// so a single busy port does not disable BitTorrent entirely. +// +// Two per-torrent features have no equivalent in anacrolix/torrent v1.61.0, +// which exposes only client-wide knobs; both must therefore be configured run-wide +// here rather than attached to an individual Torrent: +// +// 1. No per-Torrent rate limiter. Throttles are read from the client-wide +// DownloadRateLimiter/UploadRateLimiter (no TorrentSpec rate field exists), so +// per-torrent --max-download-limit / --max-upload-limit cannot be +// honored individually; the effective cap is the shared limiter set below. +// 2. No per-Torrent DHT/PEX/LSD toggle for BEP27 private torrents. Those are +// client-wide only (NoDHT, DisablePEX, etc.), so a private torrent's leak +// prevention must come from a run-wide setting (e.g. --enable-dht=false). +func NewClient(cfg ClientConfig) (*torrent.Client, error) { + c := torrent.NewDefaultClientConfig() + c.DataDir = cfg.Dir + if c.DataDir == "" { + c.DataDir = "." + } + c.Seed = true + c.NoDHT = !cfg.DHT + if cfg.MaxPeers > 0 { + c.EstablishedConnsPerTorrent = cfg.MaxPeers + } + if cfg.UserAgent != "" { + c.HTTPUserAgent = cfg.UserAgent + } + c.DisableIPv6 = cfg.DisableIPv6 + switch { + case cfg.OverallDown != nil: + c.DownloadRateLimiter = cfg.OverallDown + case cfg.DownLimit > 0: + c.DownloadRateLimiter = rate.NewLimiter(rate.Limit(cfg.DownLimit), download.LimiterBurst(cfg.DownLimit)) + } + switch { + case cfg.OverallUp != nil: + c.UploadRateLimiter = cfg.OverallUp + case cfg.UpLimit > 0: + c.UploadRateLimiter = rate.NewLimiter(rate.Limit(cfg.UpLimit), download.LimiterBurst(cfg.UpLimit)) + } + + // Candidate ports from the --listen-port spec. + ports := ParsePorts(cfg.ListenPortSpec) + if len(ports) == 0 { + // No usable spec: let the OS choose (anacrolix already retries on 0). + return torrent.NewClient(c) + } + + // Try each candidate; the library only auto-retries when the port is 0, so + // for a fixed busy port it returns an address-in-use error and we advance. + var lastErr error + for _, p := range ports { + c.ListenPort = p + cl, err := torrent.NewClient(c) + if err == nil { + return cl, nil + } + if !isAddrInUse(err) { + return nil, err + } + lastErr = err + } + // Every named port was busy; fall back to an OS-chosen port rather than + // giving up on BitTorrent, so the run keeps going with whatever it can bind. + c.ListenPort = 0 + cl, err := torrent.NewClient(c) + if err != nil { + return nil, fmt.Errorf("all listen ports busy (%w); fallback failed: %v", lastErr, err) + } + return cl, nil +} + +// isAddrInUse reports whether err is an "address already in use" bind failure, +// matching either missinggo's helper or the raw string the library wraps. +func isAddrInUse(err error) bool { + return err != nil && (missinggo.IsAddrInUse(err) || strings.Contains(err.Error(), "address already in use")) +} + +// Options are the per-download settings. +type Options struct { + SeedTimeSet bool // was --seed-time given? + SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding) + SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime + StopTimeout time.Duration // abort if no download progress for this long (0 = off) + MetaTimeout time.Duration // abort a magnet that can't fetch metadata in this long (0 = off) + SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all + CheckIntegrity bool // re-verify data against piece hashes before downloading + DryRun bool // fetch metadata only, then stop without downloading (--dry-run) +} + +// Download implements download.Download for a single torrent or magnet, added +// to a shared client. +type Download struct { + client *torrent.Client + source string // magnet URI or path to a .torrent file + isFile bool + opts Options + + name atomic.Pointer[string] + status int32 // download.Status + + // t is published once Run adds the torrent to the client. It is atomic so + // Stat stays lock-free, exactly like httpdl.Stat and bt's own name field. + t atomic.Pointer[torrent.Torrent] +} + +// New builds a torrent download on the shared client. source is a .torrent path +// when isFile is true, otherwise a magnet URI. +func New(client *torrent.Client, source string, isFile bool, opts Options) *Download { + d := &Download{client: client, source: source, isFile: isFile, opts: opts} + if isFile { + d.setName(strings.TrimSuffix(filepath.Base(source), ".torrent")) + } else { + d.setName(magnetName(source)) + } + return d +} + +func (d *Download) Name() string { + if p := d.name.Load(); p != nil { + return *p + } + return "torrent" +} + +func (d *Download) setName(s string) { d.name.Store(&s) } + +// setStatus stores the lifecycle state with a single atomic op so Stat stays +// lock-free, mirroring httpdl. Errored is set once, by Run's deferred guard, when +// Run returns a non-nil error. +func (d *Download) setStatus(s download.Status) { + atomic.StoreInt32(&d.status, int32(s)) +} + +// id is the process-unique identity for this download: the infohash once known, +// otherwise the source string (a pre-metadata magnet has no infohash yet, and +// two such magnets would otherwise collide on Name). +func (d *Download) id(t *torrent.Torrent) string { + if t != nil { + if h := t.InfoHash(); h != (metainfo.Hash{}) { + return h.HexString() + } + } + return d.source +} + +func (d *Download) Stat() download.Stat { + t := d.t.Load() + + st := download.Status(atomic.LoadInt32(&d.status)) + if t == nil || t.Info() == nil { + return download.Stat{Name: d.Name(), ID: d.id(t), IsBT: true, Status: st, Total: -1} + } + s := t.Stats() + total, completed := t.Length(), t.BytesCompleted() + // With --select-file, report progress against the selected subset, not the + // whole torrent, so the percentage and ETA mean something. + if len(d.opts.SelectFiles) > 0 { + total, completed = 0, 0 + for i, f := range t.Files() { + if d.opts.SelectFiles[i+1] { + total += f.Length() + completed += f.BytesCompleted() + } + } + } + return download.Stat{ + Name: d.Name(), + ID: d.id(t), + IsBT: true, + Status: st, + Total: total, + Completed: completed, + Uploaded: s.BytesWrittenData.Int64(), + Conns: s.ActivePeers, + Seeders: s.ConnectedSeeders, + } +} + +// Run adds the torrent to the shared client, downloads it, then seeds. On any +// failure the named return is non-nil, and the deferred guard flips the status +// to Errored exactly once — so each failure site below just returns its error, +// mirroring httpdl.Run's idiom. +func (d *Download) Run(ctx context.Context) (err error) { + defer func() { + if err != nil { + d.setStatus(download.Errored) + } + }() + d.setStatus(download.Active) + if d.client == nil { + return fmt.Errorf("bittorrent is unavailable") + } + + t, err := d.add() + if err != nil { + return err + } + defer t.Drop() // release this torrent; the client lives on for others + d.t.Store(t) + + if err := d.awaitInfo(ctx, t); err != nil { + return err + } + d.setName(t.Name()) + + // --dry-run: fetching the metadata is all a torrent does under a dry run + // (the content download is cancelled). Report success without downloading. + if d.opts.DryRun { + d.setStatus(download.Complete) + return nil + } + + if err := d.choose(t); err != nil { + return err + } + if d.opts.CheckIntegrity { + if err := t.VerifyDataContext(ctx); err != nil { + return err + } + } + + if err := d.wait(ctx, t); err != nil { + return err + } + + d.setStatus(download.Seeding) + d.seed(ctx, t) + d.setStatus(download.Complete) + return nil +} + +// add hands the source to the shared client. It recovers from the library's +// panics on malformed input (e.g. a magnet with a zero infohash) so a bad +// argument fails as an ordinary error instead of crashing the whole program. +func (d *Download) add() (t *torrent.Torrent, err error) { + defer func() { + if r := recover(); r != nil { + t, err = nil, fmt.Errorf("invalid torrent/magnet: %v", r) + } + }() + if d.isFile { + return d.client.AddTorrentFromFile(d.source) + } + return d.client.AddMagnet(d.source) +} + +// choose applies --select-file: when set, only the listed files are fetched. +// With no selection we ask for everything. It is an error for a selection to +// match no file, so a typo'd index fails loudly instead of "completing" nothing. +func (d *Download) choose(t *torrent.Torrent) error { + if len(d.opts.SelectFiles) == 0 { + t.DownloadAll() + return nil + } + files := t.Files() + matched := 0 + for i, f := range files { + if d.opts.SelectFiles[i+1] { + f.SetPriority(torrent.PiecePriorityNormal) + matched++ + } else { + f.SetPriority(torrent.PiecePriorityNone) + } + } + if matched == 0 { + return fmt.Errorf("--select-file: no file index in 1..%d", len(files)) + } + return nil +} + +// awaitInfo blocks until the torrent metadata arrives. It gives up after +// --bt-metadata-timeout (default 60s) so a magnet with no reachable peers — a +// dead link, or UDP trackers behind a firewall with no DHT — fails fast instead +// of hanging forever. Setting --bt-metadata-timeout=0 disables the timeout and +// waits indefinitely. +func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error { + var deadline <-chan time.Time + if d.opts.MetaTimeout > 0 { + tm := time.NewTimer(d.opts.MetaTimeout) + defer tm.Stop() + deadline = tm.C + } + select { + case <-t.GotInfo(): + return nil + case <-ctx.Done(): + return ctx.Err() + case <-deadline: + return fmt.Errorf("timed out fetching metadata after %s", d.opts.MetaTimeout) + } +} + +// wait blocks until the wanted data is complete, ctx is cancelled, or the +// download stalls past --bt-stop-timeout. +func (d *Download) wait(ctx context.Context, t *torrent.Torrent) error { + tick := time.NewTicker(500 * time.Millisecond) + defer tick.Stop() + + last := t.BytesCompleted() + lastChange := time.Now() + for { + if d.complete(t) { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + if n := t.BytesCompleted(); n != last { + last, lastChange = n, time.Now() + } else if d.opts.StopTimeout > 0 && time.Since(lastChange) > d.opts.StopTimeout { + return fmt.Errorf("no progress for %s", d.opts.StopTimeout) + } + } + } +} + +// complete reports whether everything we asked for is downloaded AND verified. +// For a selection we check each wanted file's pieces are hash-complete rather +// than trusting byte counts, which include not-yet-verified (dirty) bytes. +func (d *Download) complete(t *torrent.Torrent) bool { + if len(d.opts.SelectFiles) == 0 { + return t.Complete().Bool() + } + for i, f := range t.Files() { + if !d.opts.SelectFiles[i+1] { + continue + } + for _, ps := range f.State() { + if !ps.Complete { + return false + } + } + } + return true +} + +// seed keeps the torrent up after completion. Seeding stops as soon as EITHER +// criterion fires: the --seed-time elapses or the --seed-ratio is reached. Both +// are active at once when both are set; with neither set we seed forever (until +// ctx is cancelled). One short-circuit applies: --seed-time=0 means do not seed. +func (d *Download) seed(ctx context.Context, t *torrent.Torrent) { + if d.opts.SeedTimeSet && d.opts.SeedTime == 0 { + return + } + // Arm the time criterion whenever --seed-time was given (independently of the + // ratio criterion below). + var timer <-chan time.Time + if d.opts.SeedTimeSet { + tm := time.NewTimer(d.opts.SeedTime) + defer tm.Stop() + timer = tm.C + } + tick := time.NewTicker(time.Second) + defer tick.Stop() + + // Share ratio is uploaded/downloaded. The denominator is the torrent's + // completed bytes at the moment seeding starts — under --select-file only the + // selected files are ever fetched, so this is already close to the subset, but + // it is not subset-filtered: any boundary/shared pieces count toward it too. + dl := t.BytesCompleted() + for { + select { + case <-ctx.Done(): + return + case <-timer: + return + case <-tick.C: + // Ratio criterion is checked every tick whenever a ratio is set, + // regardless of whether --seed-time is also set. + if d.opts.SeedRatio > 0 && dl > 0 { + st := t.Stats() + up := st.BytesWrittenData.Int64() + if float64(up)/float64(dl) >= d.opts.SeedRatio { + return + } + } + } + } +} + +// FileInfo describes one file inside a torrent, for --show-files. +type FileInfo struct { + Index int + Path string + Length int64 +} + +// Files lists the files in a .torrent without starting a download. +func Files(torrentPath string) ([]FileInfo, error) { + mi, err := metainfo.LoadFromFile(torrentPath) + if err != nil { + return nil, err + } + info, err := mi.UnmarshalInfo() + if err != nil { + return nil, err + } + if len(info.Files) == 0 { + return []FileInfo{{1, info.Name, info.Length}}, nil + } + out := make([]FileInfo, len(info.Files)) + for i, f := range info.Files { + out[i] = FileInfo{i + 1, strings.Join(f.Path, "/"), f.Length} + } + return out, nil +} + +// SourceInfoHash returns the v1 infohash of a .torrent file (isFile) or a magnet +// URI, with ok=false when it can't be known without a network fetch (an HTTP +// URL), the source is malformed, or the v1 infohash is zero — so a BitTorrent +// v2-only source is treated as unique (not deduped). Callers use it to collapse +// two sources that name the same torrent into one download: the shared client +// keys torrents by infohash and does not refcount, so two jobs sharing one +// torrent would have the first to finish Drop it out from under the other. +func SourceInfoHash(source string, isFile bool) (metainfo.Hash, bool) { + var h metainfo.Hash + if isFile { + mi, err := metainfo.LoadFromFile(source) + if err != nil { + return metainfo.Hash{}, false + } + h = mi.HashInfoBytes() + } else { + m, err := metainfo.ParseMagnetUri(source) + if err != nil { + return metainfo.Hash{}, false + } + h = m.InfoHash + } + if h == (metainfo.Hash{}) { + return metainfo.Hash{}, false + } + return h, true +} + +func magnetName(uri string) string { + if m, err := metainfo.ParseMagnetUri(uri); err == nil && m.DisplayName != "" { + return m.DisplayName + } + return "magnet" +} diff --git a/bt/bt_test.go b/bt/bt_test.go new file mode 100644 index 0000000..a386df5 --- /dev/null +++ b/bt/bt_test.go @@ -0,0 +1,47 @@ +package bt + +import ( + "reflect" + "testing" +) + +func TestParsePorts(t *testing.T) { + tests := []struct { + spec string + want []int + }{ + // port spec examples. + {"6881,6885", []int{6881, 6885}}, + {"6881-6999", rangeInts(6881, 6999)}, + {"7000-7001,8000", []int{7000, 7001, 8000}}, + {"6881-6889,6999", append(rangeInts(6881, 6889), 6999)}, + // single bare port. + {"6881", []int{6881}}, + // whitespace and empty fields are tolerated. + {" 6881 , 6885 ", []int{6881, 6885}}, + {"6881,,6885", []int{6881, 6885}}, + // empty / all-bad specs yield nil. + {"", nil}, + {",", nil}, + {"abc", nil}, + // out-of-range ports are dropped. + {"0", nil}, + {"70000", nil}, + // a range that overruns 65535 keeps only the valid part. + {"65534-65540", []int{65534, 65535}}, + } + for _, tt := range tests { + got := ParsePorts(tt.spec) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParsePorts(%q) = %v, want %v", tt.spec, got, tt.want) + } + } +} + +func rangeInts(a, b int) []int { + out := make([]int, 0, b-a+1) + for i := a; i <= b; i++ { + out = append(out, i) + } + return out +} diff --git a/cli/help.go b/cli/help.go new file mode 100644 index 0000000..73cbf02 --- /dev/null +++ b/cli/help.go @@ -0,0 +1,107 @@ +package cli + +import ( + "fmt" + "io" + "strings" +) + +// Version is the program version, overridable at build time via -ldflags. +var Version = "0.1.0" + +// PrintVersion writes the version banner. +func PrintVersion(w io.Writer) { + fmt.Fprintf(w, "got %s — a Rob-Pike-style command-line downloader in Go\n", Version) +} + +// PrintHelp writes usage. A bare -h/--help (empty tag) shows only the Basic +// group, keeping the default terse; --help=all shows every group and +// --help= (basic/http/bittorrent/advanced) filters to one. +func PrintHelp(w io.Writer, tag string) { + all := tag == "all" + // Empty tag means a bare -h/--help: default to the Basic group only. + bare := tag == "" + want, filtered := tagByName(tag) + if bare { + want, filtered = Basic, true + } + if !all && !bare && !filtered { + fmt.Fprintf(w, "got: unknown help tag %q (use all|basic|http|bittorrent|advanced)\n\n", tag) + } + fmt.Fprint(w, `Usage: got [OPTIONS] URI | magnet:... | file.torrent ... + +A multi-protocol download tool: segmented HTTP(S) and BitTorrent. + +`) + for _, group := range []Tag{Basic, HTTP, BitTorrent, Advanced} { + if filtered && !all && group != want { + continue + } + fmt.Fprintf(w, "%s options:\n", title(group.String())) + for i := range options { + o := &options[i] + if o.Tag != group { + continue + } + fmt.Fprintf(w, " %s %s\n", flagLabel(o), o.Help) + } + fmt.Fprintln(w) + } + // Only the full listing (all groups) repeats the meta flags. + if all || !filtered { + fmt.Fprint(w, " -h, --help[=TAG] show help (TAG: all|basic|http|bittorrent|advanced)\n") + fmt.Fprint(w, " -v, --version show version\n\n") + } + if bare { + fmt.Fprint(w, "Showing basic options. Use --help=all for every option, or --help=advanced for advanced ones.\n") + } +} + +// title upper-cases the first letter of a single word. +func title(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func tagByName(name string) (Tag, bool) { + switch name { + case "basic": + return Basic, true + case "http": + return HTTP, true + case "bittorrent", "bt": + return BitTorrent, true + case "advanced": + return Advanced, true + default: + return Basic, false + } +} + +// flagLabel renders the "-x, --long=ARG" column for one option. +func flagLabel(o *Opt) string { + var b strings.Builder + if o.Short != 0 { + fmt.Fprintf(&b, "-%c, ", o.Short) + } else { + b.WriteString(" ") + } + b.WriteString("--" + o.Long) + switch o.Kind { + case Bool: + // no argument shown + case Size: + b.WriteString("=SIZE") + case Int: + b.WriteString("=N") + case Float: + b.WriteString("=RATIO") + case Enum: + b.WriteString("=" + strings.Join(o.Enum, "|")) + default: + b.WriteString("=VAL") + } + return b.String() +} diff --git a/cli/option.go b/cli/option.go new file mode 100644 index 0000000..416c209 --- /dev/null +++ b/cli/option.go @@ -0,0 +1,143 @@ +// Package cli turns command-line arguments and config files into a resolved set +// of options. The whole design is one flat table of option descriptors; help +// text, defaults, parsing and validation all derive from it. The whole option +// model is plain data rather than a class hierarchy. +package cli + +// Kind is the value type of an option. It drives parsing and how the value is +// later read back. +type Kind int + +const ( + Bool Kind = iota // bare flag means true; --flag=false also accepted + Int // integer, optionally bounded by Min/Max + Size // byte size with optional K/M/G suffix + Float // floating point + Str // free-form string + Enum // one of Opt.Enum + List // repeatable; values accumulate, newline-joined +) + +// Tag groups options for --help. +type Tag int + +const ( + Basic Tag = iota + HTTP + BitTorrent + Advanced +) + +func (t Tag) String() string { + switch t { + case Basic: + return "basic" + case HTTP: + return "http" + case BitTorrent: + return "bittorrent" + case Advanced: + return "advanced" + default: + return "?" + } +} + +// Opt describes one option: its names, value kind, default and help text. +type Opt struct { + Long string // long name without dashes, e.g. "max-connection-per-server" + Short byte // short flag byte, e.g. 'x'; 0 if none + Kind Kind + Default string + Help string + Tag Tag + Enum []string // valid values when Kind == Enum + // Min is the inclusive lower Int/Size bound. The zero value enforces a + // floor of 0 (negative inputs are rejected). + Min int64 + // Max is the inclusive upper Int/Size bound; the zero value means no upper + // bound. For Size options Min/Max are byte counts. + Max int64 +} + +// options is the single source of truth: a focused, practical subset. +var options = []Opt{ + // --- basic --- + {Long: "dir", Short: 'd', Kind: Str, Help: "directory to store downloaded files", Tag: Basic}, + {Long: "out", Short: 'o', Kind: Str, Help: "output filename for the download", Tag: Basic}, + {Long: "input-file", Short: 'i', Kind: Str, Help: "read URIs line by line from FILE (- for stdin)", Tag: Basic}, + {Long: "continue", Short: 'c', Kind: Bool, Default: "false", Help: "resume a partially downloaded file", Tag: Basic}, + {Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "max number of parallel downloads", Tag: Basic}, + {Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes (BitTorrent)", Tag: Basic}, + {Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic}, + {Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Basic}, + {Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Basic}, + {Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic}, + {Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: Basic}, + {Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic}, + {Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Basic}, + + // --- http --- + {Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "max connections to one server (1-16)", Tag: HTTP}, + {Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "split a download into N connections; actual connections are min(max-connection-per-server, split), and -x defaults to 1", Tag: HTTP}, + {Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: HTTP}, + {Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP}, + {Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP}, + {Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection timeout in seconds", Tag: HTTP}, + {Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable)", Tag: HTTP}, + {Long: "user-agent", Short: 'U', Kind: Str, Default: "got/1.0", Help: "HTTP User-Agent", Tag: HTTP}, + {Long: "referer", Kind: Str, Help: "HTTP Referer header", Tag: HTTP}, + {Long: "all-proxy", Kind: Str, Help: "proxy for all protocols (host:port)", Tag: HTTP}, + {Long: "check-certificate", Kind: Bool, Default: "true", Help: "verify the server's TLS certificate", Tag: HTTP}, + {Long: "ca-certificate", Kind: Str, Help: "verify HTTPS servers against the CA certificates in FILE (PEM)", Tag: HTTP}, + {Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP}, + {Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP}, + {Long: "load-cookies", Kind: Str, Help: "load Cookies from FILE (Netscape/Mozilla format)", Tag: HTTP}, + {Long: "save-cookies", Kind: Str, Help: "save Cookies to FILE on exit", Tag: HTTP}, + {Long: "conditional-get", Kind: Bool, Default: "false", Help: "download only if the remote file is newer than the local one", Tag: HTTP}, + {Long: "remote-time", Kind: Bool, Default: "false", Help: "set the local file's mtime from the server", Tag: HTTP}, + {Long: "http-user", Kind: Str, Help: "HTTP basic-auth user", Tag: HTTP}, + {Long: "http-passwd", Kind: Str, Help: "HTTP basic-auth password", Tag: HTTP}, + {Long: "lowest-speed-limit", Kind: Size, Default: "0", Help: "abort if speed stays below SIZE bytes/sec (0 = off)", Tag: HTTP}, + {Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256= (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: HTTP}, + + // --- bittorrent --- + {Long: "torrent-file", Short: 'T', Kind: Str, Help: "path to a .torrent file", Tag: BitTorrent}, + {Long: "seed-time", Kind: Float, Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent}, + {Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent}, + {Long: "bt-stop-timeout", Kind: Int, Default: "0", Min: 0, Help: "stop a torrent with no download progress for N seconds (0 = off)", Tag: BitTorrent}, + {Long: "bt-metadata-timeout", Kind: Int, Default: "60", Min: 0, Help: "stop a magnet that can't fetch metadata in N seconds (0 = off)", Tag: BitTorrent}, + {Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent}, + {Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT", Tag: BitTorrent}, + {Long: "bt-max-peers", Kind: Int, Default: "55", Min: 0, Help: "max peers per torrent (0 = unlimited)", Tag: BitTorrent}, + {Long: "max-overall-upload-limit", Kind: Size, Default: "0", Help: "global upload speed limit (0 = unlimited)", Tag: BitTorrent}, + {Long: "max-upload-limit", Short: 'u', Kind: Size, Default: "0", Help: "per-torrent upload speed limit", Tag: BitTorrent}, + {Long: "select-file", Kind: Str, Help: "download only these file indexes, e.g. 1,3-5", Tag: BitTorrent}, + {Long: "follow-torrent", Kind: Bool, Default: "true", Help: "after fetching a .torrent over HTTP, download its content", Tag: BitTorrent}, + + // --- advanced --- + {Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to allocate disk space", Tag: Advanced}, + {Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced}, + {Long: "stop", Kind: Int, Default: "0", Min: 0, Help: "stop the program after N seconds (0 = off)", Tag: Advanced}, + {Long: "disable-ipv6", Kind: Bool, Default: "false", Help: "disable IPv6 (force IPv4-only connections)", Tag: Advanced}, + {Long: "save-session", Kind: Str, Help: "on exit, write unfinished downloads to FILE (reload with -i)", Tag: Advanced}, + {Long: "auto-save-interval", Kind: Int, Default: "60", Min: 0, Max: 600, Help: "save the session file every N seconds (0 = only on exit)", Tag: Advanced}, + {Long: "conf-path", Kind: Str, Help: "path to the config file", Tag: Advanced}, + {Long: "no-conf", Kind: Bool, Default: "false", Help: "do not read a config file", Tag: Advanced}, +} + +// byLong and byShort index the table. Built once at init. +var ( + byLong = map[string]*Opt{} + byShort = map[byte]*Opt{} +) + +func init() { + for i := range options { + o := &options[i] + byLong[o.Long] = o + if o.Short != 0 { + byShort[o.Short] = o + } + } +} diff --git a/cli/options.go b/cli/options.go new file mode 100644 index 0000000..d46d901 --- /dev/null +++ b/cli/options.go @@ -0,0 +1,109 @@ +package cli + +import ( + "fmt" + "strconv" + "strings" +) + +// Options is the resolved configuration: a flat name->value map plus a record +// of which names were explicitly set (so callers can tell a default from a +// chosen value). Values are kept as strings end-to-end and converted only at +// the point of use by the typed getters below. +type Options struct { + vals map[string]string + set map[string]bool +} + +func newOptions() *Options { + return &Options{vals: map[string]string{}, set: map[string]bool{}} +} + +// IsSet reports whether name was given on the command line or in the config +// (as opposed to coming from the built-in default). +func (o *Options) IsSet(name string) bool { return o.set[name] } + +// Str returns the raw string value (empty if unset and no default). +func (o *Options) Str(name string) string { return o.vals[name] } + +// boolWords is the single accepted vocabulary for boolean options, mapping each +// recognised spelling to its truth value. The Bool reader, truthy, and +// validate() all consult it, so exactly the words that validate are honoured +// (no "accepted by the reader but rejected by validate" surprises like --x=on). +var boolWords = map[string]bool{ + "true": true, "yes": true, "1": true, "on": true, + "false": false, "no": false, "0": false, "off": false, +} + +// boolWord reports a value's truth and whether it is a recognised boolean word. +func boolWord(s string) (val, ok bool) { + val, ok = boolWords[strings.ToLower(strings.TrimSpace(s))] + return val, ok +} + +// Bool reports whether the value is a truthy boolean word. +func (o *Options) Bool(name string) bool { + val, _ := boolWord(o.vals[name]) + return val +} + +// Int returns the value as an int, or 0 if empty/invalid. +func (o *Options) Int(name string) int { return int(o.Int64(name)) } + +// Int64 returns the value as an int64, or 0 if empty/invalid. +func (o *Options) Int64(name string) int64 { + n, _ := strconv.ParseInt(strings.TrimSpace(o.vals[name]), 10, 64) + return n +} + +// Float returns the value as a float64, or 0 if empty/invalid. +func (o *Options) Float(name string) float64 { + f, _ := strconv.ParseFloat(strings.TrimSpace(o.vals[name]), 64) + return f +} + +// Size returns a byte count parsed from a value like "20M" or "512K". +func (o *Options) Size(name string) int64 { + n, _ := parseSize(o.vals[name]) + return n +} + +// List returns a repeatable option's accumulated values. +func (o *Options) List(name string) []string { + v := o.vals[name] + if v == "" { + return nil + } + return strings.Split(v, "\n") +} + +// parseSize converts a size string into a byte count: the first 'K'/'k' or +// 'M'/'m' in the string selects the multiplier (1024 or 1024*1024) and +// everything from that byte on is discarded; with no such unit the whole string +// is the byte count. There is no gigabyte unit, so "1G" is rejected; "1Mi" and +// "10MB" are 1M and 10M (the trailing bytes are dropped). An empty string is 0 +// bytes; a negative value is rejected. +func parseSize(s string) (int64, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + mult := int64(1) + if i := strings.IndexAny(s, "KkMm"); i >= 0 { + if c := s[i]; c == 'M' || c == 'm' { + mult = 1 << 20 + } else { + mult = 1 << 10 + } + s = s[:i] + } + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return 0, fmt.Errorf("bad size %q", s) + } + // Size parsing rejects negative sizes outright. + if n < 0 { + return 0, fmt.Errorf("negative size %q", s) + } + return n * mult, nil +} diff --git a/cli/parse.go b/cli/parse.go new file mode 100644 index 0000000..8530c44 --- /dev/null +++ b/cli/parse.go @@ -0,0 +1,311 @@ +package cli + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// Action tells main what the user asked for. +type Action int + +const ( + Run Action = iota + ShowHelp + ShowVersion +) + +// Result is everything Parse extracts from the command line. +type Result struct { + Action Action + HelpTag string // when Action == ShowHelp; "" means all + Options *Options + URIs []string +} + +// Parse resolves args into options and URIs, applying the layered override +// order: built-in defaults, then the config file, then proxy +// environment variables, then the command line (last wins). +func Parse(args []string) (*Result, error) { + cmdVals, uris, action, helpTag, err := parseArgs(args) + if err != nil { + return nil, err + } + if action != Run { + return &Result{Action: action, HelpTag: helpTag}, nil + } + + opts := newOptions() + applyDefaults(opts) + + if !truthy(cmdVals["no-conf"]) { + explicit := cmdVals["conf-path"] + path, fromUser := confPath(explicit) + if err := applyConfig(opts, path, fromUser); err != nil { + return nil, err + } + } + applyEnv(opts) + + for name, val := range cmdVals { + if err := set(opts, name, val); err != nil { + return nil, err + } + } + return &Result{Action: Run, Options: opts, URIs: uris}, nil +} + +// parseArgs walks the argument list once. It recognises --long, --long=val, +// -x, -xval, -x val and short bool bundles like -cq. Like getopt_long, it +// permutes: a non-flag token becomes a URI but scanning continues so flags may +// follow URIs. A bare "--" is a hard terminator forcing the rest to URIs. +// --help/-h and --version/-v short-circuit. Values are returned raw; layering +// happens in Parse. +func parseArgs(args []string) (vals map[string]string, uris []string, action Action, helpTag string, err error) { + vals = map[string]string{} + for i := 0; i < len(args); i++ { + a := args[i] + switch { + case a == "--": // end of options: the rest are URIs + uris = append(uris, args[i+1:]...) + return + case a == "-" || !strings.HasPrefix(a, "-"): // URI (or stdin); keep scanning + uris = append(uris, a) + continue + case strings.HasPrefix(a, "--"): + name, val, hasVal := strings.Cut(a[2:], "=") + if name == "help" { + return vals, uris, ShowHelp, val, nil + } + if name == "version" { + return vals, uris, ShowVersion, "", nil + } + o := byLong[name] + if o == nil { + return nil, nil, Run, "", fmt.Errorf("unknown option --%s", name) + } + if !hasVal { + if o.Kind == Bool { + val = "true" + } else { + if i+1 >= len(args) { + return nil, nil, Run, "", fmt.Errorf("option --%s needs a value", name) + } + i++ + val = args[i] + } + } + accumulate(vals, o, val) + default: // short flags: -x, -xval, -x val, bundles -cq + j := 1 + for j < len(a) { + c := a[j] + if c == 'h' { + return vals, uris, ShowHelp, "", nil + } + if c == 'v' { + return vals, uris, ShowVersion, "", nil + } + o := byShort[c] + if o == nil { + return nil, nil, Run, "", fmt.Errorf("unknown option -%c", c) + } + if o.Kind == Bool { + accumulate(vals, o, "true") + j++ + continue + } + // value-taking short: rest of token, else next arg. + val := a[j+1:] + if val == "" { + if i+1 >= len(args) { + return nil, nil, Run, "", fmt.Errorf("option -%c needs a value", c) + } + i++ + val = args[i] + } + accumulate(vals, o, val) + break // consumed the rest of the token as the value + } + } + } + return +} + +// accumulate stores a value, joining repeatable List options with newlines. +func accumulate(vals map[string]string, o *Opt, val string) { + if o.Kind == List { + if prev, ok := vals[o.Long]; ok { + vals[o.Long] = prev + "\n" + val + return + } + } + vals[o.Long] = val +} + +func applyDefaults(o *Options) { + for i := range options { + opt := &options[i] + if opt.Default != "" { + o.vals[opt.Long] = opt.Default + } + } +} + +// applyConfig reads "name=value" lines (with # comments) into o. A missing +// file is silently ignored for the default path, but is fatal when the user +// named it explicitly via --conf-path: an explicitly requested config file +// that cannot be read is an error. +func applyConfig(o *Options, path string, explicit bool) error { + if path == "" { + return nil + } + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) && !explicit { + return nil // a missing default config file is not an error + } + return err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name, val, _ := strings.Cut(line, "=") + name = strings.TrimSpace(strings.TrimPrefix(name, "--")) + opt := byLong[name] + if opt == nil { + return fmt.Errorf("%s: unknown option %q", path, name) + } + if err := set(o, name, strings.TrimSpace(val)); err != nil { + return err + } + } + return sc.Err() +} + +// applyEnv folds the conventional proxy environment variables into all-proxy +// unless it was already set explicitly. +func applyEnv(o *Options) { + if o.set["all-proxy"] { + return + } + for _, key := range []string{"all_proxy", "https_proxy", "http_proxy"} { + if v := os.Getenv(key); v != "" { + o.vals["all-proxy"] = v + return + } + } +} + +// set validates a value against its option descriptor and stores it, marking +// the option as explicitly set. +func set(o *Options, name, val string) error { + opt := byLong[name] + if opt == nil { + return fmt.Errorf("unknown option %q", name) + } + if err := validate(opt, val); err != nil { + return err + } + if opt.Kind == List { + if prev, ok := o.vals[name]; ok && o.set[name] { + val = prev + "\n" + val + } + } + o.vals[name] = val + o.set[name] = true + return nil +} + +func validate(o *Opt, val string) error { + switch o.Kind { + case Bool: + // Booleans accept true/false plus the yes/no/1/0/on/off spellings, but + // nothing else, so e.g. --enable-dht=flase is a parse error. The accepted + // set is boolWords, the same vocabulary the readers honour. + if _, ok := boolWord(val); !ok { + return fmt.Errorf("--%s: %q is not a boolean (true/false)", o.Long, val) + } + case Int: + n, err := strconv.ParseInt(strings.TrimSpace(val), 10, 64) + if err != nil { + return fmt.Errorf("--%s: %q is not an integer", o.Long, val) + } + if err := checkBounds(o, n); err != nil { + return err + } + case Size: + n, err := parseSize(val) + if err != nil { + return fmt.Errorf("--%s: %v", o.Long, err) + } + if err := checkBounds(o, n); err != nil { + return err + } + case Float: + // seed-time and seed-ratio are both rates/durations that must be + // non-negative; a negative SeedTime would fire the seed timer immediately + // and a negative SeedRatio is silently ignored. Reject negatives here. + // Opt.Min/Max are int64, so we only enforce the floor, not full bounds. + f, err := strconv.ParseFloat(strings.TrimSpace(val), 64) + if err != nil { + return fmt.Errorf("--%s: %q is not a number", o.Long, val) + } + if f < 0 { + return fmt.Errorf("--%s: %v below minimum 0", o.Long, f) + } + case Enum: + for _, e := range o.Enum { + if val == e { + return nil + } + } + return fmt.Errorf("--%s: %q not one of %s", o.Long, val, strings.Join(o.Enum, "|")) + } + return nil +} + +// checkBounds enforces an Int/Size option's numeric range. Min defaults to a +// floor of 0 (negative sizes and counts are rejected); a zero Max means "no +// upper bound". For Size options n and the bounds are byte counts. +func checkBounds(o *Opt, n int64) error { + if n < o.Min { + return fmt.Errorf("--%s: %d below minimum %d", o.Long, n, o.Min) + } + if o.Max > 0 && n > o.Max { + return fmt.Errorf("--%s: %d above maximum %d", o.Long, n, o.Max) + } + return nil +} + +// confPath picks the config file and reports whether the user named it. An +// explicit --conf-path wins (and is treated as user-requested, so a missing +// file is fatal). Otherwise it falls back to $XDG_CONFIG_HOME/got/got.conf, +// then ~/.config/got/got.conf. +func confPath(explicit string) (path string, fromUser bool) { + if explicit != "" { + return explicit, true + } + // XDG_CONFIG_HOME is only honoured when it is an absolute path. + if x := os.Getenv("XDG_CONFIG_HOME"); filepath.IsAbs(x) { + return filepath.Join(x, "got", "got.conf"), false + } + home, err := os.UserHomeDir() + if err != nil { + return "", false + } + return filepath.Join(home, ".config", "got", "got.conf"), false +} + +func truthy(s string) bool { + val, _ := boolWord(s) + return val +} diff --git a/cli/parse_test.go b/cli/parse_test.go new file mode 100644 index 0000000..da23605 --- /dev/null +++ b/cli/parse_test.go @@ -0,0 +1,231 @@ +package cli + +import ( + "path/filepath" + "testing" +) + +func TestParseArgs(t *testing.T) { + tests := []struct { + name string + args []string + action Action + wantVal map[string]string + uris []string + }{ + {"short attached", []string{"-x16", "https://e.com"}, Run, map[string]string{"max-connection-per-server": "16"}, []string{"https://e.com"}}, + {"long equals + bool", []string{"--split=8", "-c", "u"}, Run, map[string]string{"split": "8", "continue": "true"}, []string{"u"}}, + {"long separate value", []string{"--out", "f.bin", "u"}, Run, map[string]string{"out": "f.bin"}, []string{"u"}}, + {"bool bundle", []string{"-cq", "u"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"u"}}, + {"repeatable header", []string{"--header", "A: 1", "--header", "B: 2", "u"}, Run, map[string]string{"header": "A: 1\nB: 2"}, []string{"u"}}, + {"two uris", []string{"a", "b"}, Run, map[string]string{}, []string{"a", "b"}}, + {"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}}, + {"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}}, + {"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}}, + {"bt metadata timeout", []string{"--bt-metadata-timeout=120", "magnet:x"}, Run, map[string]string{"bt-metadata-timeout": "120"}, []string{"magnet:x"}}, + {"help", []string{"--help"}, ShowHelp, nil, nil}, + {"help short", []string{"-h"}, ShowHelp, nil, nil}, + {"version", []string{"-v"}, ShowVersion, nil, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + vals, uris, action, _, err := parseArgs(tc.args) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if action != tc.action { + t.Fatalf("action = %v, want %v", action, tc.action) + } + if tc.action != Run { + return + } + for k, want := range tc.wantVal { + if vals[k] != want { + t.Errorf("vals[%q] = %q, want %q", k, vals[k], want) + } + } + if len(uris) != len(tc.uris) { + t.Fatalf("uris = %v, want %v", uris, tc.uris) + } + for i := range uris { + if uris[i] != tc.uris[i] { + t.Errorf("uris[%d] = %q, want %q", i, uris[i], tc.uris[i]) + } + } + }) + } +} + +func TestParseArgsErrors(t *testing.T) { + for _, args := range [][]string{ + {"--unknown-flag"}, + {"-Q"}, // -Q is not a defined short flag + {"--out"}, // missing value + } { + if _, _, _, _, err := parseArgs(args); err == nil { + t.Errorf("parseArgs(%v) = nil error, want error", args) + } + } +} + +// got bounds the magnet metadata fetch by default rather than waiting forever; +// guard the default value and its zero floor. +func TestBTMetadataTimeoutDefault(t *testing.T) { + o, ok := byLong["bt-metadata-timeout"] + if !ok { + t.Fatal("bt-metadata-timeout option is missing") + } + if o.Default != "60" { + t.Errorf("bt-metadata-timeout default = %q, want %q", o.Default, "60") + } + if o.Min != 0 { + t.Errorf("bt-metadata-timeout Min = %d, want 0 (reject negatives, allow 0=off)", o.Min) + } +} + +func TestParseSize(t *testing.T) { + tests := []struct { + in string + want int64 + }{ + {"", 0}, + {"100", 100}, + {"512K", 512 << 10}, + {"20M", 20 << 20}, + {"1024M", 1 << 30}, // there is no G unit; the 1G bound is typed as 1024M + {"1Mi", 1 << 20}, // the first K/M is taken and the rest discarded + {"10MB", 10 << 20}, + {"2k", 2 << 10}, + } + for _, tc := range tests { + got, err := parseSize(tc.in) + if err != nil { + t.Errorf("parseSize(%q) error: %v", tc.in, err) + continue + } + if got != tc.want { + t.Errorf("parseSize(%q) = %d, want %d", tc.in, got, tc.want) + } + } +} + +func TestValidateRange(t *testing.T) { + x := byLong["max-connection-per-server"] + if err := validate(x, "16"); err != nil { + t.Errorf("validate 16: %v", err) + } + if err := validate(x, "99"); err == nil { + t.Errorf("validate 99: want out-of-range error") + } + a := byLong["file-allocation"] + if err := validate(a, "bogus"); err == nil { + t.Errorf("validate enum bogus: want error") + } +} + +func TestValidateIntFloor(t *testing.T) { + // Min:0 now enforces a floor of 0: negatives are rejected. + m := byLong["max-tries"] // Min 0, "0 = unlimited" runtime meaning preserved + if err := validate(m, "0"); err != nil { + t.Errorf("validate max-tries 0: %v, want nil (0 stays a valid input)", err) + } + if err := validate(m, "-1"); err == nil { + t.Errorf("validate max-tries -1: want below-minimum error") + } +} + +func TestValidateFloatNonNegative(t *testing.T) { + // Float options (seed-time, seed-ratio) reject negatives: a negative + // SeedTime would fire the seed timer immediately and a negative SeedRatio + // is silently ignored, so neither is a meaningful input. + for _, name := range []string{"seed-ratio", "seed-time"} { + o := byLong[name] + if err := validate(o, "-1"); err == nil { + t.Errorf("validate %s -1: want below-minimum error", name) + } + if err := validate(o, "0"); err != nil { + t.Errorf("validate %s 0: %v, want nil", name, err) + } + if err := validate(o, "1.5"); err != nil { + t.Errorf("validate %s 1.5: %v, want nil", name, err) + } + } +} + +func TestValidateBool(t *testing.T) { + b := byLong["enable-dht"] + for _, ok := range []string{"true", "false", "yes", "no", "1", "0", "on", "off"} { + if err := validate(b, ok); err != nil { + t.Errorf("validate bool %q: %v", ok, err) + } + } + if err := validate(b, "flase"); err == nil { + t.Errorf("validate bool flase: want error") + } +} + +func TestValidateSizeBounds(t *testing.T) { + k := byLong["min-split-size"] // 1M..1024M (no G unit) + if err := validate(k, "20M"); err != nil { + t.Errorf("validate min-split-size 20M: %v", err) + } + if err := validate(k, "512K"); err == nil { + t.Errorf("validate min-split-size 512K: want below-minimum error") + } + // 1025M exceeds the 1<<30 cap and still parses, so it tests the bounds path + // (where "2G" would have failed earlier as an unknown unit). + if err := validate(k, "1025M"); err == nil { + t.Errorf("validate min-split-size 1025M: want above-maximum error") + } +} + +func TestParseSizeNegative(t *testing.T) { + if _, err := parseSize("-1"); err == nil { + t.Errorf("parseSize(-1): want error (size parsing rejects negative values)") + } + if _, err := parseSize("-5M"); err == nil { + t.Errorf("parseSize(-5M): want error") + } +} + +// size parsing knows only K and M; a "G" suffix is not a unit and must be +// rejected rather than silently treated as 2^30. +func TestParseSizeNoGigaUnit(t *testing.T) { + for _, in := range []string{"1G", "1g", "1Gi", "5GB"} { + if _, err := parseSize(in); err == nil { + t.Errorf("parseSize(%q): want error (no G unit)", in) + } + } +} + +func TestExplicitConfMissingFatal(t *testing.T) { + missing := filepath.Join(t.TempDir(), "no-such-file.conf") + // Default (non-explicit) missing path is silently ignored. + if err := applyConfig(newOptions(), missing, false); err != nil { + t.Errorf("applyConfig default missing: %v, want nil", err) + } + // Explicit missing path is fatal. + if err := applyConfig(newOptions(), missing, true); err == nil { + t.Errorf("applyConfig explicit missing: want error") + } +} + +func TestOptionsGetters(t *testing.T) { + o := newOptions() + o.vals["split"] = "8" + o.vals["min-split-size"] = "20M" + o.vals["continue"] = "true" + o.vals["seed-ratio"] = "1.5" + if o.Int("split") != 8 { + t.Errorf("Int split = %d", o.Int("split")) + } + if o.Size("min-split-size") != 20<<20 { + t.Errorf("Size min-split-size = %d", o.Size("min-split-size")) + } + if !o.Bool("continue") { + t.Errorf("Bool continue = false") + } + if o.Float("seed-ratio") != 1.5 { + t.Errorf("Float seed-ratio = %v", o.Float("seed-ratio")) + } +} diff --git a/download/download.go b/download/download.go new file mode 100644 index 0000000..9b008f0 --- /dev/null +++ b/download/download.go @@ -0,0 +1,69 @@ +// Package download defines the core abstraction of the program: a Download is +// a thing you can Run and ask for a Stat snapshot. Where a single-threaded +// Command/event-poll reactor would express this, we use one +// goroutine per Download blocking on real I/O. Following Rob Pike, the data +// (Stat) and the interface are kept small; the algorithms fall out of them. +package download + +import "context" + +// Status is the coarse lifecycle state of a download, using the +// active/waiting/complete/error vocabulary. +type Status int + +const ( + Waiting Status = iota + Active + Seeding + Complete + Errored +) + +func (s Status) String() string { + switch s { + case Waiting: + return "waiting" + case Active: + return "active" + case Seeding: + return "seeding" + case Complete: + return "complete" + case Errored: + return "error" + default: + return "unknown" + } +} + +// Stat is a snapshot of a download's progress. Every field is a plain value so +// a renderer running in another goroutine can copy it without sharing memory. +// Completed and Uploaded are cumulative byte counters; the renderer derives +// speeds from successive snapshots rather than each download tracking its own. +type Stat struct { + Name string + // ID is a process-unique stable identity for this download. Name can + // collide (two pre-metadata magnets both show their source string), so the + // renderer keys its per-download speed samples on ID, not Name. + ID string + IsBT bool // true for BitTorrent; progress always shows CN and shows SD for torrents + Status Status + Total int64 // total bytes, or -1 if not yet known + Completed int64 // bytes downloaded so far + Uploaded int64 // bytes uploaded (BitTorrent), 0 otherwise + Conns int // active connections / peers + Seeders int // connected seeders (BitTorrent only) +} + +// Done reports whether the download has reached a terminal state. +func (s Stat) Done() bool { return s.Status == Complete || s.Status == Errored } + +// A Download is one logical job: a URL, a torrent, a magnet. Run blocks in its +// own goroutine until the work finishes, fails, or ctx is cancelled. Stat may +// be called concurrently at any time and must not block; implementations back +// it with atomics or a briefly-held lock. +type Download interface { + Name() string + Run(ctx context.Context) error + Stat() Stat +} diff --git a/download/engine.go b/download/engine.go new file mode 100644 index 0000000..39c085f --- /dev/null +++ b/download/engine.go @@ -0,0 +1,103 @@ +package download + +import ( + "context" + "sync" +) + +// Result pairs a finished download with its outcome and a final stat snapshot. +type Result struct { + Name string + Index int // position in the slice passed to Run, so callers can restore input order + Err error + Stat Stat +} + +// Engine runs a set of downloads, at most maxConcurrent at a time, and exposes +// a live snapshot of the running ones for the progress renderer. A buffered +// channel of slots enforces the concurrency limit, and one goroutine per +// download replaces an explicit scheduler tick. +type Engine struct { + maxConcurrent int + + mu sync.Mutex + active []Download +} + +// NewEngine returns an engine that runs at most maxConcurrent downloads at once. +func NewEngine(maxConcurrent int) *Engine { + if maxConcurrent < 1 { + maxConcurrent = 1 + } + return &Engine{maxConcurrent: maxConcurrent} +} + +// Run starts every download, honouring the concurrency limit, and returns once +// all of them have finished or ctx is cancelled. There is one Result per +// download; results arrive in completion order, but each Result carries its +// input Index so callers can sort back to the original slice order. +func (e *Engine) Run(ctx context.Context, downloads []Download) []Result { + slots := make(chan struct{}, e.maxConcurrent) + results := make(chan Result, len(downloads)) + + var wg sync.WaitGroup + for i, d := range downloads { + wg.Add(1) + go func(i int, d Download) { + defer wg.Done() + + // Acquire a concurrency slot, or bail if we're shutting down + // before this download ever started. + select { + case slots <- struct{}{}: + case <-ctx.Done(): + results <- Result{Name: d.Name(), Index: i, Err: ctx.Err(), Stat: d.Stat()} + return + } + defer func() { <-slots }() + + e.track(d) + defer e.untrack(d) + + err := d.Run(ctx) + results <- Result{Name: d.Name(), Index: i, Err: err, Stat: d.Stat()} + }(i, d) + } + + wg.Wait() + close(results) + + out := make([]Result, 0, len(downloads)) + for r := range results { + out = append(out, r) + } + return out +} + +// Snapshot returns a Stat for every currently running download. +func (e *Engine) Snapshot() []Stat { + e.mu.Lock() + defer e.mu.Unlock() + out := make([]Stat, 0, len(e.active)) + for _, d := range e.active { + out = append(out, d.Stat()) + } + return out +} + +func (e *Engine) track(d Download) { + e.mu.Lock() + e.active = append(e.active, d) + e.mu.Unlock() +} + +func (e *Engine) untrack(d Download) { + e.mu.Lock() + defer e.mu.Unlock() + for i, x := range e.active { + if x == d { + e.active = append(e.active[:i], e.active[i+1:]...) + return + } + } +} diff --git a/download/limit.go b/download/limit.go new file mode 100644 index 0000000..ac1a372 --- /dev/null +++ b/download/limit.go @@ -0,0 +1,18 @@ +package download + +// LimiterBurst returns the token-bucket burst size for a bytes/sec rate limit, +// as used with golang.org/x/time/rate. The burst is the rate itself, but +// floored at 256 KiB: with burst==rate a tiny limit (say a few KiB/s) would +// hand out only a few bytes per refill, so a single read large enough to fill a +// network buffer could never proceed and throughput would stall well below the +// limit. The floor guarantees every limit still permits one usefully sized read +// while the long-run average stays bounded by the rate. +// +// This is the single source of truth for the burst calculation; httpdl, bt, and +// main call it rather than each keeping a private copy. +func LimiterBurst(bps int64) int { + if bps < 256*1024 { + return 256 * 1024 + } + return int(bps) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..214284d --- /dev/null +++ b/go.mod @@ -0,0 +1,94 @@ +module github.com/hanbok/got + +go 1.25 + +require ( + github.com/anacrolix/missinggo/v2 v2.10.0 + github.com/anacrolix/torrent v1.61.0 + golang.org/x/sys v0.38.0 + golang.org/x/term v0.37.0 + golang.org/x/time v0.14.0 +) + +require ( + github.com/RoaringBitmap/roaring v1.2.3 // indirect + github.com/alecthomas/atomic v0.1.0-alpha2 // indirect + github.com/anacrolix/btree v0.0.0-20251201064447-d86c3fa41bd8 // indirect + github.com/anacrolix/chansync v0.7.0 // indirect + github.com/anacrolix/dht/v2 v2.23.0 // indirect + github.com/anacrolix/envpprof v1.4.0 // indirect + github.com/anacrolix/generics v0.1.1-0.20251125230353-15d98d46693b // indirect + github.com/anacrolix/go-libutp v1.3.2 // indirect + github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect + github.com/anacrolix/missinggo v1.3.0 // indirect + github.com/anacrolix/missinggo/perf v1.0.0 // indirect + github.com/anacrolix/mmsg v1.0.1 // indirect + github.com/anacrolix/multiless v0.4.0 // indirect + github.com/anacrolix/stm v0.5.0 // indirect + github.com/anacrolix/sync v0.5.5-0.20251119100342-d78dd1f686f1 // indirect + github.com/anacrolix/upnp v0.1.4 // indirect + github.com/anacrolix/utp v0.1.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d // indirect + github.com/bits-and-blooms/bitset v1.2.2 // indirect + github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/edsrzf/mmap-go v1.1.0 // indirect + github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect + github.com/go-llsqlite/crawshaw v0.5.6-0.20250312230104-194977a03421 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.6 // indirect + github.com/pion/datachannel v1.5.9 // indirect + github.com/pion/dtls/v3 v3.0.3 // indirect + github.com/pion/ice/v4 v4.0.2 // indirect + github.com/pion/interceptor v0.1.40 // indirect + github.com/pion/logging v0.2.3 // indirect + github.com/pion/mdns/v2 v2.0.7 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.15 // indirect + github.com/pion/rtp v1.8.18 // indirect + github.com/pion/sctp v1.8.33 // indirect + github.com/pion/sdp/v3 v3.0.9 // indirect + github.com/pion/srtp/v3 v3.0.4 // indirect + github.com/pion/stun/v3 v3.0.0 // indirect + github.com/pion/transport/v3 v3.0.7 // indirect + github.com/pion/turn/v4 v4.0.0 // indirect + github.com/pion/webrtc/v4 v4.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/protolambda/ctxlock v0.1.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tidwall/btree v1.8.1 // indirect + github.com/wlynxg/anet v0.0.3 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + golang.org/x/crypto v0.44.0 // indirect + golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/text v0.31.0 // indirect + lukechampine.com/blake3 v1.1.6 // indirect + modernc.org/libc v1.22.3 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.21.1 // indirect + zombiezen.com/go/sqlite v0.13.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5794544 --- /dev/null +++ b/go.sum @@ -0,0 +1,509 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= +crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= +filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= +filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= +github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/assert/v2 v2.0.0-alpha3 h1:pcHeMvQ3OMstAWgaeaXIAL8uzB9xMm2zlxt+/4ml8lk= +github.com/alecthomas/assert/v2 v2.0.0-alpha3/go.mod h1:+zD0lmDXTeQj7TgDgCt0ePWxb0hMC1G+PGTsTCv1B9o= +github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8= +github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anacrolix/btree v0.0.0-20251201064447-d86c3fa41bd8 h1:c02PsmoaChabVqAFm7pqPI1UIkDdDAjUaWa6ZmfxybQ= +github.com/anacrolix/btree v0.0.0-20251201064447-d86c3fa41bd8/go.mod h1:7stWJ39LeusmMI8mjJuhFNRqep//vx0AsaySRoK9or0= +github.com/anacrolix/chansync v0.7.0 h1:wgwxbsJRmOqNjil4INpxHrDp4rlqQhECxR8/WBP4Et0= +github.com/anacrolix/chansync v0.7.0/go.mod h1:DZsatdsdXxD0WiwcGl0nJVwyjCKMDv+knl1q2iBjA2k= +github.com/anacrolix/dht/v2 v2.23.0 h1:EuD17ykTTEkAMPLjBsS5QjGOwuBgLTdQhds6zPAjeVY= +github.com/anacrolix/dht/v2 v2.23.0/go.mod h1:seXRz6HLw8zEnxlysf9ye2eQbrKUmch6PyOHpe/Nb/U= +github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= +github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= +github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= +github.com/anacrolix/envpprof v1.4.0 h1:QHeIcrgHcRChhnxR8l6rlaLlRQx9zd7Q2NII6Zbt83w= +github.com/anacrolix/envpprof v1.4.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= +github.com/anacrolix/generics v0.0.0-20230113004304-d6428d516633/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/generics v0.1.1-0.20251125230353-15d98d46693b h1:Kuvx/A/TTJuT9x8mn7DeGx2KW9tWn1LI8bira67xdT0= +github.com/anacrolix/generics v0.1.1-0.20251125230353-15d98d46693b/go.mod h1:NGehhfeXJPBujPx0s6cstSj8B+TERsTY32Xckfx5ftc= +github.com/anacrolix/go-libutp v1.3.2 h1:WswiaxTIogchbkzNgGHuHRfbrYLpv4o290mlvcx+++M= +github.com/anacrolix/go-libutp v1.3.2/go.mod h1:fCUiEnXJSe3jsPG554A200Qv+45ZzIIyGEvE56SHmyA= +github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= +github.com/anacrolix/log v0.6.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= +github.com/anacrolix/log v0.13.1/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= +github.com/anacrolix/log v0.14.2/go.mod h1:1OmJESOtxQGNMlUO5rcv96Vpp9mfMqXXbe2RdinFLdY= +github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb h1:nGNLCQbxFQZz7/9PXLGQ9GmavI/W+eX66pSwVeUwugU= +github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb/go.mod h1:YjBZbwe2v3RsU7WdoBlVSPVpfKuOAno9SRQ/8tIl+hk= +github.com/anacrolix/lsan v0.0.0-20211126052245-807000409a62/go.mod h1:66cFKPCO7Sl4vbFnAaSq7e4OXtdMhRSBagJGWgmpJbM= +github.com/anacrolix/lsan v0.1.0 h1:TbgB8fdVXgBwrNsJGHtht9+9FepNFu5H7dU8ek6XYAY= +github.com/anacrolix/lsan v0.1.0/go.mod h1:66cFKPCO7Sl4vbFnAaSq7e4OXtdMhRSBagJGWgmpJbM= +github.com/anacrolix/missinggo v0.0.0-20180725070939-60ef2fbf63df/go.mod h1:kwGiTUTZ0+p4vAz3VbAI5a30t2YbvemcmspjKwrAz5s= +github.com/anacrolix/missinggo v1.1.0/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= +github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= +github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= +github.com/anacrolix/missinggo v1.3.0 h1:06HlMsudotL7BAELRZs0yDZ4yVXsHXGi323QBjAVASw= +github.com/anacrolix/missinggo v1.3.0/go.mod h1:bqHm8cE8xr+15uVfMG3BFui/TxyB6//H5fwlq/TeqMc= +github.com/anacrolix/missinggo/perf v1.0.0 h1:7ZOGYziGEBytW49+KmYGTaNfnwUqP1HBsy6BqESAJVw= +github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= +github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= +github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= +github.com/anacrolix/missinggo/v2 v2.10.0 h1:pg0iO4Z/UhP2MAnmGcaMtp5ZP9kyWsusENWN9aolrkY= +github.com/anacrolix/missinggo/v2 v2.10.0/go.mod h1:nCRMW6bRCMOVcw5z9BnSYKF+kDbtenx+hQuphf4bK8Y= +github.com/anacrolix/mmsg v1.0.1 h1:TxfpV7kX70m3f/O7ielL/2I3OFkMPjrRCPo7+4X5AWw= +github.com/anacrolix/mmsg v1.0.1/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= +github.com/anacrolix/multiless v0.4.0 h1:lqSszHkliMsZd2hsyrDvHOw4AbYWa+ijQ66LzbjqWjM= +github.com/anacrolix/multiless v0.4.0/go.mod h1:zJv1JF9AqdZiHwxqPgjuOZDGWER6nyE48WBCi/OOrMM= +github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= +github.com/anacrolix/stm v0.5.0 h1:9df1KBpttF0TzLgDq51Z+TEabZKMythqgx89f1FQJt8= +github.com/anacrolix/stm v0.5.0/go.mod h1:MOwrSy+jCm8Y7HYfMAwPj7qWVu7XoVvjOiYwJmpeB/M= +github.com/anacrolix/sync v0.0.0-20180808010631-44578de4e778/go.mod h1:s735Etp3joe/voe2sdaXLcqDdJSay1O0OPnM0ystjqk= +github.com/anacrolix/sync v0.3.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g= +github.com/anacrolix/sync v0.5.5-0.20251119100342-d78dd1f686f1 h1:oLCfNgEOR3/Z98mSwmwTM1pcqCDb/1zIjxCNn7dzVaE= +github.com/anacrolix/sync v0.5.5-0.20251119100342-d78dd1f686f1/go.mod h1:21cUWerw9eiu/3T3kyoChu37AVO+YFue1/H15qqubS0= +github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= +github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= +github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= +github.com/anacrolix/torrent v1.61.0 h1:vxo+B4SwnoP5AQWbhvnTYIaTgPSX+llYUVuQVsN4Jg8= +github.com/anacrolix/torrent v1.61.0/go.mod h1:yKUKuZSSDdyOsCbuH+rDOpswl/g546gICapdrU7aUmQ= +github.com/anacrolix/upnp v0.1.4 h1:+2t2KA6QOhm/49zeNyeVwDu1ZYS9dB9wfxyVvh/wk7U= +github.com/anacrolix/upnp v0.1.4/go.mod h1:Qyhbqo69gwNWvEk1xNTXsS5j7hMHef9hdr984+9fIic= +github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4= +github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= +github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d h1:2qVb9bsAMtmAfnxXltm+6eBzrrS7SZ52c3SedsulaMI= +github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= +github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= +github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= +github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= +github.com/go-llsqlite/crawshaw v0.5.6-0.20250312230104-194977a03421 h1:GClwZI0at7xwV0TpgUMTYr/DoTE7TJZ/tc29LcPcs7o= +github.com/go-llsqlite/crawshaw v0.5.6-0.20250312230104-194977a03421/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pion/datachannel v1.5.9 h1:LpIWAOYPyDrXtU+BW7X0Yt/vGtYxtXQ8ql7dFfYUVZA= +github.com/pion/datachannel v1.5.9/go.mod h1:kDUuk4CU4Uxp82NH4LQZbISULkX/HtzKa4P7ldf9izE= +github.com/pion/dtls/v3 v3.0.3 h1:j5ajZbQwff7Z8k3pE3S+rQ4STvKvXUdKsi/07ka+OWM= +github.com/pion/dtls/v3 v3.0.3/go.mod h1:weOTUyIV4z0bQaVzKe8kpaP17+us3yAuiQsEAG1STMU= +github.com/pion/ice/v4 v4.0.2 h1:1JhBRX8iQLi0+TfcavTjPjI6GO41MFn4CeTBX+Y9h5s= +github.com/pion/ice/v4 v4.0.2/go.mod h1:DCdqyzgtsDNYN6/3U8044j3U7qsJ9KFJC92VnOWHvXg= +github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= +github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= +github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= +github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= +github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= +github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= +github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= +github.com/pion/rtp v1.8.18 h1:yEAb4+4a8nkPCecWzQB6V/uEU18X1lQCGAQCjP+pyvU= +github.com/pion/rtp v1.8.18/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= +github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw= +github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= +github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ= +github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= +github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= +github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= +github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= +github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= +github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= +github.com/pion/webrtc/v4 v4.0.0 h1:x8ec7uJQPP3D1iI8ojPAiTOylPI7Fa7QgqZrhpLyqZ8= +github.com/pion/webrtc/v4 v4.0.0/go.mod h1:SfNn8CcFxR6OUVjLXVslAQ3a3994JhyE3Hw1jAuqEto= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/protolambda/ctxlock v0.1.0 h1:rCUY3+vRdcdZXqT07iXgyr744J2DU2LCBIXowYAjBCE= +github.com/protolambda/ctxlock v0.1.0/go.mod h1:vefhX6rIZH8rsg5ZpOJfEDYQOppZi19SfPiGOFrNnwM= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= +github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA= +github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/wlynxg/anet v0.0.3 h1:PvR53psxFXstc12jelG6f1Lv4MWqE0tI76/hHGjh9rg= +github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0= +golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= +lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= +modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.21.1 h1:GyDFqNnESLOhwwDRaHGdp2jKLDzpyT/rNLglX3ZkMSU= +modernc.org/sqlite v1.21.1/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI= +zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o= +zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4= diff --git a/httpdl/alloc_linux.go b/httpdl/alloc_linux.go new file mode 100644 index 0000000..89d3979 --- /dev/null +++ b/httpdl/alloc_linux.go @@ -0,0 +1,27 @@ +//go:build linux + +package httpdl + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// allocate reserves disk space for the output file according to the +// --file-allocation mode: none does nothing (sparse), trunc sets the size, +// prealloc/falloc reserve real blocks with fallocate (falling back to a plain +// truncate if the filesystem does not support it). +func allocate(f *os.File, mode string, size int64) error { + switch mode { + case "none": + return nil + case "trunc": + return f.Truncate(size) + default: // prealloc, falloc + if err := unix.Fallocate(int(f.Fd()), 0, 0, size); err != nil { + return f.Truncate(size) + } + return nil + } +} diff --git a/httpdl/alloc_other.go b/httpdl/alloc_other.go new file mode 100644 index 0000000..effd0b6 --- /dev/null +++ b/httpdl/alloc_other.go @@ -0,0 +1,14 @@ +//go:build !linux + +package httpdl + +import "os" + +// allocate is the portable fallback: fallocate is Linux-specific, so on other +// platforms prealloc/falloc degrade to a plain truncate. +func allocate(f *os.File, mode string, size int64) error { + if mode == "none" { + return nil + } + return f.Truncate(size) +} diff --git a/httpdl/ca_test.go b/httpdl/ca_test.go new file mode 100644 index 0000000..8ebd5b0 --- /dev/null +++ b/httpdl/ca_test.go @@ -0,0 +1,22 @@ +package httpdl + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadCAPool(t *testing.T) { + // A missing file is an error, not a silent fall-back to the system roots. + if _, err := loadCAPool(filepath.Join(t.TempDir(), "absent.pem")); err == nil { + t.Errorf("loadCAPool(missing): want error") + } + // A file with no PEM certificates is an error (AppendCertsFromPEM found none). + bad := filepath.Join(t.TempDir(), "bad.pem") + if err := os.WriteFile(bad, []byte("not a certificate\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadCAPool(bad); err == nil { + t.Errorf("loadCAPool(garbage): want error (no certificates)") + } +} diff --git a/httpdl/checksum.go b/httpdl/checksum.go new file mode 100644 index 0000000..e2edd90 --- /dev/null +++ b/httpdl/checksum.go @@ -0,0 +1,82 @@ +package httpdl + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "os" + "strings" +) + +// ErrChecksum reports that a finished download did not match the digest given +// with --checksum. main maps it to exit code 32 via errors.Is, so it +// must stay on the error chain (CONTRACT). +var ErrChecksum = errors.New("checksum mismatch") + +// hashers maps each supported hash-type name to a constructor. These are exactly +// the names accepted in --checksum. +var hashers = map[string]func() hash.Hash{ + "md5": md5.New, + "sha-1": sha1.New, + "sha-224": sha256.New224, + "sha-256": sha256.New, + "sha-384": sha512.New384, + "sha-512": sha512.New, +} + +// parseChecksum splits a "=" spec into a hash constructor +// and the expected lowercase-hex digest. Type and digest are matched +// case-insensitively. An unknown type or a malformed digest is an +// error, so a bad --checksum fails before the download rather than after it. +func parseChecksum(spec string) (func() hash.Hash, string, error) { + typ, want, ok := strings.Cut(spec, "=") + if !ok { + return nil, "", fmt.Errorf("bad --checksum %q: want TYPE=DIGEST", spec) + } + newHash, ok := hashers[strings.ToLower(strings.TrimSpace(typ))] + if !ok { + return nil, "", fmt.Errorf("unsupported checksum type %q (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", typ) + } + want = strings.ToLower(strings.TrimSpace(want)) + if _, err := hex.DecodeString(want); err != nil { + return nil, "", fmt.Errorf("bad --checksum digest %q: not hex", want) + } + if n := newHash().Size() * 2; len(want) != n { + return nil, "", fmt.Errorf("bad --checksum digest %q: %s wants %d hex chars", want, typ, n) + } + return newHash, want, nil +} + +// ValidChecksum reports whether spec is a well-formed --checksum value: a +// supported type and a correctly-sized hex digest. main calls it once at startup +// so a bad spec aborts as a usage error before any download begins, validating +// --checksum at option-parse time. +func ValidChecksum(spec string) error { + _, _, err := parseChecksum(spec) + return err +} + +// verifyFile streams the file at path through newHash and compares the result to +// want, returning ErrChecksum on a mismatch. Streaming keeps memory use constant +// regardless of file size. +func verifyFile(path string, newHash func() hash.Hash, want string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + h := newHash() + if _, err := io.Copy(h, f); err != nil { + return err + } + if got := hex.EncodeToString(h.Sum(nil)); got != want { + return fmt.Errorf("%s: %w (want %s, got %s)", path, ErrChecksum, want, got) + } + return nil +} diff --git a/httpdl/checksum_test.go b/httpdl/checksum_test.go new file mode 100644 index 0000000..0ea7f3b --- /dev/null +++ b/httpdl/checksum_test.go @@ -0,0 +1,60 @@ +package httpdl + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestParseChecksum(t *testing.T) { + // A valid spec parses; the type and digest match case-insensitively. + good := "SHA-256=" + hex.EncodeToString(make([]byte, 32)) + if _, want, err := parseChecksum(good); err != nil || len(want) != 64 { + t.Fatalf("parseChecksum(%q) = %q, %v; want a 64-char digest and nil error", good, want, err) + } + full := hex.EncodeToString(make([]byte, 32)) + bad := map[string]string{ + "no '='": "sha-256", + "unknown type": "sha-999=" + full, + "wrong type name": "sha256=" + full, // the type is spelled sha-256 + "digest not hex": "sha-256=zzzz", + "wrong length": "sha-256=abcd", + } + for name, spec := range bad { + if _, _, err := parseChecksum(spec); err == nil { + t.Errorf("%s: parseChecksum(%q) = nil error, want error", name, spec) + } + } +} + +func TestValidChecksum(t *testing.T) { + good := "sha-256=" + hex.EncodeToString(make([]byte, 32)) + if err := ValidChecksum(good); err != nil { + t.Errorf("ValidChecksum(%q) = %v, want nil", good, err) + } + if err := ValidChecksum("crc32=deadbeef"); err == nil { + t.Errorf("ValidChecksum(crc32): want error (unsupported type)") + } +} + +func TestVerifyFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "f") + data := []byte("the data is the design") + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + + // A matching digest verifies clean. + if err := verifyFile(path, sha256.New, hex.EncodeToString(sum[:])); err != nil { + t.Errorf("verifyFile match: %v", err) + } + // A wrong digest returns ErrChecksum, which main maps to exit code 32. + wrong := hex.EncodeToString(make([]byte, 32)) + if err := verifyFile(path, sha256.New, wrong); !errors.Is(err, ErrChecksum) { + t.Errorf("verifyFile mismatch = %v; want ErrChecksum", err) + } +} diff --git a/httpdl/conditional_test.go b/httpdl/conditional_test.go new file mode 100644 index 0000000..c89341c --- /dev/null +++ b/httpdl/conditional_test.go @@ -0,0 +1,97 @@ +package httpdl + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/hanbok/got/download" +) + +// TestConditionalGet304 checks that --conditional-get sends If-Modified-Since +// for an existing local file and treats a 304 as success without rewriting the +// file. +func TestConditionalGet304(t *testing.T) { + dir := t.TempDir() + out := filepath.Join(dir, "file.bin") + const existing = "already here" + if err := os.WriteFile(out, []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + + gotIfModSince := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-Modified-Since") != "" { + gotIfModSince = true + } + w.WriteHeader(http.StatusNotModified) + })) + defer srv.Close() + + d := New([]string{srv.URL + "/file.bin"}, Config{ + Dir: dir, + Out: "file.bin", + MaxConnPerServer: 1, + ConditionalGet: true, + AllowOverwrite: true, + UserAgent: "got-test", + }) + if err := d.Run(context.Background()); err != nil { + t.Fatalf("conditional-get 304 should succeed, got %v", err) + } + if !gotIfModSince { + t.Fatal("expected an If-Modified-Since header on the request") + } + if d.Stat().Status != download.Complete { + t.Fatalf("status = %v, want Complete", d.Stat().Status) + } + // The file must be untouched: 304 means "up to date". + b, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if string(b) != existing { + t.Fatalf("file was rewritten: %q, want %q", b, existing) + } +} + +// TestConditionalGetSkippedWithControlFile verifies the rule that +// --conditional-get is ignored when a .got control file exists, so an +// in-progress resume is never short-circuited by a 304. +func TestConditionalGetSkippedWithControlFile(t *testing.T) { + dir := t.TempDir() + out := filepath.Join(dir, "file.bin") + if err := os.WriteFile(out, []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + // A control file present means a resume is in flight. + if err := os.WriteFile(controlPath(out), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + + sawIfModSince := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-Modified-Since") != "" { + sawIfModSince = true + } + // Serve a small body so the run can complete. + http.Error(w, "no range", http.StatusOK) + })) + defer srv.Close() + + d := New([]string{srv.URL + "/file.bin"}, Config{ + Dir: dir, + Out: "file.bin", + MaxConnPerServer: 1, + ConditionalGet: true, + AllowOverwrite: true, + UserAgent: "got-test", + }) + _ = d.Run(context.Background()) + if sawIfModSince { + t.Fatal("If-Modified-Since must not be sent when a control file exists") + } +} diff --git a/httpdl/control.go b/httpdl/control.go new file mode 100644 index 0000000..b11b7ee --- /dev/null +++ b/httpdl/control.go @@ -0,0 +1,79 @@ +package httpdl + +import ( + "encoding/json" + "os" + "path/filepath" + "sync/atomic" +) + +// control is the on-disk resume state, a small JSON sidecar next to the output +// file (.got). It is a Go-simple binary-free control file: enough to skip +// finished segments and resume partial ones, plus validators (Total + +// ETag/LastModified) so we never trust a stale file. +type control struct { + URL string `json:"url"` + Total int64 `json:"total"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + Segs []segState `json:"segs"` +} + +type segState struct { + Start int64 `json:"start"` + End int64 `json:"end"` + Written int64 `json:"written"` +} + +func controlPath(out string) string { return out + ".got" } + +// snapshot builds a control record from the live segments. +func snapshot(url string, total int64, etag, lastmod string, segs []seg) control { + c := control{URL: url, Total: total, ETag: etag, LastModified: lastmod, Segs: make([]segState, len(segs))} + for i := range segs { + c.Segs[i] = segState{segs[i].start, segs[i].end, atomic.LoadInt64(&segs[i].written)} + } + return c +} + +// save writes the control file atomically (temp + rename). +func (c control) save(out string) error { + data, err := json.Marshal(c) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return err + } + tmp := controlPath(out) + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, controlPath(out)) +} + +// loadControl reads a control file if it is present and still matches the +// download (same URL, length and validator). It returns nil when there is +// nothing trustworthy to resume from. +func loadControl(out, url string, total int64, etag, lastmod string) *control { + data, err := os.ReadFile(controlPath(out)) + if err != nil { + return nil + } + var c control + if json.Unmarshal(data, &c) != nil { + return nil + } + if c.URL != url || c.Total != total { + return nil + } + if (etag != "" || c.ETag != "") && c.ETag != etag { + return nil + } + if (lastmod != "" || c.LastModified != "") && c.LastModified != lastmod { + return nil + } + return &c +} + +func removeControl(out string) { os.Remove(controlPath(out)) } diff --git a/httpdl/cookies.go b/httpdl/cookies.go new file mode 100644 index 0000000..66db54b --- /dev/null +++ b/httpdl/cookies.go @@ -0,0 +1,177 @@ +package httpdl + +import ( + "bufio" + "fmt" + "math" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +// Cookie handling implements the --load-cookies / --save-cookies options, which +// read and write the Netscape/Mozilla cookies.txt format (one tab-separated +// cookie per line). We back it with the standard net/http/cookiejar so the +// http.Client applies and tracks Set-Cookie headers automatically; the file +// format is just the on-disk representation of that jar. + +// loadCookieJar builds a cookie jar seeded from a Netscape cookies.txt file. A +// missing file yields an empty jar (not an error): there are simply no cookies +// to load. A malformed line is skipped, giving per-line tolerance. +func loadCookieJar(path string) (*cookiejar.Jar, error) { + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + if path == "" { + return jar, nil + } + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return jar, nil // nothing to load yet + } + return nil, fmt.Errorf("load-cookies %s: %w", path, err) + } + defer f.Close() + + // Group cookies by the URL their domain implies, since SetCookies keys on a + // URL. We synthesise a representative URL per (domain, secure, path). + byURL := map[string][]*http.Cookie{} + urls := map[string]*url.URL{} + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + c, u := parseNetscapeCookie(line) + if c == nil { + continue + } + key := u.String() + if _, ok := urls[key]; !ok { + urls[key] = u + } + byURL[key] = append(byURL[key], c) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("load-cookies %s: %w", path, err) + } + for key, cs := range byURL { + jar.SetCookies(urls[key], cs) + } + return jar, nil +} + +// parseNetscapeCookie parses one cookies.txt line into a cookie plus the URL it +// belongs to, or (nil, nil) for a comment, blank or malformed line. The format +// is seven tab-separated fields: +// +// domain includeSubdomains path secure expiry name value +// +// Curl marks HttpOnly cookies with a "#HttpOnly_" line prefix rather than a +// real comment, so we recognise that prefix instead of dropping the line. +// Honouring HttpOnly_ rather than skipping every "#" line never loses a cookie. +func parseNetscapeCookie(line string) (*http.Cookie, *url.URL) { + httpOnly := false + if strings.HasPrefix(line, "#HttpOnly_") { + httpOnly = true + line = strings.TrimPrefix(line, "#HttpOnly_") + } else if strings.HasPrefix(line, "#") { + return nil, nil + } + if strings.TrimSpace(line) == "" { + return nil, nil + } + f := strings.Split(line, "\t") + if len(f) < 6 { + return nil, nil + } + domain := f[0] + hostOnly := !strings.EqualFold(f[1], "TRUE") + path := f[2] + secure := strings.EqualFold(f[3], "TRUE") + name := f[5] + value := "" + if len(f) >= 7 { + value = f[6] + } + if name == "" || domain == "" { + return nil, nil + } + + c := &http.Cookie{ + Name: name, + Value: value, + Path: path, + Secure: secure, + HttpOnly: httpOnly, + } + // A non-host-only cookie (includeSubdomains TRUE) carries a Domain attribute + // so the jar applies it to subdomains; a host-only one leaves Domain empty so + // the jar binds it to the exact request host. + if !hostOnly { + c.Domain = domain + } + // expiry 0 means a session cookie; otherwise it is a Unix timestamp. The jar + // drops already-expired cookies on load. + if exp, err := strconv.ParseInt(strings.TrimSpace(f[4]), 10, 64); err == nil && exp > 0 { + c.Expires = time.Unix(exp, 0) + } + + // The jar needs a URL whose host and scheme match the cookie. A leading dot + // denotes a domain cookie; strip it for the host but keep the cookie Domain + // so the jar treats it as domain-wide. + host := strings.TrimPrefix(domain, ".") + scheme := "http" + if secure { + scheme = "https" + } + u := &url.URL{Scheme: scheme, Host: host, Path: path} + return c, u +} + +// saveCookieJar writes every cookie the jar holds (for the hosts we have +// contacted) back to path in Netscape format, implementing --save-cookies on +// exit. Because net/http/cookiejar exposes cookies only per URL, we ask it for +// the cookies of each host we recorded a request against. +func saveCookieJar(jar *cookiejar.Jar, urls []*url.URL, path string) error { + if path == "" || jar == nil { + return nil + } + var b strings.Builder + b.WriteString("# Netscape HTTP Cookie File\n") + seen := map[string]bool{} + for _, u := range urls { + for _, c := range jar.Cookies(u) { + // Cookies() drops the domain/path/expiry, so reconstruct sensible + // values: host-only domain, root path, far-future expiry. This is the + // best a host-only readback can do; it round-trips the name/value pair + // that authenticated downloads actually need. + key := u.Host + "\x00" + c.Name + if seen[key] { + continue + } + seen[key] = true + expiry := int64(math.MaxInt32) // session-stable far future + line := strings.Join([]string{ + u.Host, "FALSE", "/", + boolToTRUE(u.Scheme == "https"), + strconv.FormatInt(expiry, 10), + c.Name, c.Value, + }, "\t") + b.WriteString(line) + b.WriteByte('\n') + } + } + return os.WriteFile(path, []byte(b.String()), 0o600) +} + +func boolToTRUE(b bool) string { + if b { + return "TRUE" + } + return "FALSE" +} diff --git a/httpdl/cookies_test.go b/httpdl/cookies_test.go new file mode 100644 index 0000000..30602b9 --- /dev/null +++ b/httpdl/cookies_test.go @@ -0,0 +1,126 @@ +package httpdl + +import ( + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" +) + +func TestParseNetscapeCookie(t *testing.T) { + future := time.Now().Add(24 * time.Hour).Unix() + tests := []struct { + name string + line string + wantNil bool + wantName string + wantValue string + wantDom string // expected http.Cookie.Domain (empty for host-only) + wantHTTP bool + }{ + { + name: "domain cookie", + line: ".example.com\tTRUE\t/\tFALSE\t" + itoa(future) + "\tSID\tabc123", + wantName: "SID", + wantValue: "abc123", + wantDom: ".example.com", + }, + { + name: "host-only cookie", + line: "example.com\tFALSE\t/\tTRUE\t" + itoa(future) + "\ttoken\txyz", + wantName: "token", + wantValue: "xyz", + wantDom: "", // host-only: no Domain attribute + }, + { + name: "httponly prefix kept", + line: "#HttpOnly_example.com\tFALSE\t/\tFALSE\t" + itoa(future) + "\tHO\tsecret", + wantName: "HO", + wantValue: "secret", + wantHTTP: true, + }, + { + name: "empty value field", + line: "example.com\tFALSE\t/\tFALSE\t0\tflag\t", + wantName: "flag", + wantValue: "", + }, + { + name: "comment line", + line: "# this is a comment", + wantNil: true, + }, + { + name: "blank line", + line: " ", + wantNil: true, + }, + { + name: "too few fields", + line: "example.com\tFALSE\t/\tFALSE\t0", + wantNil: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c, u := parseNetscapeCookie(tc.line) + if tc.wantNil { + if c != nil { + t.Fatalf("expected nil cookie, got %+v", c) + } + return + } + if c == nil { + t.Fatal("expected a cookie, got nil") + } + if c.Name != tc.wantName || c.Value != tc.wantValue { + t.Fatalf("name/value = %q/%q, want %q/%q", c.Name, c.Value, tc.wantName, tc.wantValue) + } + if c.Domain != tc.wantDom { + t.Fatalf("domain = %q, want %q", c.Domain, tc.wantDom) + } + if c.HttpOnly != tc.wantHTTP { + t.Fatalf("httponly = %v, want %v", c.HttpOnly, tc.wantHTTP) + } + if u == nil { + t.Fatal("expected a URL, got nil") + } + }) + } +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } +func itoa64(n int64) string { return strconv.FormatInt(n, 10) } + +func TestLoadCookieJarMissingFile(t *testing.T) { + // A missing cookies file is not an error: there are simply no cookies to load. + jar, err := loadCookieJar(filepath.Join(t.TempDir(), "nope.txt")) + if err != nil { + t.Fatalf("missing file should not error: %v", err) + } + if jar == nil { + t.Fatal("expected an empty jar, got nil") + } +} + +func TestLoadCookieJarAppliesCookie(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cookies.txt") + future := time.Now().Add(24 * time.Hour).Unix() + data := "# Netscape HTTP Cookie File\n" + + ".example.com\tTRUE\t/\tFALSE\t" + itoa64(future) + "\tSID\tabc123\n" + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + jar, err := loadCookieJar(path) + if err != nil { + t.Fatal(err) + } + req, _ := http.NewRequest(http.MethodGet, "http://www.example.com/file", nil) + got := jar.Cookies(req.URL) + if len(got) != 1 || got[0].Name != "SID" || got[0].Value != "abc123" { + t.Fatalf("jar did not apply the domain cookie to a subdomain: %+v", got) + } +} diff --git a/httpdl/httpdl.go b/httpdl/httpdl.go new file mode 100644 index 0000000..b5ad333 --- /dev/null +++ b/httpdl/httpdl.go @@ -0,0 +1,1270 @@ +// Package httpdl downloads a single HTTP(S) resource over one or more +// connections. The model is deliberately flat: split the file into byte-range +// segments, hand them to a pool of worker goroutines, and have each worker +// stream its range straight to disk with WriteAt (safe for concurrent, +// non-overlapping writes — no shared seek, no mutex). Goroutines blocking on +// real I/O keep the model simple: no segment manager, no piece storage, no +// command reactor. +package httpdl + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "hash" + "io" + "mime" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/hanbok/got/download" + "golang.org/x/time/rate" +) + +const readBuf = 32 * 1024 + +// ErrOutputExists reports that the resolved output file already exists and +// neither --allow-overwrite nor -c was given. main maps it to a distinct exit +// code via errors.Is, so it must stay on the error chain (CONTRACT). +var ErrOutputExists = errors.New("output file already exists") + +// ErrNotFound reports a permanent "not found" HTTP status (e.g. 404). main maps +// it to a distinct exit code via errors.Is, so it must stay on the error chain +// (CONTRACT). +var ErrNotFound = errors.New("not found") + +// ErrTooSlow reports a transfer aborted by --lowest-speed-limit. main maps it to +// exit code 5 via errors.Is, so it must stay on the error chain (CONTRACT). +var ErrTooSlow = errors.New("download speed too low") + +// Config holds everything one HTTP download needs. It is assembled by main from +// the resolved CLI options. +type Config struct { + Dir string + Out string + Split int // --split: total connections across all mirrors + MaxConnPerServer int // --max-connection-per-server: per-host connection cap + MinSplit int64 + Tries int // 0 = unlimited + Timeout time.Duration + FileAlloc string // none | prealloc | trunc | falloc + Continue bool + AllowOverwrite bool + AutoRename bool + Headers []string + UserAgent string + Referer string + Proxy string + CheckCert bool + Limit int64 // per-download bytes/s, 0 = unlimited + OverallLimiter *rate.Limiter // shared across all downloads, may be nil + + // RetryWait is the pause between retries of a transient failure + // (--retry-wait). 0 means retry immediately with no sleep. + RetryWait time.Duration + // AutoSaveInterval controls how often the .got control file is rewritten + // (--auto-save-interval). <=0 falls back to 60s. + AutoSaveInterval time.Duration + + // ConnectTimeout is the time allowed to establish the TCP/TLS connection + // (--connect-timeout, default 60s), distinct from Timeout which + // covers stalls once data is flowing. <=0 falls back to Timeout. + ConnectTimeout time.Duration + // LoadCookies / SaveCookies are paths to a Netscape cookies.txt read into + // the client's jar before the download and written back on exit. + LoadCookies string + SaveCookies string + // ConditionalGet sends If-Modified-Since (from the existing local file's + // mtime) so a 304 reports success without re-downloading + // (--conditional-get). + ConditionalGet bool + // RemoteTime sets the finished file's mtime from the server's Last-Modified + // header (-R / --remote-time). + RemoteTime bool + // HTTPUser / HTTPPasswd enable HTTP Basic auth (--http-user/passwd). + HTTPUser string + HTTPPasswd string + // LowestSpeedLimit aborts a download whose speed stays at or below this many + // bytes/sec for a sustained window (--lowest-speed-limit). 0 = off. + LowestSpeedLimit int64 + // Checksum is a "=" spec (e.g. "sha-256=ab…"); when set, + // the finished file is hashed and compared, failing with ErrChecksum on a + // mismatch (--checksum). Empty means no verification. + Checksum string + // DryRun probes the resource — proving it exists and its size — but downloads + // nothing, then reports success (--dry-run). + DryRun bool + // CACert is a path to a PEM bundle of CA certificates used to verify HTTPS + // servers (--ca-certificate). Empty uses the system roots. + CACert string + // DisableIPv6 forces IPv4-only dialing (--disable-ipv6). + DisableIPv6 bool +} + +// Download implements download.Download for one HTTP(S) file, fetched from one +// or more mirror URLs that serve identical bytes. uris[0] is the primary — it +// names the file and keys the resume sidecar; the rest are fallbacks/extra +// connections. +type Download struct { + uris []string // mirror list; uris[0] is the primary (naming + resume key) + maxConns int // segment workers = min(split, len(uris)*max-connection-per-server) + cfg Config + client *http.Client + limit []*rate.Limiter + jar *cookiejar.Jar // non-nil when cookies are in use, for save-cookies + + // name and out are resolved in Run after the probe response is known (the + // final post-redirect URL and any Content-Disposition can change them), then + // never mutated again, so Stat can read name from another goroutine without a + // lock once Run has set it. They are seeded in New so a Stat before Run still + // has a sensible best-effort name. + name string + out string + initErr error + + // mu guards seenURLs, which records every URL we sent a request to so + // SaveCookies can ask the jar for the cookies of each contacted host. + mu sync.Mutex + seenURLs []*url.URL + + // live counters, read by Stat from any goroutine + total int64 // -1 until known + completed int64 + status int32 // download.Status + conns int32 // active connections +} + +// loadCAPool reads a PEM bundle of CA certificates for verifying HTTPS servers +// (--ca-certificate). An unreadable file or one containing no +// certificates is an error rather than a silent fall-back to the system roots. +func loadCAPool(path string) (*x509.CertPool, error) { + pem, err := os.ReadFile(path) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("ca-certificate %s: no certificates found", path) + } + return pool, nil +} + +// primary is the canonical URL — used for naming and the resume sidecar so they +// stay stable regardless of which mirror served the bytes. +func (d *Download) primary() string { return d.uris[0] } + +// mirror returns the URL for the n-th pick. Choosing by a caller-supplied index +// (segment index, retry attempt) spreads connections across the mirrors and +// rotates on retry, with no shared state — so it stays lock-free. +func (d *Download) mirror(n int) string { return d.uris[n%len(d.uris)] } + +// New builds an HTTP download for uris, which must all serve identical content +// (mirrors). uris[0] is the primary. +func New(uris []string, cfg Config) *Download { + // --connect-timeout bounds only connection establishment; once data flows + // --timeout (the idle guard) takes over. The dial and the TLS handshake each + // get the full connectTimeout (Go applies it per phase), so an HTTPS connect + // can take up to ~2x connectTimeout in the worst case — there is no single + // shared connect budget. + connectTimeout := cfg.ConnectTimeout + if connectTimeout <= 0 { + connectTimeout = cfg.Timeout + } + var initErr error + + // --ca-certificate: verify HTTPS servers against a custom CA bundle instead of + // the system roots. A bad bundle becomes an initErr so Run fails loudly. + tlsCfg := &tls.Config{InsecureSkipVerify: !cfg.CheckCert} + if cfg.CACert != "" { + if pool, err := loadCAPool(cfg.CACert); err != nil { + initErr = err + } else { + tlsCfg.RootCAs = pool + } + } + + // --disable-ipv6: force IPv4 by dialing "tcp4"; a plain "tcp" would let the + // resolver hand back a v6 address. + dialer := &net.Dialer{Timeout: connectTimeout} + dialContext := dialer.DialContext + if cfg.DisableIPv6 { + dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + if network == "tcp" { + network = "tcp4" + } + return dialer.DialContext(ctx, network, addr) + } + } + + // MaxConnsPerHost is the per-host cap (--max-connection-per-server): with + // mirrors the total worker count can exceed it, but no single host does. + perHost := cfg.MaxConnPerServer + if perHost < 1 { + perHost = 1 + } + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxConnsPerHost: perHost, + MaxIdleConnsPerHost: perHost, + ResponseHeaderTimeout: cfg.Timeout, + TLSHandshakeTimeout: connectTimeout, + DialContext: dialContext, + TLSClientConfig: tlsCfg, + } + if cfg.Proxy != "" { + if pu, err := url.Parse(cfg.Proxy); err == nil { + tr.Proxy = http.ProxyURL(pu) + } + } + + // Seed the cookie jar from --load-cookies; a load error becomes an initErr so + // Run surfaces it rather than silently downloading uncredentialed. + var jar *cookiejar.Jar + if cfg.LoadCookies != "" || cfg.SaveCookies != "" { + if j, err := loadCookieJar(cfg.LoadCookies); err != nil { + if initErr == nil { + initErr = err + } + } else { + jar = j + } + } + + var limit []*rate.Limiter + if cfg.OverallLimiter != nil { + limit = append(limit, cfg.OverallLimiter) + } + if cfg.Limit > 0 { + limit = append(limit, rate.NewLimiter(rate.Limit(cfg.Limit), download.LimiterBurst(cfg.Limit))) + } + + // Seed a best-effort name from -o or the primary URL. The real name is + // finalised in Run after probe(), where Content-Disposition and the + // post-redirect URL are known (item [8]); name and out stay immutable then on. + primary := "" + if len(uris) > 0 { + primary = uris[0] + } + name := cfg.Out + if name == "" { + name = nameFromURL(primary) + } + out := filepath.Join(cfg.Dir, name) + + // An empty mirror list can download nothing; surface it as an initErr so Run + // returns cleanly instead of indexing uris[0]. + if len(uris) == 0 && initErr == nil { + initErr = errors.New("no URL to download") + } + // Worker count: --split connections are spread across the mirrors, capped + // at --max-connection-per-server per host, so the ceiling is + // min(split, M*max-conn-per-server). One mirror reduces to min(split, x). + maxConns := cfg.Split + if m := max(1, len(uris)) * perHost; maxConns > m { + maxConns = m + } + if maxConns < 1 { + maxConns = 1 + } + + // Only attach the jar when one was built: a typed-nil *cookiejar.Jar stored + // in the http.Client.Jar interface is non-nil and would panic on use. + client := &http.Client{Transport: tr} + if jar != nil { + client.Jar = jar + } + + return &Download{ + uris: uris, + maxConns: maxConns, + cfg: cfg, + client: client, + limit: limit, + jar: jar, + name: name, + out: out, + total: -1, + initErr: initErr, + } +} + +func (d *Download) Name() string { return d.name } + +// Path returns the resolved output file path (used by --follow-torrent). +func (d *Download) Path() string { return d.out } + +func (d *Download) Stat() download.Stat { + return download.Stat{ + Name: d.name, + ID: d.out, // resolved output path; stable per process (CONTRACT) + IsBT: false, + Status: download.Status(atomic.LoadInt32(&d.status)), + Total: atomic.LoadInt64(&d.total), + Completed: atomic.LoadInt64(&d.completed), + Conns: int(atomic.LoadInt32(&d.conns)), + } +} + +// setStatus stores the lifecycle state with a single atomic op so Stat stays +// lock-free. +func (d *Download) setStatus(s download.Status) { + atomic.StoreInt32(&d.status, int32(s)) +} + +// probeResult carries everything probe() learned, so name finalisation can run +// after the request (the final URL and Content-Disposition are only known then). +type probeResult struct { + total int64 + ranged bool + etag string + lastmod string + cdisp string // raw Content-Disposition header + finalURL *url.URL + notModified bool // server answered 304 to a conditional GET +} + +// Run probes the resource, finalises the output name, decides between a single +// stream and a segmented download, transfers the bytes, and finalises the +// resume file. Errored is set exactly once via the deferred check of err. +func (d *Download) Run(ctx context.Context) (err error) { + // One place sets Errored, keyed on whatever err we return (item [37]). + defer func() { + if err != nil { + d.setStatus(download.Errored) + } + }() + + if d.initErr != nil { + return d.initErr + } + d.setStatus(download.Active) + + pr, err := d.probeRetry(ctx) + if err != nil { + return err + } + // --conditional-get: the server said our local copy is current, so there is + // nothing to download. Report success without touching the file. + if pr.notModified { + d.setStatus(download.Complete) + return nil + } + atomic.StoreInt64(&d.total, pr.total) + + // Finalise name/out now that the post-redirect URL and Content-Disposition + // are known (item [8]), then resolve overwrite/auto-rename (items [2],[13]). + if err = d.resolveName(pr); err != nil { + return err + } + + // --checksum: parse the spec up front so a bad type or digest fails before we + // download rather than after. The finished file is verified below. + var ( + newSum func() hash.Hash + sumWant string + ) + if d.cfg.Checksum != "" { + if newSum, sumWant, err = parseChecksum(d.cfg.Checksum); err != nil { + return err + } + } + + // --dry-run: the probe already proved the resource exists and (above) its + // size, which is all a dry run needs — stop here without downloading. + if d.cfg.DryRun { + d.setStatus(download.Complete) + return nil + } + + // --lowest-speed-limit: cancel the transfer if its speed stays at or below + // the limit past a startup grace. The watcher cancels xferCtx and sets a + // flag so we can return the distinct "too slow" error rather than a bare + // cancellation. + xferCtx, stopGuard, tooSlow := d.speedGuard(ctx) + if !pr.ranged || pr.total <= 0 { + err = d.single(xferCtx, d.out) + } else { + err = d.segmented(xferCtx, d.out, pr.total, pr.etag, pr.lastmod) + } + stopGuard() + if err != nil { + if tooSlow() { + return fmt.Errorf("%s: %w (<= %d bytes/sec)", d.primary(), ErrTooSlow, d.cfg.LowestSpeedLimit) + } + return err + } + // --checksum: verify the finished file before declaring success, so a + // corrupted or tampered download fails (--checksum, exit code 32). + if newSum != nil { + if err = verifyFile(d.out, newSum, sumWant); err != nil { + return err + } + } + // -R / --remote-time: stamp the finished file with the server's mtime. + if d.cfg.RemoteTime && pr.lastmod != "" { + if t, err := http.ParseTime(pr.lastmod); err == nil { + os.Chtimes(d.out, t, t) + } + } + d.setStatus(download.Complete) + return nil +} + +// resolveName settles d.name and d.out after the probe. When -o is absent the +// name comes from Content-Disposition, else the final (post-redirect) URL, else +// the original URL. Overwrite / auto-rename is then resolved against the final +// path, honouring -c so a resumable file is never renamed (items [2],[8],[13]). +func (d *Download) resolveName(pr probeResult) error { + if d.cfg.Out == "" { + name := "" + if cd := pr.cdisp; cd != "" { + name = nameFromContentDisposition(cd) + } + if name == "" && pr.finalURL != nil { + name = nameFromURLPath(pr.finalURL) + } + if name == "" { + name = nameFromURL(d.primary()) + } + d.name = name + d.out = filepath.Join(d.cfg.Dir, name) + } + + // --dry-run writes nothing, so skip overwrite/auto-rename resolution; the name + // was only needed for the report. + if d.cfg.DryRun { + return nil + } + + // -c resumes either our own .got sidecar or a foreign/browser partial file, + // so an existing file is never auto-renamed under -c (item [2]). + if d.cfg.Continue && fileExists(d.out) { + return nil + } + if fileExists(d.out) && !d.cfg.AllowOverwrite { + if d.cfg.AutoRename { + u, err := uniqueName(d.out) + if err != nil { + return err + } + d.out = u + d.name = filepath.Base(u) + return nil + } + return fmt.Errorf("%s: %w (use --allow-overwrite or -c)", d.out, ErrOutputExists) + } + return nil +} + +// backoff waits RetryWait between attempts, but never after the final attempt, +// and returns ctx.Err() if cancelled while waiting (item [20]). +func backoff(ctx context.Context, wait time.Duration, tries, attempt int) error { + if wait <= 0 || (tries != 0 && attempt >= tries-1) { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + return nil + } +} + +// retriesExhausted wraps the final cause once the retry budget is spent, so the +// caller — and the exit-code mapping in main — can still see whether it was DNS, +// a refused connection, or a timeout, rather than an opaque "too many retries". +func retriesExhausted(what string, tries int, cause error) error { + if cause == nil { + return fmt.Errorf("%s: failed", what) + } + // Only mention a count when we actually retried (tries >= 2). tries == 0 + // (unlimited) reaches this only after the loop is broken some other way and + // has no meaningful fixed count, so it too falls through to the bare form. + if tries > 1 { + return fmt.Errorf("%s: %w (after %d tries)", what, cause, tries) + } + return fmt.Errorf("%s: %w", what, cause) +} + +// withRetries runs attempt up to d.cfg.Tries times (0 = unlimited), the shared +// tries/backoff/fatal policy that probe, the single stream, and each segment all +// use so one transient DNS/connect/5xx does not fail the whole download (item +// [18]). It returns immediately on success (nil), on context cancellation, or on +// a fatal{} error (retrying cannot help), sleeps RetryWait between attempts, and +// wraps the final cause with retriesExhausted(what, ...) once the budget is +// spent so the underlying cause stays inspectable. +func (d *Download) withRetries(ctx context.Context, what string, attempt func() error) error { + tries := d.cfg.Tries + var lastErr error + for n := 0; tries == 0 || n < tries; n++ { + if ctx.Err() != nil { + return ctx.Err() + } + err := attempt() + if err == nil || ctx.Err() != nil { + return err + } + lastErr = err + var fe fatal + if errors.As(err, &fe) { + return err // permanent: retrying cannot help + } + if err := backoff(ctx, d.cfg.RetryWait, tries, n); err != nil { + return err + } + } + return retriesExhausted(what, tries, lastErr) +} + +// probeRetry runs probe within the shared retry policy (item [18]). The probe +// result is captured through pr because withRetries only carries an error. +func (d *Download) probeRetry(ctx context.Context) (probeResult, error) { + // Probe mirrors in order (primary first); the first that answers anchors the + // size and validators for the whole download. A dead/404 primary falls over + // to the next mirror. + var lastErr error + for _, u := range d.uris { + if ctx.Err() != nil { + return probeResult{}, ctx.Err() + } + var pr probeResult + err := d.withRetries(ctx, u, func() error { + var perr error + pr, perr = d.probe(ctx, u) + return perr + }) + if err == nil { + return pr, nil + } + lastErr = err + } + return probeResult{}, lastErr +} + +// probe issues a one-byte ranged GET to learn the total size, whether the +// server honours ranges, and the validator headers used for safe resume. It +// also reports the final URL and Content-Disposition for name finalisation. +func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) { + req, err := d.request(ctx, uri, "bytes=0-0") + if err != nil { + return probeResult{}, err + } + // --conditional-get: ask the server to skip the transfer when our local file + // is already current. Only done when there is no .got control file (an + // in-progress resume must not be short-circuited), using the local file's + // mtime for If-Modified-Since. + if d.cfg.ConditionalGet && !fileExists(controlPath(d.out)) { + if fi, err := os.Stat(d.out); err == nil { + req.Header.Set("If-Modified-Since", fi.ModTime().UTC().Format(http.TimeFormat)) + } + } + resp, err := d.client.Do(req) + if err != nil { + return probeResult{}, err + } + defer resp.Body.Close() + io.Copy(io.Discard, io.LimitReader(resp.Body, 1)) + + pr := probeResult{ + etag: resp.Header.Get("ETag"), + lastmod: resp.Header.Get("Last-Modified"), + cdisp: resp.Header.Get("Content-Disposition"), + } + if resp.Request != nil { + pr.finalURL = resp.Request.URL + } + + if resp.StatusCode == http.StatusNotModified { + pr.notModified = true + return pr, nil + } + + switch resp.StatusCode { + case http.StatusPartialContent: + // Content-Range: bytes 0-0/12345 + pr.ranged = true + if _, _, t, ok := parseContentRange(resp.Header.Get("Content-Range")); ok { + pr.total = t + } + return pr, nil + case http.StatusOK: + pr.total = resp.ContentLength + return pr, nil + default: + return probeResult{}, statusError(fmt.Sprintf("%s: %s", uri, resp.Status), resp.StatusCode) + } +} + +// segmented downloads total bytes over d.maxConns workers, with per-segment +// resume from the control file. +func (d *Download) segmented(ctx context.Context, out string, total int64, etag, lastmod string) error { + segs := makeSegments(total, d.cfg.MinSplit, d.maxConns) + if c := d.resumeControl(out, etag, lastmod); c != nil { + segs = segsFromControl(c) + } else if d.cfg.Continue && !fileExists(controlPath(out)) { + // Resume a foreign/browser partial file: no .got sidecar exists, so treat + // the bytes already on disk as a completed prefix and seed the segments + // accordingly (item [2]): open the existing file and mark its bytes done. + if fi, err := os.Stat(out); err == nil { + seedSegmentsFromPrefix(segs, fi.Size()) + } + } + var have int64 + for i := range segs { + have += segs[i].progress() + } + atomic.StoreInt64(&d.completed, have) + + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + f, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return err + } + defer f.Close() + if err := allocate(f, d.cfg.FileAlloc, total); err != nil { + return err + } + // Force the file to exactly total bytes: prealloc/falloc only grow, so an + // existing larger file (overwrite, or a changed resume target) would keep a + // stale tail past the real content otherwise. + if err := f.Truncate(total); err != nil { + return err + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Periodically persist progress so a crash can resume. + saveDone := make(chan struct{}) + go d.saveLoop(ctx, out, total, etag, lastmod, segs, saveDone) + + jobs := make(chan *seg) + go func() { + defer close(jobs) + for i := range segs { + if segs[i].done() { + continue + } + select { + case jobs <- &segs[i]: + case <-ctx.Done(): // stop feeding once we're tearing down + return + } + } + }() + + var ( + wg sync.WaitGroup + errOnce sync.Once + runErr error + ) + for w := 0; w < d.maxConns; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for s := range jobs { + if err := d.fetchSeg(ctx, f, s, total); err != nil { + errOnce.Do(func() { runErr = err; cancel() }) + return + } + } + }() + } + wg.Wait() + cancel() + <-saveDone + + if runErr != nil { + return runErr // keep the control file for a later -c + } + snapshot(d.primary(), total, etag, lastmod, segs).save(out) + removeControl(out) + return nil +} + +// fetchSeg downloads one segment, retrying from its resume point on error. +// Each attempt re-reads s.offset(), so a retry continues from the bytes already +// written rather than restarting the range. +func (d *Download) fetchSeg(ctx context.Context, f *os.File, s *seg, total int64) error { + // Try each mirror in turn for this segment: a transient error retries the + // same mirror under withRetries; an exhausted budget or a per-mirror permanent + // error (a 404, a non-206, a length mismatch) falls over to the next mirror. + // The segment fails only once every mirror is spent. Starting at s.index + // spreads the segments across mirrors round-robin, and each attempt resumes + // from s.offset(). + var lastErr error + for i := range d.uris { + if ctx.Err() != nil { + return ctx.Err() + } + uri := d.mirror(s.index + i) + err := d.withRetries(ctx, fmt.Sprintf("segment %d", s.index), func() error { + if s.done() { + return nil + } + return d.fetchOnce(ctx, f, s, uri, total) + }) + if err == nil { + return nil + } + lastErr = err + } + return lastErr +} + +func (d *Download) fetchOnce(ctx context.Context, f *os.File, s *seg, uri string, total int64) error { + reqCtx, cancel := context.WithCancel(ctx) + defer cancel() + req, err := d.request(reqCtx, uri, fmt.Sprintf("bytes=%d-%d", s.offset(), s.end)) + if err != nil { + return err + } + atomic.AddInt32(&d.conns, 1) + resp, err := d.client.Do(req) + if err != nil { + atomic.AddInt32(&d.conns, -1) + return err + } + defer func() { + resp.Body.Close() + atomic.AddInt32(&d.conns, -1) + }() + if resp.StatusCode != http.StatusPartialContent { + return statusError(fmt.Sprintf("segment %d: expected 206, got %s", s.index, resp.Status), resp.StatusCode) + } + // A mirror must serve the same file as the primary, or its bytes written at + // this offset would corrupt the output. Reject a 206 whose total length + // disagrees with the probe (validate the total length); fatal so withRetries + // stops and fetchSeg falls over to the next mirror instead of writing it. + if _, _, t, ok := parseContentRange(resp.Header.Get("Content-Range")); ok && t != total { + return fatal{fmt.Errorf("segment %d: %s reports length %d, want %d", s.index, uri, t, total)} + } + body, stop := d.idleGuard(resp.Body, cancel) + defer stop() + return d.pump(ctx, f, s, body) +} + +// errIdleTimeout marks a read that stalled past the idle window, so callers can +// report a distinct "timed out" instead of the bare "context canceled" that the +// cancelled request would otherwise surface (item [39]). +var errIdleTimeout = errors.New("idle timeout: no data received") + +// idleReader resets a watchdog timer on every read; if a read stalls longer +// than the idle window the timer fires and cancels the request, so the blocked +// Read returns an error instead of hanging forever on a wedged connection. +type idleReader struct { + r io.Reader + timer *time.Timer + idle time.Duration + fired atomic.Bool // set when the watchdog cancelled the request +} + +func (ir *idleReader) Read(p []byte) (int, error) { + ir.timer.Reset(ir.idle) + n, err := ir.r.Read(p) + ir.timer.Stop() // only the Read itself counts as idle, not write/throttle gaps + if err != nil && ir.fired.Load() { + // The cancellation came from our watchdog, not the caller's ctx, so + // translate the read error into the distinct idle-timeout sentinel. + return n, fmt.Errorf("%w (after %s)", errIdleTimeout, ir.idle) + } + return n, err +} + +// idleGuard wraps body with an idle timeout of d.cfg.Timeout, cancelling the +// request if no data arrives in that window. It returns the reader to use and a +// stop func to call when the transfer is done. +func (d *Download) idleGuard(body io.Reader, cancel context.CancelFunc) (io.Reader, func()) { + if d.cfg.Timeout <= 0 { + return body, func() {} + } + ir := &idleReader{r: body, idle: d.cfg.Timeout} + ir.timer = time.AfterFunc(d.cfg.Timeout, func() { + ir.fired.Store(true) + cancel() + }) + return ir, func() { ir.timer.Stop() } +} + +// speedStartupGrace is how long a download may stay slow before the +// lowest-speed-limit watchdog starts checking it: a startup idle time of +// 10 seconds. +const speedStartupGrace = 10 * time.Second + +// belowSpeed reports whether the bytes transferred between two cumulative +// d.completed samples (prev -> now over one window) put the download at or +// below limit. A negative delta is not a slow second: d.completed is not +// strictly monotonic — singleOnce rewrites it with an absolute StoreInt64 and a +// resumed Range answered by a 200 resets the offset to 0, so the counter can +// jump backward on a retry. Treating that backward jump as "below limit" would +// false-abort a healthy download, so a reset (delta < 0) reports false; the +// caller has already advanced its baseline, so the next full window measures the +// real speed. Only a genuine non-negative delta at or under the limit aborts. +func belowSpeed(prev, now, limit int64) bool { + delta := now - prev + if delta < 0 { + return false // counter reset, not a slow window + } + return delta <= limit +} + +// speedGuard implements --lowest-speed-limit. It returns a child context that +// the watcher cancels (setting a flag) once the measured download speed has +// stayed at or below the limit past the startup grace, plus a stop func and a +// predicate reporting whether the watcher fired. When the limit is 0 it is a +// no-op passthrough. +// +// Speed is sampled from d.completed over a one-second window: a single slow +// second after the grace aborts the download as too slow. +func (d *Download) speedGuard(ctx context.Context) (context.Context, func(), func() bool) { + if d.cfg.LowestSpeedLimit <= 0 { + return ctx, func() {}, func() bool { return false } + } + child, cancel := context.WithCancel(ctx) + var fired atomic.Bool + done := make(chan struct{}) + go func() { + defer close(done) + start := time.Now() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + last := atomic.LoadInt64(&d.completed) + for { + select { + case <-child.Done(): + return + case <-ticker.C: + now := atomic.LoadInt64(&d.completed) + prev := last + last = now + if time.Since(start) < speedStartupGrace { + continue + } + if belowSpeed(prev, now, d.cfg.LowestSpeedLimit) { + fired.Store(true) + cancel() + return + } + } + } + }() + stop := func() { + cancel() + <-done + } + return child, stop, func() bool { return fired.Load() } +} + +// fatal marks an error that retrying cannot fix. Unwrap exposes the wrapped +// error so errors.Is/As traverse a fatal{} — main's exit-code mapping relies on +// errors.Is(err, ErrNotFound) seeing through it (CONTRACT). +type fatal struct{ error } + +func (f fatal) Unwrap() error { return f.error } + +// permanent reports whether an HTTP status is a client error that will not +// change on retry (4xx except the transient 408 Request Timeout and 429 Too +// Many Requests). +func permanent(code int) bool { + return code >= 400 && code < 500 && code != 408 && code != 429 +} + +// statusError turns an unexpected HTTP status into the error the retry policy +// expects: a 404 becomes fatal{} wrapping ErrNotFound so it is permanent and +// main's exit-code mapping can still see it via errors.Is(err, ErrNotFound) +// (CONTRACT); any other permanent() 4xx becomes a plain fatal{} (retrying +// cannot help); everything else is returned bare so withRetries retries it. +// ctxMsg is the caller's ": " message (already formatted). +func statusError(ctxMsg string, code int) error { + err := errors.New(ctxMsg) + if code == http.StatusNotFound { + return fatal{fmt.Errorf("%w: %s", ErrNotFound, err)} + } + if permanent(code) { + return fatal{err} + } + return err +} + +// pump copies the response body into the file at the segment's running offset, +// stopping at the segment end and respecting rate limits. +func (d *Download) pump(ctx context.Context, f *os.File, s *seg, body io.Reader) error { + buf := make([]byte, readBuf) + for s.remaining() > 0 { + n := int64(len(buf)) + if s.remaining() < n { + n = s.remaining() + } + rd, err := body.Read(buf[:n]) + if rd > 0 { + if _, werr := f.WriteAt(buf[:rd], s.offset()); werr != nil { + return werr + } + s.advance(int64(rd)) + atomic.AddInt64(&d.completed, int64(rd)) + d.throttle(ctx, rd) + } + if err == io.EOF { + // A clean EOF before the range is filled means the server sent a + // short body (e.g. a re-chunking proxy). Treat it as an error so + // the segment is retried from the advanced offset, never silently + // left with a hole. + if s.remaining() > 0 { + return io.ErrUnexpectedEOF + } + return nil + } + if err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + } + return nil +} + +// single streams the whole resource over one connection (server has no ranges +// or the length is unknown). It runs through the shared retry policy so a +// transient failure mid-stream is retried instead of failing the download +// (item [1]); each attempt re-derives the resume offset from the on-disk file +// size, so a retry continues rather than restarting. With -c and an existing +// foreign partial file the first attempt resumes from that prefix (item [2]). +func (d *Download) single(ctx context.Context, out string) error { + // foreignResume is honoured only on the first attempt: it decides whether the + // bytes already on disk are a real prefix to keep (-c, no .got control) or a + // stale file to truncate. On a retry the bytes on disk are whatever this run + // wrote, so we always resume from them regardless. + foreignResume := d.cfg.Continue && !fileExists(controlPath(out)) + first := true + // One connection at a time, but try each mirror: a mirror that fails its + // retry budget falls over to the next (resuming from what's already on disk). + var lastErr error + for i := range d.uris { + if ctx.Err() != nil { + return ctx.Err() + } + uri := d.mirror(i) + err := d.withRetries(ctx, uri, func() error { + resumeFromDisk := !first || foreignResume + first = false + return d.singleOnce(ctx, out, uri, resumeFromDisk) + }) + if err == nil { + return nil + } + lastErr = err + } + return lastErr +} + +// singleOnce performs one single-stream transfer attempt. When resumeFromDisk is +// set, the current on-disk file size is the resume point and the file is opened +// without O_TRUNC so the existing prefix survives; otherwise the file is +// truncated and the transfer starts from zero. +func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDisk bool) error { + var resumeAt int64 + if resumeFromDisk { + if fi, err := os.Stat(out); err == nil { + resumeAt = fi.Size() + } + } + + rangeHdr := "" + if resumeAt > 0 { + rangeHdr = fmt.Sprintf("bytes=%d-", resumeAt) + } + + reqCtx, cancel := context.WithCancel(ctx) + defer cancel() + req, err := d.request(reqCtx, uri, rangeHdr) + if err != nil { + return err + } + atomic.AddInt32(&d.conns, 1) + defer atomic.AddInt32(&d.conns, -1) + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return statusError(fmt.Sprintf("%s: %s", uri, resp.Status), resp.StatusCode) + } + // If we asked to resume but the server replied 200 (no range honoured), start + // over from the top rather than appending past the existing bytes. + if resp.StatusCode == http.StatusOK { + resumeAt = 0 + } + if resp.ContentLength > 0 { + atomic.StoreInt64(&d.total, resumeAt+resp.ContentLength) + } + + // Open without O_TRUNC when resuming so the existing prefix survives; the + // fresh path truncates to drop any stale tail. + flags := os.O_RDWR | os.O_CREATE + if resumeAt == 0 { + flags |= os.O_TRUNC + } + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + f, err := os.OpenFile(out, flags, 0o644) + if err != nil { + return err + } + defer f.Close() + + body, stop := d.idleGuard(resp.Body, cancel) + defer stop() + buf := make([]byte, readBuf) + off := resumeAt + atomic.StoreInt64(&d.completed, off) + // When the length is known, track bytes received in this stream so a body + // that ends early (a proxy that closes mid-transfer) is reported as a + // retryable short read instead of a false Complete (item [1], mirrors pump). + var got int64 + want := resp.ContentLength // -1 when unknown + for { + rd, err := body.Read(buf) + if rd > 0 { + if _, werr := f.WriteAt(buf[:rd], off); werr != nil { + return werr + } + off += int64(rd) + got += int64(rd) + atomic.StoreInt64(&d.completed, off) + d.throttle(ctx, rd) + } + if err == io.EOF { + if want >= 0 && got < want { + return io.ErrUnexpectedEOF // short body: retry from the resumed offset + } + removeControl(out) // drop any stale control file from a prior segmented run + return nil + } + if err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + } +} + +func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, segs []seg, done chan<- struct{}) { + defer close(done) + interval := d.cfg.AutoSaveInterval + if interval <= 0 { + interval = 60 * time.Second // --auto-save-interval default + } + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + snapshot(d.primary(), total, etag, lastmod, segs).save(out) + return + case <-t.C: + snapshot(d.primary(), total, etag, lastmod, segs).save(out) + } + } +} + +func (d *Download) resumeControl(out, etag, lastmod string) *control { + if !d.cfg.Continue { + return nil + } + return loadControl(out, d.primary(), atomic.LoadInt64(&d.total), etag, lastmod) +} + +func (d *Download) throttle(ctx context.Context, n int) { + for _, l := range d.limit { + _ = l.WaitN(ctx, n) + } +} + +func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", d.cfg.UserAgent) + if d.cfg.Referer != "" { + req.Header.Set("Referer", d.cfg.Referer) + } + if d.cfg.HTTPUser != "" { + req.SetBasicAuth(d.cfg.HTTPUser, d.cfg.HTTPPasswd) + } + for _, h := range d.cfg.Headers { + if k, v, ok := strings.Cut(h, ":"); ok { + req.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } + if rangeHdr != "" { + req.Header.Set("Range", rangeHdr) + } + if d.jar != nil { + d.recordURL(req.URL) + } + return req, nil +} + +// recordURL remembers a contacted URL so SaveCookies can later read back the +// jar's cookies for that host. Duplicates are tolerated; saveCookieJar dedups. +func (d *Download) recordURL(u *url.URL) { + d.mu.Lock() + d.seenURLs = append(d.seenURLs, u) + d.mu.Unlock() +} + +// SaveCookies writes the client's cookie jar back to the --save-cookies file, +// if one was configured. It is a no-op when cookies are not in use. main calls +// it once per HTTP download on exit (cookies are saved at session end). +func (d *Download) SaveCookies() error { + if d.cfg.SaveCookies == "" || d.jar == nil { + return nil + } + d.mu.Lock() + urls := append([]*url.URL{}, d.seenURLs...) + d.mu.Unlock() + return saveCookieJar(d.jar, urls, d.cfg.SaveCookies) +} + +// seedSegmentsFromPrefix marks the first prefix bytes of the file as already +// downloaded, segment by segment. A foreign partial file has no per-segment +// metadata, so the only thing we can trust is that bytes [0,prefix) are present; +// we fill whole leading segments and a partial one at the boundary. +func seedSegmentsFromPrefix(segs []seg, prefix int64) { + for i := range segs { + if prefix <= segs[i].start { + break + } + fill := prefix - segs[i].start + if fill > segs[i].length() { + fill = segs[i].length() + } + segs[i].written = fill + } +} + +func segsFromControl(c *control) []seg { + segs := make([]seg, len(c.Segs)) + for i, st := range c.Segs { + segs[i] = seg{index: i, start: st.Start, end: st.End, written: st.Written} + } + return segs +} + +func nameFromURL(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "index" + } + return nameFromURLPath(parsed) +} + +// nameFromURLPath takes the basename of a URL's path, percent-decoded, falling +// back to "index" for a directory-style URL. +func nameFromURLPath(u *url.URL) string { + base := path.Base(u.Path) + if dec, err := url.PathUnescape(base); err == nil { + base = dec + } + if base == "" || base == "." || base == "/" { + return "index" + } + return base +} + +// nameFromContentDisposition extracts the filename from a Content-Disposition +// header per RFC 6266, preferring filename* when present (mime.ParseMediaType +// already decodes it). Returns "" when there is no usable filename, so the +// caller can fall back to the URL. Path separators are stripped to keep the +// server from writing outside the output directory. +func nameFromContentDisposition(cd string) string { + _, params, err := mime.ParseMediaType(cd) + if err != nil { + return "" + } + name := params["filename"] + if name == "" { + return "" + } + name = filepath.Base(filepath.FromSlash(name)) + if name == "" || name == "." || name == string(filepath.Separator) { + return "" + } + return name +} + +// uniqueName returns the first non-colliding variant of p by inserting ".N" +// before the final extension (stem.N.ext), or "base.N" when there is no +// extension. It caps auto file renaming at 9999; a +// candidate that exists is acceptable when its .got control file also exists +// (it is a resumable partial). Past the cap it returns an error (item [13]). +func uniqueName(p string) (string, error) { + stem, ext := splitExt(p) + for i := 1; i < 10000; i++ { + cand := fmt.Sprintf("%s.%d%s", stem, i, ext) + if !fileExists(cand) || fileExists(controlPath(cand)) { + return cand, nil + } + } + return "", fmt.Errorf("auto file renaming failed for %s: too many existing files", p) +} + +// splitExt splits p into stem and extension: a leading dot is +// not an extension (".bashrc" stays whole), and the extension search ignores +// any directory component. +func splitExt(p string) (stem, ext string) { + ext = filepath.Ext(p) + if ext == "" { + return p, "" + } + base := filepath.Base(p) + // A dotfile with no other dot ("/dir/.ext") has no extension. + if strings.HasPrefix(base, ".") && strings.Count(base, ".") == 1 { + return p, "" + } + return p[:len(p)-len(ext)], ext +} + +func fileExists(p string) bool { + _, err := os.Stat(p) + return err == nil +} + +// parseContentRange parses "bytes 0-0/12345" into start, end, total. +func parseContentRange(s string) (start, end, total int64, ok bool) { + s = strings.TrimPrefix(strings.TrimSpace(s), "bytes ") + rng, totalStr, found := strings.Cut(s, "/") + if !found { + return 0, 0, 0, false + } + startStr, endStr, found := strings.Cut(rng, "-") + if !found { + return 0, 0, 0, false + } + start, e1 := strconv.ParseInt(startStr, 10, 64) + end, e2 := strconv.ParseInt(endStr, 10, 64) + total, e3 := strconv.ParseInt(totalStr, 10, 64) + if e1 != nil || e2 != nil || e3 != nil { + return 0, 0, 0, false + } + return start, end, total, true +} diff --git a/httpdl/mirror_test.go b/httpdl/mirror_test.go new file mode 100644 index 0000000..1882e51 --- /dev/null +++ b/httpdl/mirror_test.go @@ -0,0 +1,140 @@ +package httpdl + +import ( + "bytes" + "context" + "crypto/sha256" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +func TestMirrorPick(t *testing.T) { + d := &Download{uris: []string{"a", "b", "c"}} + if d.primary() != "a" { + t.Errorf("primary() = %q, want a", d.primary()) + } + for i, want := range []string{"a", "b", "c", "a", "b"} { + if got := d.mirror(i); got != want { + t.Errorf("mirror(%d) = %q, want %q", i, got, want) + } + } + // A single mirror always returns itself, whatever the index. + one := &Download{uris: []string{"only"}} + if one.mirror(7) != "only" { + t.Errorf("single-mirror mirror(7) = %q, want only", one.mirror(7)) + } +} + +// TestMaxConns checks the connection ceiling min(split, mirrors*x), floored +// at 1, since that decides how many segment workers fan out. +func TestMaxConns(t *testing.T) { + cases := []struct{ split, x, mirrors, want int }{ + {5, 1, 1, 1}, // default -x1 -s5: one connection + {5, 16, 1, 5}, // split is the ceiling on one server + {16, 16, 1, 16}, + {5, 1, 3, 3}, // 3 mirrors * 1 per host < split + {16, 2, 3, 6}, // 3 mirrors * 2 per host < split + {0, 1, 1, 1}, // floor at 1 + } + for _, c := range cases { + uris := make([]string, c.mirrors) + for i := range uris { + uris[i] = "http://h/x" + } + d := New(uris, Config{Split: c.split, MaxConnPerServer: c.x}) + if d.maxConns != c.want { + t.Errorf("split=%d x=%d mirrors=%d: maxConns=%d, want %d", c.split, c.x, c.mirrors, d.maxConns, c.want) + } + } +} + +// TestMirrorFallback proves a per-segment 404 on one mirror falls over to a +// healthy mirror, and the assembled file is byte-correct. +func TestMirrorFallback(t *testing.T) { + data := make([]byte, 300000) + for i := range data { + data[i] = byte(i * 7) + } + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "f.bin", time.Time{}, bytes.NewReader(data)) + })) + defer good.Close() + + var badHits int32 + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&badHits, 1) + http.Error(w, "gone", http.StatusNotFound) // 404 every range + })) + defer bad.Close() + + dir := t.TempDir() + out := filepath.Join(dir, "f.bin") + // Primary is healthy (so the probe anchors size/validators), bad is a mirror + // that 404s every segment it's handed. + d := New([]string{good.URL + "/f.bin", bad.URL + "/f.bin"}, Config{ + Dir: dir, Out: "f.bin", Split: 4, MaxConnPerServer: 2, MinSplit: 1, + Tries: 1, UserAgent: "got-test", + }) + if err := d.Run(context.Background()); err != nil { + t.Fatalf("Run with a 404 mirror should still complete via the healthy one: %v", err) + } + got, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(got) != sha256.Sum256(data) { + t.Fatalf("content mismatch: got %d bytes, want %d", len(got), len(data)) + } + if atomic.LoadInt32(&badHits) == 0 { + t.Error("the bad mirror was never tried — the fallback path wasn't exercised") + } +} + +// TestMirrorDivergentRejected proves a mirror serving a DIFFERENT file (a 206 +// with the wrong total length) is rejected and falls over, so its bytes never +// corrupt the output via total-length validation. +func TestMirrorDivergentRejected(t *testing.T) { + good := make([]byte, 300000) + for i := range good { + good[i] = byte(i * 7) + } + bad := make([]byte, 250000) // different length and content, but ranges work + for i := range bad { + bad[i] = 0xAA + } + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "f.bin", time.Time{}, bytes.NewReader(good)) + })) + defer primary.Close() + var divHits int32 + div := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&divHits, 1) + http.ServeContent(w, r, "f.bin", time.Time{}, bytes.NewReader(bad)) + })) + defer div.Close() + + dir := t.TempDir() + out := filepath.Join(dir, "f.bin") + d := New([]string{primary.URL + "/f.bin", div.URL + "/f.bin"}, Config{ + Dir: dir, Out: "f.bin", Split: 4, MaxConnPerServer: 2, MinSplit: 1, + Tries: 1, UserAgent: "got-test", + }) + if err := d.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + got, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(got) != sha256.Sum256(good) { + t.Fatalf("a divergent mirror corrupted the output: got %d bytes, want %d (content differs)", len(got), len(good)) + } + if atomic.LoadInt32(&divHits) == 0 { + t.Error("divergent mirror never contacted — the length-validation path wasn't exercised") + } +} diff --git a/httpdl/retry_test.go b/httpdl/retry_test.go new file mode 100644 index 0000000..c13ddb1 --- /dev/null +++ b/httpdl/retry_test.go @@ -0,0 +1,28 @@ +package httpdl + +import ( + "errors" + "net" + "strings" + "testing" +) + +// retriesExhausted must wrap (not discard) the underlying cause, so main's +// exit-code mapping can still tell a DNS failure from a refused connection. +func TestRetriesExhausted(t *testing.T) { + cause := &net.DNSError{Err: "no such host", Name: "x.invalid", IsNotFound: true} + + err := retriesExhausted("http://x", 5, cause) + var de *net.DNSError + if !errors.As(err, &de) { + t.Fatalf("dropped the cause: %v", err) + } + if !strings.Contains(err.Error(), "after 5 tries") { + t.Errorf("missing try count: %v", err) + } + + // A single attempt did not "retry", so it must not claim it did. + if one := retriesExhausted("segment 0", 1, cause); strings.Contains(one.Error(), "tries") { + t.Errorf("single attempt should not mention tries: %v", one) + } +} diff --git a/httpdl/segment.go b/httpdl/segment.go new file mode 100644 index 0000000..2bc0438 --- /dev/null +++ b/httpdl/segment.go @@ -0,0 +1,53 @@ +package httpdl + +import "sync/atomic" + +// seg is one contiguous byte range of the output file, downloaded by a single +// ranged GET. A segment IS a byte range — a plain HTTP downloader does not need +// the Piece/Segment/block layering that exists only to share code with +// BitTorrent. written is updated with atomic ops so Stat() can read it while a +// worker advances it. +type seg struct { + index int + start int64 // first byte offset, inclusive + end int64 // last byte offset, inclusive + written int64 // bytes already written into this segment (the resume point) +} + +func (s *seg) length() int64 { return s.end - s.start + 1 } +func (s *seg) done() bool { return atomic.LoadInt64(&s.written) >= s.length() } +func (s *seg) advance(n int64) { atomic.AddInt64(&s.written, n) } +func (s *seg) progress() int64 { return atomic.LoadInt64(&s.written) } +func (s *seg) offset() int64 { return s.start + atomic.LoadInt64(&s.written) } +func (s *seg) remaining() int64 { return s.length() - atomic.LoadInt64(&s.written) } + +// makeSegments divides a file of total bytes into contiguous segments, using at +// most conns of them and never splitting below minSplit. The remainder lands in +// the last segment. With conns==1 (the default) this yields one segment. +func makeSegments(total, minSplit int64, conns int) []seg { + if conns < 1 { + conns = 1 + } + if minSplit < 1 { + minSplit = 1 + } + n := int64(conns) + if max := total / minSplit; max < n { + n = max + } + if n < 1 { + n = 1 + } + chunk := total / n + segs := make([]seg, n) + var start int64 + for i := int64(0); i < n; i++ { + end := start + chunk - 1 + if i == n-1 { + end = total - 1 + } + segs[i] = seg{index: int(i), start: start, end: end} + start = end + 1 + } + return segs +} diff --git a/httpdl/segment_test.go b/httpdl/segment_test.go new file mode 100644 index 0000000..8798412 --- /dev/null +++ b/httpdl/segment_test.go @@ -0,0 +1,188 @@ +package httpdl + +import ( + "os" + "path/filepath" + "testing" +) + +func TestMakeSegments(t *testing.T) { + tests := []struct { + name string + total int64 + minSplit int64 + conns int + wantN int + }{ + {"even split", 1000, 100, 5, 5}, + {"min-split caps count", 1000, 400, 5, 2}, + {"tiny file is one segment", 50, 100, 5, 1}, + {"single connection", 1000, 1, 1, 1}, + {"min-split exact", 1000, 200, 5, 5}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + segs := makeSegments(tc.total, tc.minSplit, tc.conns) + if len(segs) != tc.wantN { + t.Fatalf("got %d segments, want %d", len(segs), tc.wantN) + } + // Segments must be contiguous and cover the whole file exactly. + var next int64 + for i, s := range segs { + if s.start != next { + t.Errorf("segment %d starts at %d, want %d", i, s.start, next) + } + if s.end < s.start { + t.Errorf("segment %d has end %d < start %d", i, s.end, s.start) + } + next = s.end + 1 + } + if next != tc.total { + t.Errorf("segments cover %d bytes, want %d", next, tc.total) + } + }) + } +} + +func TestSegProgress(t *testing.T) { + s := seg{start: 100, end: 199} // length 100 + if s.length() != 100 { + t.Fatalf("length = %d", s.length()) + } + if s.done() { + t.Fatalf("new segment should not be done") + } + s.advance(60) + if s.offset() != 160 { + t.Errorf("offset = %d, want 160", s.offset()) + } + if s.remaining() != 40 { + t.Errorf("remaining = %d, want 40", s.remaining()) + } + s.advance(40) + if !s.done() { + t.Errorf("segment should be done after writing full length") + } +} + +func TestSplitExt(t *testing.T) { + tests := []struct { + in string + stem, ext string + }{ + {"foo.txt", "foo", ".txt"}, + {"foo", "foo", ""}, + {"/dir/foo.tar.gz", "/dir/foo.tar", ".gz"}, + {"/dir/.bashrc", "/dir/.bashrc", ""}, // dotfile, no extension + {".bashrc", ".bashrc", ""}, + {"archive.", "archive", "."}, + } + for _, tc := range tests { + stem, ext := splitExt(tc.in) + if stem != tc.stem || ext != tc.ext { + t.Errorf("splitExt(%q) = %q,%q; want %q,%q", tc.in, stem, ext, tc.stem, tc.ext) + } + } +} + +func TestUniqueName(t *testing.T) { + dir := t.TempDir() + + // foo.txt taken -> foo.1.txt (.N inserted before the extension, item [13]). + taken := filepath.Join(dir, "foo.txt") + if err := os.WriteFile(taken, nil, 0o644); err != nil { + t.Fatal(err) + } + got, err := uniqueName(taken) + if err != nil { + t.Fatalf("uniqueName: %v", err) + } + if want := filepath.Join(dir, "foo.1.txt"); got != want { + t.Errorf("uniqueName(%q) = %q, want %q", taken, got, want) + } + + // No-extension base -> base.1. + plain := filepath.Join(dir, "data") + if err := os.WriteFile(plain, nil, 0o644); err != nil { + t.Fatal(err) + } + got, err = uniqueName(plain) + if err != nil { + t.Fatalf("uniqueName: %v", err) + } + if want := filepath.Join(dir, "data.1"); got != want { + t.Errorf("uniqueName(%q) = %q, want %q", plain, got, want) + } + + // foo.txt and foo.1.txt both taken -> foo.2.txt. + if err := os.WriteFile(filepath.Join(dir, "foo.1.txt"), nil, 0o644); err != nil { + t.Fatal(err) + } + got, err = uniqueName(taken) + if err != nil { + t.Fatalf("uniqueName: %v", err) + } + if want := filepath.Join(dir, "foo.2.txt"); got != want { + t.Errorf("uniqueName second pass = %q, want %q", got, want) + } + + // A candidate that exists but has a .got control file is acceptable (it is a + // resumable partial). + ctrlBase := filepath.Join(dir, "r.bin") + cand1 := filepath.Join(dir, "r.1.bin") + if err := os.WriteFile(ctrlBase, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cand1, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(controlPath(cand1), nil, 0o644); err != nil { + t.Fatal(err) + } + got, err = uniqueName(ctrlBase) + if err != nil { + t.Fatalf("uniqueName: %v", err) + } + if got != cand1 { + t.Errorf("uniqueName with control sidecar = %q, want %q", got, cand1) + } +} + +func TestNameFromContentDisposition(t *testing.T) { + tests := []struct { + cd, want string + }{ + {`attachment; filename="report.pdf"`, "report.pdf"}, + {`attachment; filename=report.pdf`, "report.pdf"}, + {`inline; filename*=UTF-8''na%C3%AFve.txt`, "naïve.txt"}, + {`attachment; filename="../../etc/passwd"`, "passwd"}, // path stripped + {`attachment`, ""}, + {`garbage`, ""}, + } + for _, tc := range tests { + if got := nameFromContentDisposition(tc.cd); got != tc.want { + t.Errorf("nameFromContentDisposition(%q) = %q, want %q", tc.cd, got, tc.want) + } + } +} + +func TestSeedSegmentsFromPrefix(t *testing.T) { + segs := makeSegments(1000, 100, 5) // five 200-byte segments + seedSegmentsFromPrefix(segs, 450) // 2 full segments + 50 bytes of the third + wants := []int64{200, 200, 50, 0, 0} + for i := range segs { + if segs[i].written != wants[i] { + t.Errorf("segment %d written = %d, want %d", i, segs[i].written, wants[i]) + } + } +} + +func TestParseContentRange(t *testing.T) { + start, end, total, ok := parseContentRange("bytes 0-0/12345") + if !ok || start != 0 || end != 0 || total != 12345 { + t.Errorf("parseContentRange = %d,%d,%d,%v", start, end, total, ok) + } + if _, _, _, ok := parseContentRange("garbage"); ok { + t.Errorf("parseContentRange(garbage) ok = true, want false") + } +} diff --git a/httpdl/single_test.go b/httpdl/single_test.go new file mode 100644 index 0000000..8d38bc7 --- /dev/null +++ b/httpdl/single_test.go @@ -0,0 +1,88 @@ +package httpdl + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/hanbok/got/download" +) + +// TestSingleShortBody verifies that a single-stream transfer whose body ends +// before the declared Content-Length is reported as an error (a retryable short +// read) rather than a false success (item [1]). A proxy that closes early must +// not leave the download marked Complete. +func TestSingleShortBody(t *testing.T) { + const declared = 1000 + // Hijack the connection so we can declare a long Content-Length but write + // only a few bytes and then close — exactly the early-close a re-chunking + // proxy produces. httptest's normal writer would otherwise flag the mismatch + // on its own terms; hijacking gives us full control of the wire bytes. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + t.Errorf("server does not support hijacking") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + defer conn.Close() + fmt.Fprintf(conn, "HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n", declared) + conn.Write([]byte("short")) // only 5 of the promised 1000 bytes + // closing the connection now is the early EOF + })) + defer srv.Close() + + dir := t.TempDir() + out := filepath.Join(dir, "file.bin") + + d := New([]string{srv.URL + "/file.bin"}, Config{ + Dir: dir, + Out: "file.bin", + MaxConnPerServer: 1, + Tries: 1, // a single attempt so the retryable short read surfaces, not loops + UserAgent: "got-test", + }) + + err := d.single(context.Background(), out) + if err == nil { + t.Fatal("single() returned nil for a body shorter than Content-Length; want an error") + } + if d.Stat().Status == download.Complete { + t.Fatalf("status = Complete after a short body; want not Complete") + } +} + +// TestSingleCompleteFullBody is the positive control: a body that matches its +// Content-Length succeeds. This guards against the short-body check rejecting a +// legitimately complete stream. +func TestSingleCompleteFullBody(t *testing.T) { + const body = "the whole thing" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + w.WriteHeader(http.StatusOK) + w.Write([]byte(body)) + })) + defer srv.Close() + + dir := t.TempDir() + out := filepath.Join(dir, "file.bin") + + d := New([]string{srv.URL + "/file.bin"}, Config{ + Dir: dir, + Out: "file.bin", + MaxConnPerServer: 1, + Tries: 1, + UserAgent: "got-test", + }) + + if err := d.single(context.Background(), out); err != nil { + t.Fatalf("single() of a full body returned %v; want nil", err) + } +} diff --git a/httpdl/speed_test.go b/httpdl/speed_test.go new file mode 100644 index 0000000..fb0f172 --- /dev/null +++ b/httpdl/speed_test.go @@ -0,0 +1,32 @@ +package httpdl + +import "testing" + +// TestBelowSpeed covers the speedGuard decision, including the counter-reset +// case (item [1]): d.completed can jump backward when singleOnce rewrites it +// with an absolute StoreInt64 and a resumed Range is answered by a 200 that +// resets the offset to 0. A backward jump must NOT be read as a slow window, or +// a healthy download would be false-aborted. +func TestBelowSpeed(t *testing.T) { + const limit = 1000 + cases := []struct { + name string + prev, now int64 + want bool + }{ + {"fast", 0, 5000, false}, + {"exactly at limit aborts", 1000, 2000, true}, + {"just under limit aborts", 1000, 1999, true}, + {"just over limit ok", 1000, 2001, false}, + {"stalled aborts", 4242, 4242, true}, + {"counter reset is not slow", 1_000_000, 0, false}, + {"small backward jump is not slow", 5000, 4500, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := belowSpeed(c.prev, c.now, limit); got != c.want { + t.Errorf("belowSpeed(%d, %d, %d) = %v; want %v", c.prev, c.now, limit, got, c.want) + } + }) + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..faa8761 --- /dev/null +++ b/main.go @@ -0,0 +1,826 @@ +// Command got is a Rob-Pike-style command-line download tool: a small +// multi-protocol downloader. It downloads HTTP(S) URLs over several connections +// and BitTorrent magnets/torrents, with a comprehensive set of command-line +// options. Each download runs as one goroutine; a scheduler bounds how many run +// at once; a single goroutine draws the progress line. +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "net" + "os" + "os/signal" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" + "github.com/hanbok/got/bt" + "github.com/hanbok/got/cli" + "github.com/hanbok/got/download" + "github.com/hanbok/got/httpdl" + "github.com/hanbok/got/progress" + "golang.org/x/time/rate" +) + +func main() { + os.Exit(run(os.Args[1:])) +} + +func run(args []string) int { + res, err := cli.Parse(args) + if err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + return 1 + } + switch res.Action { + case cli.ShowVersion: + cli.PrintVersion(os.Stdout) + return 0 + case cli.ShowHelp: + cli.PrintHelp(os.Stdout, res.HelpTag) + return 0 + } + + opts := res.Options + + // --checksum is validated up front so a bad type or digest aborts before any + // download, rejecting it at option-parse time rather than failing mid-run + // (and once, not once per HTTP download that shares the spec). + if cs := opts.Str("checksum"); cs != "" { + if err := httpdl.ValidChecksum(cs); err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + return 1 + } + } + + targets := gatherURIs(opts, res.URIs) + if len(targets) == 0 { + fmt.Fprintln(os.Stderr, "got: no URI, torrent or magnet given (try --help)") + return 1 + } + + if opts.Bool("show-files") { + return showFiles(allURIs(targets)) + } + + jobs, btClient, err := build(opts, targets) + if err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + return 1 + } + if btClient != nil { + defer btClient.Close() + } + + // A single registered signal channel handles both presses: the first cancels + // the graceful context, the second saves the session best-effort and forces + // exit. Using one channel avoids a startup race where two separate handlers + // could split the two presses and need a third to actually quit. + sessionFile := opts.Str("save-session") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // --stop: cancel everything after N seconds, exactly like a Ctrl-C, so an + // unattended run can't hang forever (0 = off). + if stop := opts.Int("stop"); stop > 0 { + t := time.AfterFunc(time.Duration(stop)*time.Second, cancel) + defer t.Stop() + } + jobsRef := &jobsHolder{} + go handleSignals(cancel, sessionFile, jobsRef) + + eng := download.NewEngine(opts.Int("max-concurrent-downloads")) + + uiCtx, uiCancel := context.WithCancel(context.Background()) + if !opts.Bool("quiet") && !opts.Bool("dry-run") { + rep := progress.New(eng.Snapshot, opts.Bool("human-readable")) + go rep.Run(uiCtx) + } + + jobsRef.set(jobs) + results := runJobs(ctx, eng, jobs) + + // --follow-torrent: a .torrent fetched over HTTP becomes a BitTorrent + // download of its content (on by default). Done as a second pass so the + // engine and scheduler stay simple. + if opts.Bool("follow-torrent") && ctx.Err() == nil && !opts.Bool("dry-run") { + if follow := followUps(jobs, btClient, btOptions(opts)); len(follow) > 0 { + // The engine numbers each Result by its position in the slice it was + // passed, so the follow pass restarts at Index 0. Offset those by the + // initial job count to give the concatenated slice one contiguous + // Index space, so the sort below restores input order deterministically. + nInitial := len(jobs) + jobs = append(jobs, follow...) + jobsRef.set(jobs) + followResults := runJobs(ctx, eng, follow) + for i := range followResults { + followResults[i].Index += nInitial + } + results = append(results, followResults...) + } + } + + uiCancel() + time.Sleep(20 * time.Millisecond) // let the renderer clear its line + + saveCookies(jobs) + + if sessionFile != "" { + if err := saveSession(sessionFile, jobs); err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + } + } + // Restore input order so the OK/FAIL lines and the exit code are + // deterministic regardless of which download finished first. + sort.Slice(results, func(i, j int) bool { return results[i].Index < results[j].Index }) + if opts.Bool("dry-run") { + return reportDryRun(results) + } + return report(results) +} + +// jobsHolder lets the signal handler reach the latest job list for a +// best-effort session save on a forced (second-Ctrl-C) exit. +type jobsHolder struct { + mu sync.Mutex + jobs []job +} + +func (h *jobsHolder) set(jobs []job) { + h.mu.Lock() + h.jobs = jobs + h.mu.Unlock() +} + +func (h *jobsHolder) get() []job { + h.mu.Lock() + defer h.mu.Unlock() + return h.jobs +} + +// job pairs a download with the source string that produced it, so we can write +// the still-unfinished ones back to a session file. +type job struct { + source string + dl download.Download +} + +// build turns each URI into a Download, classified by scheme, and creates the +// one shared BitTorrent client the run needs (nil if no torrents are involved). +// The overall speed limits are one shared limiter each, applied to both HTTP +// downloads and the BitTorrent client so the cap is truly global. +// errDuplicate marks a source that names the same torrent as an earlier one. +// exitCode maps it to the duplicate-infohash exit code (12) via errors.Is. +var errDuplicate = errors.New("duplicate download") + +// failedDownload is a Download that does no work and fails immediately with a +// fixed error. build() uses it for a duplicate torrent source, so the duplicate +// is reported as a failed download without ever obtaining — or Dropping — a +// torrent on the shared client. +type failedDownload struct { + name string + err error +} + +func (f failedDownload) Name() string { return f.name } +func (f failedDownload) Run(context.Context) error { return f.err } +func (f failedDownload) Stat() download.Stat { + return download.Stat{Name: f.name, Status: download.Errored, Total: -1} +} + +// dedupeTorrents finds torrent/magnet sources that name the same torrent as an +// earlier source (same infohash), returning each duplicate URI mapped to the +// earlier source it repeats. HTTP(S) URLs and any source whose infohash can't be +// read without a network fetch are never duplicates here. The shared client keys +// torrents by infohash and does not refcount, so build() turns each duplicate +// into a failed download (duplicate infohash) rather than letting two jobs +// share — and Drop — one torrent. +func dedupeTorrents(uris []string, follow bool) map[string]string { + seen := map[metainfo.Hash]string{} + dup := map[string]string{} + for _, u := range uris { + k := uriKind(u, follow) + if k != kindMagnet && k != kindTorrent { + continue + } + h, ok := bt.SourceInfoHash(u, k == kindTorrent) + if !ok { + continue + } + if first, isDup := seen[h]; isDup { + dup[u] = first + } else { + seen[h] = u + } + } + return dup +} + +func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) { + flat := allURIs(targets) + // Two sources naming the same torrent (a magnet and its own .torrent, say) + // would share one torrent in the client, which does not refcount: the first to + // finish would Drop it out from under the other. The duplicate is reported as + // a failed download (duplicate infohash, exit 12); the loop below does this, + // so the duplicate never obtains or Drops a shared torrent. + dups := dedupeTorrents(flat, opts.Bool("follow-torrent")) + + overallDL := makeLimiter(opts.Size("max-overall-download-limit")) + overallUL := makeLimiter(opts.Size("max-overall-upload-limit")) + // --out names a single output file, so it only applies when exactly one HTTP + // download (one mirror group) is queued; warn rather than silently misname. + nhttp := countHTTP(targets) + single := nhttp == 1 + if nhttp > 1 && opts.Str("out") != "" { + fmt.Fprintln(os.Stderr, "got: --out ignored: it applies only to a single HTTP download") + } + hc := httpConfig(opts, single, overallDL) + bo := btOptions(opts) + + // One client serves every torrent, shared across the session. A failure to start + // it (e.g. the listen port is taken) disables BitTorrent but still lets HTTP + // downloads run; those torrents then report the error. + var client *torrent.Client + if needsBT(opts, flat) { + c, err := bt.NewClient(btClientConfig(opts, overallDL, overallUL)) + if err != nil { + fmt.Fprintln(os.Stderr, "got: bittorrent disabled:", err) + } else { + client = c + } + } + + var out []job + for _, t := range targets { + u := t.uris[0] // primary: classifies the target and names the session source + src := strings.Join(t.uris, "\t") + if first, isDup := dups[u]; isDup { + out = append(out, job{source: src, dl: failedDownload{ + name: u, + err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first), + }}) + continue + } + var dl download.Download + switch uriKind(u, opts.Bool("follow-torrent")) { + case kindMagnet: + dl = bt.New(client, u, false, bo) + case kindHTTP, kindFollow: + dl = httpdl.New(t.uris, hc) // every mirror URL for this one file + case kindTorrent: + dl = bt.New(client, u, true, bo) + default: + if client != nil { + client.Close() + } + return nil, nil, fmt.Errorf("unsupported URI: %s", u) + } + out = append(out, job{source: src, dl: dl}) + } + return out, client, nil +} + +// uriKind classifies a URI by scheme into the taxonomy build() and needsBT() +// both switch on. Scheme wins over suffix: an http URL ending in .torrent is a +// file to fetch over HTTP, and with --follow-torrent it becomes a torrent only +// after the fetch (kindFollow), so it still downloads over HTTP first. +func uriKind(u string, follow bool) kind { + lu := strings.ToLower(u) + switch { + case strings.HasPrefix(u, "magnet:"): + return kindMagnet + case strings.HasPrefix(u, "http://"), strings.HasPrefix(u, "https://"): + if follow && strings.HasSuffix(lu, ".torrent") { + return kindFollow + } + return kindHTTP + case strings.HasSuffix(lu, ".torrent"): + return kindTorrent + } + return kindUnsupported +} + +type kind int + +const ( + kindUnsupported kind = iota + kindMagnet // magnet: link, a BitTorrent download + kindHTTP // plain HTTP(S) download + kindTorrent // local .torrent file, a BitTorrent download + kindFollow // HTTP(S) .torrent fetched, then followed as BitTorrent +) + +// needsBT reports whether the run will involve any torrent, so we only start a +// client when one is actually needed. +func needsBT(opts *cli.Options, uris []string) bool { + follow := opts.Bool("follow-torrent") + for _, u := range uris { + switch uriKind(u, follow) { + case kindMagnet, kindTorrent, kindFollow: + return true + } + } + return false +} + +// countHTTP counts HTTP download targets. A mirror group is one output file, so +// it counts groups, not URLs, for the --out single-file decision. +func countHTTP(targets []target) int { + n := 0 + for _, t := range targets { + u := t.uris[0] + if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") { + n++ + } + } + return n +} + +// runJobs runs a set of jobs through the engine and returns their results. +func runJobs(ctx context.Context, eng *download.Engine, jobs []job) []download.Result { + dls := make([]download.Download, len(jobs)) + for i := range jobs { + dls[i] = jobs[i].dl + } + return eng.Run(ctx, dls) +} + +// followUps turns each completed HTTP .torrent download into a BitTorrent +// download of its content. A file is followed only if it parses as a torrent, +// so a regular download that merely ends in ".torrent" is left alone. +func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job { + var out []job + for _, j := range jobs { + h, ok := j.dl.(*httpdl.Download) + if !ok || j.dl.Stat().Status != download.Complete { + continue + } + // j.source is the TAB-joined mirror group; the primary (uris[0]) is what + // build() classified and named the download on, so test its suffix. + primary, _, _ := strings.Cut(j.source, "\t") + if !strings.HasSuffix(strings.ToLower(primary), ".torrent") { + continue + } + path := h.Path() + if _, err := bt.Files(path); err != nil { + continue // downloaded file is not a valid torrent + } + out = append(out, job{source: path, dl: bt.New(client, path, true, bo)}) + } + return out +} + +// saveCookies asks each HTTP download to write its cookie jar back to the +// --save-cookies file, persisting cookies at session end. A download with no +// --save-cookies configured is a no-op. +func saveCookies(jobs []job) { + for _, j := range jobs { + if h, ok := j.dl.(*httpdl.Download); ok { + if err := h.SaveCookies(); err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + } + } + } +} + +// target is one download: a single torrent/magnet/URL, or several HTTP mirror +// URLs that serve identical bytes (uris[0] is the primary). +type target struct{ uris []string } + +// allURIs flattens targets to a plain URL list for the per-URL helpers +// (dedupe, needsBT, show-files) that don't care about grouping. +func allURIs(ts []target) []string { + var u []string + for _, t := range ts { + u = append(u, t.uris...) + } + return u +} + +// gatherURIs collects download targets from the command line, --torrent-file, +// and --input-file. Plain command-line http/https URIs are grouped into ONE +// mirrored download by default (every command-line URI names the same file); +// -Z/--force-sequential makes each its own download. Torrents and magnets +// are always one target each; --input-file groups TAB-separated URIs per line. +func gatherURIs(opts *cli.Options, cmdURIs []string) []target { + var ts []target + follow := opts.Bool("follow-torrent") + seq := opts.Bool("force-sequential") + var httpGroup []string + for _, u := range cmdURIs { + switch uriKind(u, follow) { + case kindHTTP, kindFollow: + if seq { + ts = append(ts, target{uris: []string{u}}) + } else { + httpGroup = append(httpGroup, u) + } + default: + ts = append(ts, target{uris: []string{u}}) + } + } + if len(httpGroup) > 0 { + ts = append(ts, target{uris: httpGroup}) + } + if tf := opts.Str("torrent-file"); tf != "" { + ts = append(ts, target{uris: []string{tf}}) + } + if in := opts.Str("input-file"); in != "" { + ts = append(ts, readInputFile(in)...) + } + return ts +} + +// readInputFile parses an input file: one download per line, and TAB-separated +// URIs on a line are mirrors of that one download. Blank lines and '#' comments +// are skipped. +func readInputFile(path string) []target { + f := os.Stdin + if path != "-" { + var err error + if f, err = os.Open(path); err != nil { + // A missing input file is fine: it just means "nothing to resume + // yet", which is what the load+save-session idiom needs on first run. + if !os.IsNotExist(err) { + fmt.Fprintln(os.Stderr, "got:", err) + } + return nil + } + defer f.Close() + } + var ts []target + sc := bufio.NewScanner(f) + // Magnet links with many trackers (and long signed URLs) can exceed the + // 64KB default token size, so raise the cap. + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for sc.Scan() { + line := sc.Text() + if t := strings.TrimSpace(line); t == "" || strings.HasPrefix(t, "#") { + continue + } + var g []string + for _, field := range strings.Split(line, "\t") { + if field = strings.TrimSpace(field); field != "" { + g = append(g, field) + } + } + if len(g) > 0 { + ts = append(ts, target{uris: g}) + } + } + if err := sc.Err(); err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + } + return ts +} + +func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.Config { + out := "" + if single { + out = opts.Str("out") + } + return httpdl.Config{ + Dir: opts.Str("dir"), + Out: out, + Split: opts.Int("split"), + MaxConnPerServer: opts.Int("max-connection-per-server"), + MinSplit: opts.Size("min-split-size"), + Tries: opts.Int("max-tries"), + Timeout: time.Duration(opts.Int("timeout")) * time.Second, + FileAlloc: opts.Str("file-allocation"), + Continue: opts.Bool("continue"), + AllowOverwrite: opts.Bool("allow-overwrite"), + AutoRename: opts.Bool("auto-file-renaming"), + Headers: opts.List("header"), + UserAgent: opts.Str("user-agent"), + Referer: opts.Str("referer"), + Proxy: opts.Str("all-proxy"), + CheckCert: opts.Bool("check-certificate"), + Limit: opts.Size("max-download-limit"), + OverallLimiter: overallDL, + RetryWait: time.Duration(opts.Int("retry-wait")) * time.Second, + AutoSaveInterval: time.Duration(opts.Int("auto-save-interval")) * time.Second, + ConnectTimeout: time.Duration(opts.Int("connect-timeout")) * time.Second, + LoadCookies: opts.Str("load-cookies"), + SaveCookies: opts.Str("save-cookies"), + ConditionalGet: opts.Bool("conditional-get"), + RemoteTime: opts.Bool("remote-time"), + HTTPUser: opts.Str("http-user"), + HTTPPasswd: opts.Str("http-passwd"), + LowestSpeedLimit: opts.Size("lowest-speed-limit"), + Checksum: opts.Str("checksum"), + DryRun: opts.Bool("dry-run"), + CACert: opts.Str("ca-certificate"), + DisableIPv6: opts.Bool("disable-ipv6"), + } +} + +// btClientConfig is the run-wide configuration for the one shared torrent client. +func btClientConfig(opts *cli.Options, overallDL, overallUL *rate.Limiter) bt.ClientConfig { + return bt.ClientConfig{ + Dir: opts.Str("dir"), + ListenPortSpec: opts.Str("listen-port"), + DHT: opts.Bool("enable-dht"), + MaxPeers: opts.Int("bt-max-peers"), + OverallDown: overallDL, + OverallUp: overallUL, + DownLimit: opts.Size("max-download-limit"), + UpLimit: opts.Size("max-upload-limit"), + UserAgent: opts.Str("user-agent"), + DisableIPv6: opts.Bool("disable-ipv6"), + } +} + +// btOptions is the per-torrent configuration. +func btOptions(opts *cli.Options) bt.Options { + return bt.Options{ + SeedTimeSet: opts.IsSet("seed-time"), + SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)), + SeedRatio: opts.Float("seed-ratio"), + StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second, + MetaTimeout: time.Duration(opts.Int("bt-metadata-timeout")) * time.Second, + SelectFiles: parseSelect(opts.Str("select-file")), + CheckIntegrity: opts.Bool("check-integrity"), + DryRun: opts.Bool("dry-run"), + } +} + +// showFiles prints the file list of each torrent argument and returns an exit +// code. Magnets need their metadata fetched first, which we don't do here. +func showFiles(uris []string) int { + printed := false + for _, u := range uris { + if !strings.HasSuffix(strings.ToLower(u), ".torrent") { + if strings.HasPrefix(u, "magnet:") { + fmt.Printf("%s: --show-files needs a .torrent (magnet metadata is fetched at download time)\n", u) + } + continue + } + files, err := bt.Files(u) + if err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + return 1 + } + fmt.Printf("%s:\n", u) + for _, f := range files { + fmt.Printf(" %d %s (%d bytes)\n", f.Index, f.Path, f.Length) + } + printed = true + } + if !printed { + return 1 + } + return 0 +} + +// reportDryRun prints what --dry-run learned about each source — that it is +// reachable and how big it is — none of it having been downloaded, and returns +// the exit code: 0 if every source was reachable, else the dominant error's +// code (so a 404 under --dry-run still exits 3, a refused host 6, etc.). +func reportDryRun(results []download.Result) int { + var lastReal error + for _, r := range results { + switch { + case r.Err == nil: + if r.Stat.Total >= 0 { + fmt.Printf("DRY %s (%d bytes)\n", r.Name, r.Stat.Total) + } else { + fmt.Printf("DRY %s (size unknown)\n", r.Name) + } + case errors.Is(r.Err, context.Canceled): + fmt.Printf("STOP %s\n", r.Name) + default: + fmt.Printf("FAIL %s: %v\n", r.Name, r.Err) + lastReal = r.Err + } + } + if lastReal != nil { + return exitCode(lastReal) + } + return 0 +} + +// report prints a one-line outcome per download and returns the process exit +// code, mapped to the documented exit-status table: +// 0 all OK, 7 cancelled with at least one completed, 1 cancelled with nothing +// completed, else the code of the dominant (last) real error, mapped to an +// exit code by exitCode. +func report(results []download.Result) int { + completed, cancelled := 0, 0 + var lastReal error + for _, r := range results { + switch { + case r.Err == nil: + fmt.Printf("OK %s\n", r.Name) + completed++ + case errors.Is(r.Err, context.Canceled): + fmt.Printf("STOP %s (interrupted%s)\n", r.Name, progressNote(r.Stat)) + cancelled++ + default: + fmt.Printf("FAIL %s: %v\n", r.Name, r.Err) + lastReal = r.Err + } + } + + // Exit code 7 is returned when the only failures were interruptions and at + // least one download finished: unfinished downloads remained in the queue when + // the process exited by pressing Ctrl-C. With no real error and nothing + // complete, fall back to 1. + if lastReal == nil { + switch { + case cancelled > 0 && completed > 0: + return 7 + case cancelled > 0: + return 1 + default: + return 0 + } + } + return exitCode(lastReal) +} + +// exitCode maps a real (non-cancellation) download error to the documented +// exit-status codes. The retry path wraps the underlying cause with %w, so we can inspect the +// chain: an existing output file (13) and a 404 (3) are classified by their +// httpdl sentinel type; name resolution (19), timeout (2), and a connection-level +// network problem (6) are classified by typed errors with a substring fallback +// for third-party net text; everything else is the generic unknown error (1). +// Telling 19 from 6 from 2 apart is the point — it says whether the user's own +// network, the remote host, or a stall is to blame. +func exitCode(err error) int { + switch { + case errors.Is(err, httpdl.ErrOutputExists): + return 13 + case isNameResolution(err): + return 19 + case isTimeout(err): + return 2 + case isNetwork(err): + return 6 + case errors.Is(err, httpdl.ErrNotFound): + return 3 + case errors.Is(err, httpdl.ErrTooSlow): + return 5 + case errors.Is(err, httpdl.ErrChecksum): + return 32 + case errors.Is(err, errDuplicate): + return 12 + } + return 1 +} + +// isNameResolution reports whether err is (or wraps) a DNS lookup failure — +// "name resolution failed" (19), which usually points at the user's own +// network or resolver rather than a dead remote. +func isNameResolution(err error) bool { + var de *net.DNSError + if errors.As(err, &de) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "no such host") || strings.Contains(msg, "server misbehaving") +} + +// isNetwork reports whether err is a connection-level failure — refused, reset, +// or an unreachable host/network — a "network problem" (6). +func isNetwork(err error) bool { + if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTUNREACH) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "connection reset") || + strings.Contains(msg, "network is unreachable") || + strings.Contains(msg, "no route to host") +} + +// isTimeout reports whether err is (or wraps) a timeout: a deadline exceeded, a +// net.Error that timed out, or an idle-read timeout reported as such. +func isTimeout(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "timeout") || strings.Contains(msg, "timed out") +} + +// saveSession writes the sources of every download that did not finish to path, +// one per line, so a later `got -i path` resumes them (HTTP via -c, BitTorrent +// via anacrolix's recheck of existing data). An all-done run leaves the file +// empty. +func saveSession(path string, jobs []job) error { + lines := unfinishedSources(jobs) + data := "" + if len(lines) > 0 { + data = strings.Join(lines, "\n") + "\n" + } + // Write to a temp file in the same directory, then rename over the target so + // a crash or a force-quit mid-write can never leave a half-written (and thus + // truncated) resume list. The rename is atomic within one filesystem. + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".got-session-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.WriteString(data); err != nil { + tmp.Close() + os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + os.Remove(tmpName) + return err + } + return nil +} + +// unfinishedSources returns the source string of each download not in the +// Complete state. +func unfinishedSources(jobs []job) []string { + var out []string + for _, j := range jobs { + if j.dl.Stat().Status != download.Complete { + out = append(out, j.source) + } + } + return out +} + +// progressNote describes how far an interrupted download got, so the user knows +// resuming is worthwhile. +func progressNote(s download.Stat) string { + if s.Total > 0 && s.Completed > 0 { + return fmt.Sprintf("; %d%% done", int(s.Completed*100/s.Total)) + } + return "" +} + +// handleSignals owns the one registered signal channel. The first Ctrl-C (or +// TERM) cancels the graceful context so run() can wind down and save normally; +// a second forces exit, but first writes the session best-effort so the resume +// list survives a force-quit. +func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *jobsHolder) { + sig := make(chan os.Signal, 2) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + <-sig // first: cancel the graceful context + cancel() + <-sig // second: force exit + if sessionFile != "" { + if err := saveSession(sessionFile, jobs.get()); err != nil { + fmt.Fprintln(os.Stderr, "got:", err) + } + } + fmt.Fprintln(os.Stderr, "\ngot: forced exit") + os.Exit(130) +} + +func makeLimiter(bps int64) *rate.Limiter { + if bps <= 0 { + return nil + } + return rate.NewLimiter(rate.Limit(bps), download.LimiterBurst(bps)) +} + +// parseSelect parses "1,3-5" into a set of 1-based file indexes. +func parseSelect(s string) map[int]bool { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + out := map[int]bool{} + for _, part := range strings.Split(s, ",") { + lo, hi, ok := strings.Cut(part, "-") + a, _ := strconv.Atoi(strings.TrimSpace(lo)) + if !ok { + if a > 0 { + out[a] = true + } + continue + } + b, _ := strconv.Atoi(strings.TrimSpace(hi)) + for i := a; i <= b; i++ { + if i > 0 { + out[i] = true + } + } + } + return out +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..4c4c678 --- /dev/null +++ b/main_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "reflect" + "syscall" + "testing" + + "github.com/hanbok/got/cli" + "github.com/hanbok/got/download" + "github.com/hanbok/got/httpdl" +) + +// fakeDL is a stand-in Download whose status we control. +type fakeDL struct { + name string + status download.Status +} + +func (f fakeDL) Name() string { return f.name } +func (f fakeDL) Run(ctx context.Context) error { return nil } +func (f fakeDL) Stat() download.Stat { return download.Stat{Name: f.name, Status: f.status} } + +func TestUnfinishedSources(t *testing.T) { + jobs := []job{ + {source: "http://a/done", dl: fakeDL{status: download.Complete}}, + {source: "http://b/failed", dl: fakeDL{status: download.Errored}}, + {source: "magnet:c", dl: fakeDL{status: download.Active}}, + {source: "http://d/done", dl: fakeDL{status: download.Complete}}, + } + got := unfinishedSources(jobs) + want := []string{"http://b/failed", "magnet:c"} + if !reflect.DeepEqual(got, want) { + t.Errorf("unfinishedSources = %v, want %v", got, want) + } +} + +func TestParseSelect(t *testing.T) { + tests := []struct { + in string + want map[int]bool + }{ + {"", nil}, + {"1", map[int]bool{1: true}}, + {"1,3", map[int]bool{1: true, 3: true}}, + {"3-5", map[int]bool{3: true, 4: true, 5: true}}, + {"1,3-5,8", map[int]bool{1: true, 3: true, 4: true, 5: true, 8: true}}, + } + for _, tc := range tests { + if got := parseSelect(tc.in); !reflect.DeepEqual(got, tc.want) { + t.Errorf("parseSelect(%q) = %v, want %v", tc.in, got, tc.want) + } + } +} + +func TestExitCode(t *testing.T) { + dnsErr := &net.DNSError{Err: "no such host", Name: "nope.invalid", IsNotFound: true} + refused := &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)} + tests := []struct { + name string + err error + want int + }{ + {"idle timeout", errors.New("idle timeout: no data received"), 2}, + {"metadata timed out", errors.New("timed out fetching metadata after 3s"), 2}, + {"deadline", context.DeadlineExceeded, 2}, + {"file exists", fmt.Errorf("/tmp/x: %w (use --allow-overwrite or -c)", httpdl.ErrOutputExists), 13}, + {"dns typed", dnsErr, 19}, + {"dns wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", dnsErr), 19}, + {"dns by message", errors.New("dial tcp: lookup nope.invalid: no such host"), 19}, + {"refused typed", refused, 6}, + {"refused wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", refused), 6}, + {"refused by message", errors.New("dial tcp 127.0.0.1:1: connect: connection refused"), 6}, + {"not found", fmt.Errorf("%w: http://x/y: 404 Not Found", httpdl.ErrNotFound), 3}, + {"too slow", fmt.Errorf("http://x: %w (<= %d bytes/sec)", httpdl.ErrTooSlow, 1024), 5}, + {"checksum", fmt.Errorf("/tmp/x: %w (want a, got b)", httpdl.ErrChecksum), 32}, + {"duplicate", fmt.Errorf("%w: same torrent as y", errDuplicate), 12}, + {"unknown", errors.New("disk full"), 1}, + } + for _, tc := range tests { + if got := exitCode(tc.err); got != tc.want { + t.Errorf("%s: exitCode = %d, want %d", tc.name, got, tc.want) + } + } + // The whole point: a local DNS failure and a dead remote must not look alike. + if dns, nw := exitCode(dnsErr), exitCode(refused); dns == nw { + t.Errorf("DNS (%d) and refused (%d) must map to different codes", dns, nw) + } +} + +// TestDedupeTorrents guards the same-infohash hazard: the shared client does not +// refcount torrents, so two sources naming one torrent must collapse to a single +// download or the first to finish would Drop it out from under the other. +func TestDedupeTorrents(t *testing.T) { + const h = "c892b8de86be44003d92041a16787911c139ab1e" + a := "magnet:?xt=urn:btih:" + h + "&dn=one" + b := "magnet:?xt=urn:btih:" + h + "&dn=two" // same infohash as a + c := "magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=other" + web := "http://example.com/file.iso" + + dup := dedupeTorrents([]string{a, web, b, c}, true) + + // b is the only duplicate (of a); a, web and c are not duplicates. + if len(dup) != 1 || dup[b] != a { + t.Errorf("dup = %v, want {b: a}", dup) + } +} + +// TestGatherURIsGrouping checks the command-line grouping: plain http URIs +// become one mirrored download by default, -Z splits them, and torrents/magnets +// stay separate. +func TestGatherURIsGrouping(t *testing.T) { + parse := func(args ...string) []target { + res, err := cli.Parse(append([]string{"--no-conf"}, args...)) + if err != nil { + t.Fatalf("parse %v: %v", args, err) + } + return gatherURIs(res.Options, res.URIs) + } + if ts := parse("http://a/x", "http://b/x"); len(ts) != 1 || len(ts[0].uris) != 2 { + t.Errorf("two http args: got %v, want one 2-mirror group", ts) + } + if ts := parse("-Z", "http://a/x", "http://b/x"); len(ts) != 2 { + t.Errorf("-Z: got %d targets, want 2 separate downloads", len(ts)) + } + if ts := parse("http://a/x", "magnet:?xt=urn:btih:abc"); len(ts) != 2 { + t.Errorf("http + magnet: got %d targets, want 2 (magnet never mirrors)", len(ts)) + } +} + +// TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on +// a line are mirrors of one download; a plain line is one download. +func TestReadInputFileTAB(t *testing.T) { + p := filepath.Join(t.TempDir(), "in.txt") + if err := os.WriteFile(p, []byte("# comment\nhttp://a/x\thttp://b/x\nhttp://c/y\n\n"), 0o644); err != nil { + t.Fatal(err) + } + ts := readInputFile(p) + if len(ts) != 2 { + t.Fatalf("got %d targets, want 2: %v", len(ts), ts) + } + if want := []string{"http://a/x", "http://b/x"}; !reflect.DeepEqual(ts[0].uris, want) { + t.Errorf("line 1 mirrors = %v, want %v", ts[0].uris, want) + } + if want := []string{"http://c/y"}; !reflect.DeepEqual(ts[1].uris, want) { + t.Errorf("line 2 = %v, want %v", ts[1].uris, want) + } +} + +func TestReportExit(t *testing.T) { + canceled := download.Result{Name: "a", Err: context.Canceled} + done := download.Result{Name: "b", Err: nil} + timeout := download.Result{Name: "c", Err: errors.New("idle timeout")} + + tests := []struct { + name string + results []download.Result + want int + }{ + {"all ok", []download.Result{done}, 0}, + {"cancelled with a completion", []download.Result{done, canceled}, 7}, + {"cancelled only", []download.Result{canceled}, 1}, + {"real error wins over cancel", []download.Result{done, canceled, timeout}, 2}, + } + for _, tc := range tests { + if got := report(tc.results); got != tc.want { + t.Errorf("%s: report = %d, want %d", tc.name, got, tc.want) + } + } +} + +func TestSaveSessionAtomic(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "session.txt") + jobs := []job{ + {source: "http://a/done", dl: fakeDL{status: download.Complete}}, + {source: "magnet:c", dl: fakeDL{status: download.Active}}, + } + if err := saveSession(path, jobs); err != nil { + t.Fatalf("saveSession: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got, want := string(data), "magnet:c\n"; got != want { + t.Errorf("session = %q, want %q", got, want) + } + // No temp files should be left behind by an atomic write. + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Errorf("expected only the session file, got %d entries", len(entries)) + } +} diff --git a/progress/format.go b/progress/format.go new file mode 100644 index 0000000..862adba --- /dev/null +++ b/progress/format.go @@ -0,0 +1,79 @@ +package progress + +import ( + "fmt" + "time" +) + +// humanSize formats a byte count by abbreviating it: divide by +// 1024 while the value is at least one unit and a larger unit remains, then bump +// to the next unit when the quotient is >=922 (so 0.9Mi shows instead of 922Ki). +// The unit ladder is capped at Gi, so a terabyte prints +// as "1024.0GiB". When human is false it prints the raw integer with a B suffix. +func humanSize(n int64, human bool) string { + if !human { + return fmt.Sprintf("%dB", n) + } + const unit = 1024 + if n < unit { + return fmt.Sprintf("%dB", n) + } + // "KMG" caps the ladder at Gi (the units {"","Ki","Mi","Gi"}). + const suffixes = "KMG" + div, exp := int64(unit), 0 + for v := n / unit; v >= unit && exp+1 < len(suffixes); v /= unit { + div *= unit + exp++ + } + // Bump to the next unit when the quotient is >=922 and a unit remains, so + // 922*1024 bytes reads as 0.9MiB rather than 922KiB. + if q := n / div; q >= 922 && exp+1 < len(suffixes) { + div *= unit + exp++ + } + val := float64(n) / float64(div) + suffix := suffixes[exp] + // One decimal for small mantissas and for anything that overflowed the + // capped Gi unit (1024.0GiB and up); a bare integer otherwise (e.g. 20MiB). + if val < 10 || exp == len(suffixes)-1 && val >= unit { + return fmt.Sprintf("%.1f%ciB", val, suffix) + } + return fmt.Sprintf("%.0f%ciB", val, suffix) +} + +// speed formats a bytes-per-second rate. The DL:/UL: fields show the bare +// abbreviated size with no "/s" suffix. +func speed(bytesPerSec int64, human bool) string { + return humanSize(bytesPerSec, human) +} + +// secfmt renders a duration as "1h2m3s", appending each unit only when it is +// nonzero (but still showing seconds when the whole input is 0): +// 3600s->"1h", 120s->"2m", 3720s->"1h2m". A non-positive or absurd +// duration renders as "--". +func secfmt(d time.Duration) string { + s := int64(d.Seconds()) + if s < 0 || d > 99*time.Hour { + return "--" + } + h, m, sec := s/3600, (s%3600)/60, s%60 + var str string + if h > 0 { + str += fmt.Sprintf("%dh", h) + } + if m > 0 { + str += fmt.Sprintf("%dm", m) + } + if sec > 0 || s == 0 { + str += fmt.Sprintf("%ds", sec) + } + return str +} + +// percent returns completed/total as a 0-100 integer, or 0 when total is unknown. +func percent(completed, total int64) int { + if total <= 0 { + return 0 + } + return int(completed * 100 / total) +} diff --git a/progress/format_test.go b/progress/format_test.go new file mode 100644 index 0000000..6eed4e2 --- /dev/null +++ b/progress/format_test.go @@ -0,0 +1,73 @@ +package progress + +import ( + "testing" + "time" +) + +func TestHumanSize(t *testing.T) { + tests := []struct { + n int64 + human bool + want string + }{ + {500, true, "500B"}, + {1536, true, "1.5KiB"}, + {1 << 20, true, "1.0MiB"}, + {5 * 1 << 20, true, "5.0MiB"}, + {20 * 1 << 20, true, "20MiB"}, + {1 << 30, true, "1.0GiB"}, + {2048, false, "2048B"}, + // abbrevSize bumps to the next unit once the quotient reaches 922, so + // 922*1024 bytes reads as 0.9MiB rather than 922KiB. + {922 * 1024, true, "0.9MiB"}, + {921 * 1024, true, "921KiB"}, + // The unit ladder is capped at Gi, so a terabyte overflows GiB. + {1 << 40, true, "1024.0GiB"}, + } + for _, tc := range tests { + if got := humanSize(tc.n, tc.human); got != tc.want { + t.Errorf("humanSize(%d, %v) = %q, want %q", tc.n, tc.human, got, tc.want) + } + } +} + +func TestSecfmt(t *testing.T) { + tests := []struct { + d time.Duration + want string + }{ + {0, "0s"}, + {30 * time.Second, "30s"}, + {90 * time.Second, "1m30s"}, + {120 * time.Second, "2m"}, + {3600 * time.Second, "1h"}, + {3661 * time.Second, "1h1m1s"}, + {3720 * time.Second, "1h2m"}, + {-1 * time.Second, "--"}, + } + for _, tc := range tests { + if got := secfmt(tc.d); got != tc.want { + t.Errorf("secfmt(%v) = %q, want %q", tc.d, got, tc.want) + } + } +} + +func TestSpeedNoSuffix(t *testing.T) { + // The DL:/UL: fields show the bare abbreviated size with no "/s". + if got := speed(1536, true); got != "1.5KiB" { + t.Errorf("speed(1536,true) = %q, want %q", got, "1.5KiB") + } + if got := speed(2048, false); got != "2048B" { + t.Errorf("speed(2048,false) = %q, want %q", got, "2048B") + } +} + +func TestPercent(t *testing.T) { + if got := percent(50, 100); got != 50 { + t.Errorf("percent(50,100) = %d, want 50", got) + } + if got := percent(1, 0); got != 0 { + t.Errorf("percent(1,0) = %d, want 0", got) + } +} diff --git a/progress/progress.go b/progress/progress.go new file mode 100644 index 0000000..ee3e263 --- /dev/null +++ b/progress/progress.go @@ -0,0 +1,274 @@ +// Package progress renders a live, single-line status display. One goroutine ticks +// once a second, pulls a snapshot of the running downloads, and derives speeds +// from the change since the previous tick. There are no locks here: the data +// arrives by value through the snapshot function. +package progress + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "time" + "unicode/utf8" + + "github.com/hanbok/got/download" + "golang.org/x/term" +) + +// Reporter draws the progress line until its context is cancelled. +type Reporter struct { + snap func() []download.Stat + human bool + interval time.Duration + w io.Writer + tty bool + width int + + win map[string]*speedWindow // keyed by Stat.ID + lastLog time.Time // last non-TTY line emitted +} + +// logEvery bounds how often progress is printed when output is not a TTY, so a +// redirected or piped run does not get a line every second. +const logEvery = 30 * time.Second + +// windowTime is the span of the sliding speed window (10s): the rate is the +// byte delta across this window divided by its real duration, which smooths the +// per-tick jitter. +const windowTime = 10 * time.Second + +// sample is one (time, cumulative-counters) observation in a speedWindow. +type sample struct { + t time.Time + completed, uploaded int64 +} + +// speedWindow keeps the last ~windowTime of samples for one download so we can +// derive down/up speed from the change across the window rather than the last +// tick alone. Samples are appended in time order, so stale ones live at the +// front. +type speedWindow struct { + samples []sample +} + +// New returns a Reporter that reads live stats from snap. +func New(snap func() []download.Stat, human bool) *Reporter { + return &Reporter{ + snap: snap, + human: human, + interval: time.Second, + w: os.Stdout, + tty: term.IsTerminal(int(os.Stdout.Fd())), + width: 80, + win: map[string]*speedWindow{}, + } +} + +// Run draws on every tick and clears the line when ctx is cancelled. +func (r *Reporter) Run(ctx context.Context) { + t := time.NewTicker(r.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + r.render(true) + return + case <-t.C: + r.render(false) + } + } +} + +func (r *Reporter) render(final bool) { + stats := r.snap() + now := time.Now() + + // Record one sample per download before deriving rates, and forget windows + // for downloads that have dropped out of the snapshot. + seen := make(map[string]bool, len(stats)) + for _, s := range stats { + seen[s.ID] = true + w := r.win[s.ID] + if w == nil { + w = &speedWindow{} + r.win[s.ID] = w + } + w.add(now, s.Completed, s.Uploaded) + } + for id := range r.win { + if !seen[id] { + delete(r.win, id) + } + } + + var line string + switch len(stats) { + case 0: + if !final && r.tty { + return // nothing active; leave the line as-is + } + case 1: + line = r.lineOne(stats[0]) + default: + line = r.lineMany(stats) + } + + if r.tty { + r.width = termWidth(r.width) + line = clampRunes(line, r.width) + fmt.Fprint(r.w, "\r"+line+"\x1b[K") + if final { + fmt.Fprintln(r.w) + } + return + } + // Not a TTY: can't redraw one line, so print sparingly (and always the + // final line) to keep redirected output and logs readable. + if line != "" && (final || now.Sub(r.lastLog) >= logEvery) { + fmt.Fprintln(r.w, line) + r.lastLog = now + } +} + +func (r *Reporter) lineOne(s download.Stat) string { + dl, ul := r.rates(s) + seeding := s.Status == download.Seeding + var b strings.Builder + fmt.Fprintf(&b, "[%s ", shortName(s.Name)) + // While seeding, report the share ratio in place of the size block and drop + // the DL: field; otherwise show completed/total (or bare completed). + if seeding { + fmt.Fprintf(&b, "SEED(%.1f)", ratio(s.Uploaded, s.Completed)) + } else if s.Total > 0 { + fmt.Fprintf(&b, "%s/%s(%d%%)", humanSize(s.Completed, r.human), humanSize(s.Total, r.human), percent(s.Completed, s.Total)) + } else { + b.WriteString(humanSize(s.Completed, r.human)) + } + // CN is always shown (including HTTP); SD is shown for any torrent. + fmt.Fprintf(&b, " CN:%d", s.Conns) + if s.IsBT { + fmt.Fprintf(&b, " SD:%d", s.Seeders) + } + if !seeding { + fmt.Fprintf(&b, " DL:%s", speed(dl, r.human)) + } + if seeding || ul > 0 { + fmt.Fprintf(&b, " UL:%s", speed(ul, r.human)) + } + if !seeding && s.Total > 0 && dl > 0 { + eta := time.Duration(float64(s.Total-s.Completed)/float64(dl)) * time.Second + fmt.Fprintf(&b, " ETA:%s", secfmt(eta)) + } + b.WriteByte(']') + return b.String() +} + +func (r *Reporter) lineMany(stats []download.Stat) string { + var totDL, totUL int64 + for _, s := range stats { + dl, ul := r.rates(s) + totDL += dl + totUL += ul + } + var b strings.Builder + fmt.Fprintf(&b, "[%d active DL:%s UL:%s]", len(stats), speed(totDL, r.human), speed(totUL, r.human)) + for i, s := range stats { + if i >= 4 { + fmt.Fprintf(&b, "[+%d]", len(stats)-i) + break + } + if s.Total > 0 { + fmt.Fprintf(&b, "[%s %d%%]", shortName(s.Name), percent(s.Completed, s.Total)) + } else { + fmt.Fprintf(&b, "[%s %s]", shortName(s.Name), humanSize(s.Completed, r.human)) + } + } + return b.String() +} + +// add records a sample and drops any that have fallen out of the window. A +// sample is only appended once per second, but the latest counters are folded +// into the newest slot so the final cancel-render, which typically fires in the +// same second as the preceding tick, still sees current totals. +func (w *speedWindow) add(now time.Time, completed, uploaded int64) { + // Drop samples older than the window, keeping the first one that still falls + // inside it as the baseline for the delta. + cut := now.Add(-windowTime) + i := 0 + for i < len(w.samples) && w.samples[i].t.Before(cut) { + i++ + } + w.samples = w.samples[i:] + + if n := len(w.samples); n > 0 && now.Sub(w.samples[n-1].t) < time.Second { + w.samples[n-1].completed = completed + w.samples[n-1].uploaded = uploaded + return + } + w.samples = append(w.samples, sample{now, completed, uploaded}) +} + +// rates derives down/up speed in bytes/s from the change across the speed +// window (windowBytes / windowSeconds). With a single sample we have no span +// yet, so we report 0 rather than mistaking the cumulative bytes (e.g. resumed +// data) for one window's worth. +func (r *Reporter) rates(s download.Stat) (dl, ul int64) { + w := r.win[s.ID] + if w == nil || len(w.samples) < 2 { + return 0, 0 + } + first, last := w.samples[0], w.samples[len(w.samples)-1] + secs := last.t.Sub(first.t).Seconds() + if secs <= 0 { + return 0, 0 + } + dl = int64(float64(last.completed-first.completed) / secs) + ul = int64(float64(last.uploaded-first.uploaded) / secs) + if dl < 0 { + dl = 0 + } + if ul < 0 { + ul = 0 + } + return dl, ul +} + +// ratio is uploaded/completed for the seeding readout; 0 when nothing has been +// downloaded yet to avoid dividing by zero. +func ratio(uploaded, completed int64) float64 { + if completed <= 0 { + return 0 + } + return float64(uploaded) / float64(completed) +} + +func shortName(s string) string { + const max = 20 + // Truncate on runes, not bytes, so a multibyte name (CJK, emoji) is never + // cut mid-rune. + if utf8.RuneCountInString(s) > max { + return string([]rune(s)[:max-1]) + "~" + } + return s +} + +// clampRunes limits a line to width runes (not bytes) so multibyte glyphs are +// never split when the terminal is narrow. A non-positive width leaves it whole. +func clampRunes(s string, width int) string { + if width <= 0 || utf8.RuneCountInString(s) <= width { + return s + } + return string([]rune(s)[:width]) +} + +func termWidth(prev int) int { + if w, _, err := term.GetSize(int(os.Stdout.Fd())); err == nil && w > 0 { + return w + } + if prev > 0 { + return prev + } + return 80 +} diff --git a/progress/progress_test.go b/progress/progress_test.go new file mode 100644 index 0000000..820e81a --- /dev/null +++ b/progress/progress_test.go @@ -0,0 +1,135 @@ +package progress + +import ( + "strings" + "testing" + "time" + + "github.com/hanbok/got/download" +) + +// reporterAt builds a Reporter wired to a fixed stat snapshot for rendering tests. +func newReporter(human bool) *Reporter { + return &Reporter{human: human, width: 200, win: map[string]*speedWindow{}} +} + +// TestSlidingWindowSpeed feeds samples across a ~10s window and checks the rate +// is the windowed delta over its real duration, not the last-tick delta. +func TestSlidingWindowSpeed(t *testing.T) { + r := newReporter(false) + base := time.Unix(1000, 0) + // Five one-second ticks, +1000 bytes each: window holds 4s of deltas. + w := &speedWindow{} + r.win["x"] = w + for i := 0; i < 5; i++ { + w.add(base.Add(time.Duration(i)*time.Second), int64(i*1000), 0) + } + s := download.Stat{ID: "x", Completed: 4000} + dl, ul := r.rates(s) + if dl != 1000 { + t.Errorf("dl = %d, want 1000", dl) + } + if ul != 0 { + t.Errorf("ul = %d, want 0", ul) + } +} + +// TestWindowDropsStale ensures samples older than windowTime are evicted so the +// baseline of the delta stays inside the window. +func TestWindowDropsStale(t *testing.T) { + w := &speedWindow{} + base := time.Unix(0, 0) + // One old sample, then a fresh one 20s later. + w.add(base, 0, 0) + w.add(base.Add(20*time.Second), 5000, 0) + if len(w.samples) != 1 { + t.Fatalf("len(samples) = %d, want 1 (stale dropped)", len(w.samples)) + } +} + +// TestSingleSampleZero: with only one sample there is no span, so speed is 0. +func TestSingleSampleZero(t *testing.T) { + r := newReporter(false) + r.win["x"] = &speedWindow{} + r.win["x"].add(time.Unix(0, 0), 1<<20, 0) + if dl, _ := r.rates(download.Stat{ID: "x"}); dl != 0 { + t.Errorf("dl = %d, want 0 for single sample", dl) + } +} + +// TestRatesKeyedByID confirms two downloads sharing a Name don't cross-subtract +// because the window is keyed on Stat.ID. +func TestRatesKeyedByID(t *testing.T) { + r := newReporter(false) + base := time.Unix(0, 0) + for _, id := range []string{"a", "b"} { + r.win[id] = &speedWindow{} + r.win[id].add(base, 0, 0) + } + r.win["a"].add(base.Add(time.Second), 1000, 0) + r.win["b"].add(base.Add(time.Second), 2000, 0) + if dl, _ := r.rates(download.Stat{ID: "a", Name: "dup"}); dl != 1000 { + t.Errorf("a dl = %d, want 1000", dl) + } + if dl, _ := r.rates(download.Stat{ID: "b", Name: "dup"}); dl != 2000 { + t.Errorf("b dl = %d, want 2000", dl) + } +} + +// TestLineOneSeeding: while seeding, render SEED(ratio), drop DL:, keep UL:. +func TestLineOneSeeding(t *testing.T) { + r := newReporter(true) + s := download.Stat{ + ID: "h", Name: "f", IsBT: true, Status: download.Seeding, + Total: 1000, Completed: 1000, Uploaded: 1500, Conns: 3, Seeders: 2, + } + line := r.lineOne(s) + if !strings.Contains(line, "SEED(1.5)") { + t.Errorf("missing SEED(1.5): %q", line) + } + if strings.Contains(line, "DL:") { + t.Errorf("DL: should be dropped while seeding: %q", line) + } + if !strings.Contains(line, "UL:") { + t.Errorf("UL: should be kept while seeding: %q", line) + } +} + +// TestLineOneCNSD: CN always shown; SD shown for torrents (even with 0 seeders), +// absent for HTTP. +func TestLineOneCNSD(t *testing.T) { + r := newReporter(true) + bt := r.lineOne(download.Stat{ID: "h", Name: "f", IsBT: true, Total: 10, Completed: 1, Conns: 4, Seeders: 0}) + if !strings.Contains(bt, "CN:4") || !strings.Contains(bt, "SD:0") { + t.Errorf("torrent line missing CN/SD: %q", bt) + } + http := r.lineOne(download.Stat{ID: "h2", Name: "f", IsBT: false, Total: 10, Completed: 1, Conns: 1}) + if !strings.Contains(http, "CN:1") { + t.Errorf("http line missing CN: %q", http) + } + if strings.Contains(http, "SD:") { + t.Errorf("http line should not show SD: %q", http) + } +} + +// TestShortNameRune ensures CJK names are truncated on runes, not bytes. +func TestShortNameRune(t *testing.T) { + name := strings.Repeat("あ", 30) // 30 runes, 90 bytes + got := shortName(name) + runes := []rune(got) + if len(runes) != 20 { + t.Errorf("shortName rune count = %d, want 20", len(runes)) + } + if runes[len(runes)-1] != '~' { + t.Errorf("expected trailing ~, got %q", got) + } +} + +// TestClampRunes clamps on runes so a multibyte glyph is never split. +func TestClampRunes(t *testing.T) { + s := strings.Repeat("あ", 10) // 10 runes + got := clampRunes(s, 5) + if r := []rune(got); len(r) != 5 { + t.Errorf("clampRunes rune count = %d, want 5", len(r)) + } +}