Comments (and one test comment) now describe the rules on their own terms instead of by comparison to another downloader. No behavior change.
858 lines
28 KiB
Go
858 lines
28 KiB
Go
// Command got is a Rob-Pike-style command-line download tool: a small
|
|
// multi-protocol downloader. It downloads HTTP(S) URLs over several connections
|
|
// and BitTorrent magnets/torrents, with a comprehensive set of command-line
|
|
// options. Each download runs as one goroutine; a scheduler bounds how many run
|
|
// at once; a single goroutine draws the progress line.
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/anacrolix/torrent"
|
|
"github.com/anacrolix/torrent/metainfo"
|
|
"github.com/hanbok/got/bt"
|
|
"github.com/hanbok/got/cli"
|
|
"github.com/hanbok/got/download"
|
|
"github.com/hanbok/got/httpdl"
|
|
"github.com/hanbok/got/progress"
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
func main() {
|
|
os.Exit(run(os.Args[1:]))
|
|
}
|
|
|
|
func run(args []string) int {
|
|
res, err := cli.Parse(args)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
return 1
|
|
}
|
|
switch res.Action {
|
|
case cli.ShowVersion:
|
|
cli.PrintVersion(os.Stdout)
|
|
return 0
|
|
case cli.ShowHelp:
|
|
cli.PrintHelp(os.Stdout, res.HelpTag)
|
|
return 0
|
|
}
|
|
|
|
opts := res.Options
|
|
|
|
// --checksum is validated up front so a bad type or digest aborts before any
|
|
// download, rejecting it at option-parse time rather than failing mid-run
|
|
// (and once, not once per HTTP download that shares the spec).
|
|
if cs := opts.Str("checksum"); cs != "" {
|
|
if err := httpdl.ValidChecksum(cs); err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
return 1
|
|
}
|
|
}
|
|
|
|
targets := gatherURIs(opts, res.URIs)
|
|
if len(targets) == 0 {
|
|
fmt.Fprintln(os.Stderr, "got: no URI, torrent or magnet given (try --help)")
|
|
return 1
|
|
}
|
|
|
|
if opts.Bool("show-files") {
|
|
return showFiles(allURIs(targets))
|
|
}
|
|
|
|
jobs, btClient, err := build(opts, targets)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
return 1
|
|
}
|
|
if btClient != nil {
|
|
defer btClient.Close()
|
|
}
|
|
|
|
// A single registered signal channel handles both presses: the first cancels
|
|
// the graceful context, the second saves the session best-effort and forces
|
|
// exit. Using one channel avoids a startup race where two separate handlers
|
|
// could split the two presses and need a third to actually quit.
|
|
sessionFile := opts.Str("save-session")
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
// --stop: cancel everything after N seconds, exactly like a Ctrl-C, so an
|
|
// unattended run can't hang forever (0 = off).
|
|
if stop := opts.Int("stop"); stop > 0 {
|
|
t := time.AfterFunc(time.Duration(stop)*time.Second, cancel)
|
|
defer t.Stop()
|
|
}
|
|
var jobsRef atomic.Pointer[[]job]
|
|
// publish snapshots the slice header (j is a by-value copy) and stores a
|
|
// pointer to it, so the signal handler can read the current jobs lock-free
|
|
// without racing the `jobs = append(...)` reassignment in the follow pass.
|
|
publish := func(j []job) { jobsRef.Store(&j) }
|
|
go handleSignals(cancel, sessionFile, &jobsRef)
|
|
|
|
eng := download.NewEngine(opts.Int("max-concurrent-downloads"))
|
|
|
|
uiCtx, uiCancel := context.WithCancel(context.Background())
|
|
uiDone := make(chan struct{})
|
|
if !opts.Bool("quiet") && !opts.Bool("dry-run") {
|
|
rep := progress.New(eng.Snapshot, opts.Bool("human-readable"))
|
|
go func() {
|
|
rep.Run(uiCtx)
|
|
close(uiDone)
|
|
}()
|
|
} else {
|
|
close(uiDone)
|
|
}
|
|
|
|
publish(jobs)
|
|
results := runJobs(ctx, eng, jobs)
|
|
|
|
// --follow-torrent: a .torrent fetched over HTTP becomes a BitTorrent
|
|
// download of its content (on by default). Done as a second pass so the
|
|
// engine and scheduler stay simple.
|
|
if opts.Bool("follow-torrent") && ctx.Err() == nil && !opts.Bool("dry-run") {
|
|
if follow := followUps(jobs, btClient, btOptions(opts)); len(follow) > 0 {
|
|
// The engine numbers each Result by its position in the slice it was
|
|
// passed, so the follow pass restarts at Index 0. Offset those by the
|
|
// initial job count to give the concatenated slice one contiguous
|
|
// Index space, so the sort below restores input order deterministically.
|
|
nInitial := len(jobs)
|
|
jobs = append(jobs, follow...)
|
|
publish(jobs)
|
|
followResults := runJobs(ctx, eng, follow)
|
|
for i := range followResults {
|
|
followResults[i].Index += nInitial
|
|
}
|
|
results = append(results, followResults...)
|
|
}
|
|
}
|
|
|
|
uiCancel()
|
|
<-uiDone // wait for the renderer's final frame before printing the OK/FAIL lines
|
|
|
|
saveCookies(jobs)
|
|
|
|
if sessionFile != "" {
|
|
if err := saveSession(sessionFile, jobs); err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
}
|
|
}
|
|
// Restore input order so the OK/FAIL lines and the exit code are
|
|
// deterministic regardless of which download finished first.
|
|
sort.Slice(results, func(i, j int) bool { return results[i].Index < results[j].Index })
|
|
if opts.Bool("dry-run") {
|
|
return reportDryRun(results)
|
|
}
|
|
return report(results)
|
|
}
|
|
|
|
// job pairs a download with the source string that produced it, so we can write
|
|
// the still-unfinished ones back to a session file.
|
|
type job struct {
|
|
source string
|
|
dl download.Download
|
|
}
|
|
|
|
// build turns each URI into a Download, classified by scheme, and creates the
|
|
// one shared BitTorrent client the run needs (nil if no torrents are involved).
|
|
// The overall speed limits are one shared limiter each, applied to both HTTP
|
|
// downloads and the BitTorrent client so the cap is truly global.
|
|
// errDuplicate marks a source that names the same torrent as an earlier one.
|
|
// exitCode maps it to the duplicate-infohash exit code (12) via errors.Is.
|
|
var errDuplicate = errors.New("duplicate download")
|
|
|
|
// failedDownload is a Download that does no work and fails immediately with a
|
|
// fixed error. build() uses it for a duplicate torrent source, so the duplicate
|
|
// is reported as a failed download without ever obtaining — or Dropping — a
|
|
// torrent on the shared client.
|
|
type failedDownload struct {
|
|
name string
|
|
err error
|
|
}
|
|
|
|
func (f *failedDownload) Name() string { return f.name }
|
|
func (f *failedDownload) Run(context.Context) error { return f.err }
|
|
func (f *failedDownload) Stat() download.Stat {
|
|
return download.Stat{Name: f.name, Status: download.Errored, Total: -1}
|
|
}
|
|
|
|
// dedupeTorrents finds torrent/magnet sources that name the same torrent as an
|
|
// earlier source (same infohash), returning each duplicate URI mapped to the
|
|
// earlier source it repeats. HTTP(S) URLs and any source whose infohash can't be
|
|
// read without a network fetch are never duplicates here. The shared client keys
|
|
// torrents by infohash and does not refcount, so build() turns each duplicate
|
|
// into a failed download (duplicate infohash) rather than letting two jobs
|
|
// share — and Drop — one torrent.
|
|
func dedupeTorrents(uris []string, follow bool) map[string]string {
|
|
seen := map[metainfo.Hash]string{}
|
|
dup := map[string]string{}
|
|
for _, u := range uris {
|
|
k := uriKind(u, follow)
|
|
if k != kindMagnet && k != kindTorrent {
|
|
continue
|
|
}
|
|
h, ok := bt.SourceInfoHash(u, k == kindTorrent)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if first, isDup := seen[h]; isDup {
|
|
dup[u] = first
|
|
} else {
|
|
seen[h] = u
|
|
}
|
|
}
|
|
return dup
|
|
}
|
|
|
|
// markInfoHash records the v1 infohash of the .torrent at path in seen and
|
|
// reports whether it repeats one already seen, returning the path first recorded
|
|
// under that infohash. Callers fail the duplicate rather than let two jobs share
|
|
// one torrent on the client (see bt.SourceInfoHash for why that aliasing is
|
|
// unsafe). A v2-only or unreadable infohash is never a duplicate — the dedup
|
|
// can't see it, so it runs.
|
|
func markInfoHash(seen map[metainfo.Hash]string, path string) (first string, dup bool) {
|
|
h, ok := bt.SourceInfoHash(path, true)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
if first, dup = seen[h]; dup {
|
|
return first, true
|
|
}
|
|
seen[h] = path
|
|
return "", false
|
|
}
|
|
|
|
func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) {
|
|
flat := allURIs(targets)
|
|
// Two sources naming the same torrent (a magnet and its own .torrent, say)
|
|
// would share one torrent in the client, which does not refcount: the first to
|
|
// finish would Drop it out from under the other. The duplicate is reported as
|
|
// a failed download (duplicate infohash, exit 12); the loop below does this,
|
|
// so the duplicate never obtains or Drops a shared torrent.
|
|
dups := dedupeTorrents(flat, opts.Bool("follow-torrent"))
|
|
|
|
overallDL := makeLimiter(opts.Size("max-overall-download-limit"))
|
|
overallUL := makeLimiter(opts.Size("max-overall-upload-limit"))
|
|
// --out names a single output file, so it only applies when exactly one HTTP
|
|
// download (one mirror group) is queued; warn rather than silently misname.
|
|
nhttp := countHTTP(targets)
|
|
single := nhttp == 1
|
|
if nhttp > 1 && opts.Str("out") != "" {
|
|
fmt.Fprintln(os.Stderr, "got: --out ignored: it applies only to a single HTTP download")
|
|
}
|
|
hc := httpConfig(opts, single, overallDL)
|
|
bo := btOptions(opts)
|
|
|
|
// One client serves every torrent, shared across the session. A failure to start
|
|
// it (e.g. the listen port is taken) disables BitTorrent but still lets HTTP
|
|
// downloads run; those torrents then report the error.
|
|
var client *torrent.Client
|
|
if needsBT(opts, flat) {
|
|
c, err := bt.NewClient(btClientConfig(opts, overallDL, overallUL))
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "got: bittorrent disabled:", err)
|
|
} else {
|
|
client = c
|
|
}
|
|
}
|
|
|
|
var out []job
|
|
for _, t := range targets {
|
|
u := t.uris[0] // primary: classifies the target and names the session source
|
|
src := strings.Join(t.uris, "\t")
|
|
if first, isDup := dups[u]; isDup {
|
|
out = append(out, job{source: src, dl: &failedDownload{
|
|
name: u,
|
|
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
|
}})
|
|
continue
|
|
}
|
|
var dl download.Download
|
|
switch uriKind(u, opts.Bool("follow-torrent")) {
|
|
case kindMagnet:
|
|
dl = bt.New(client, u, false, bo)
|
|
case kindHTTP, kindFollow:
|
|
dl = httpdl.New(t.uris, hc) // every mirror URL for this one file
|
|
case kindTorrent:
|
|
dl = bt.New(client, u, true, bo)
|
|
default:
|
|
if client != nil {
|
|
client.Close()
|
|
}
|
|
return nil, nil, fmt.Errorf("unsupported URI: %s", u)
|
|
}
|
|
out = append(out, job{source: src, dl: dl})
|
|
}
|
|
return out, client, nil
|
|
}
|
|
|
|
// uriKind classifies a URI by scheme into the taxonomy build() and needsBT()
|
|
// both switch on. Scheme wins over suffix: an http URL ending in .torrent is a
|
|
// file to fetch over HTTP, and with --follow-torrent it becomes a torrent only
|
|
// after the fetch (kindFollow), so it still downloads over HTTP first.
|
|
func uriKind(u string, follow bool) kind {
|
|
lu := strings.ToLower(u)
|
|
switch {
|
|
case strings.HasPrefix(u, "magnet:"):
|
|
return kindMagnet
|
|
case strings.HasPrefix(u, "http://"), strings.HasPrefix(u, "https://"):
|
|
if follow && strings.HasSuffix(lu, ".torrent") {
|
|
return kindFollow
|
|
}
|
|
return kindHTTP
|
|
case strings.HasSuffix(lu, ".torrent"):
|
|
return kindTorrent
|
|
}
|
|
return kindUnsupported
|
|
}
|
|
|
|
type kind int
|
|
|
|
const (
|
|
kindUnsupported kind = iota
|
|
kindMagnet // magnet: link, a BitTorrent download
|
|
kindHTTP // plain HTTP(S) download
|
|
kindTorrent // local .torrent file, a BitTorrent download
|
|
kindFollow // HTTP(S) .torrent fetched, then followed as BitTorrent
|
|
)
|
|
|
|
// needsBT reports whether the run will involve any torrent, so we only start a
|
|
// client when one is actually needed.
|
|
func needsBT(opts *cli.Options, uris []string) bool {
|
|
follow := opts.Bool("follow-torrent")
|
|
for _, u := range uris {
|
|
switch uriKind(u, follow) {
|
|
case kindMagnet, kindTorrent, kindFollow:
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// countHTTP counts HTTP download targets. A mirror group is one output file, so
|
|
// it counts groups, not URLs, for the --out single-file decision.
|
|
func countHTTP(targets []target) int {
|
|
n := 0
|
|
for _, t := range targets {
|
|
u := t.uris[0]
|
|
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// runJobs runs a set of jobs through the engine and returns their results.
|
|
func runJobs(ctx context.Context, eng *download.Engine, jobs []job) []download.Result {
|
|
dls := make([]download.Download, len(jobs))
|
|
for i := range jobs {
|
|
dls[i] = jobs[i].dl
|
|
}
|
|
return eng.Run(ctx, dls)
|
|
}
|
|
|
|
// followUps turns each completed HTTP .torrent download into a BitTorrent
|
|
// download of its content. A file is followed only if it parses as a torrent,
|
|
// so a regular download that merely ends in ".torrent" is left alone.
|
|
func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
|
var out []job
|
|
seen := map[metainfo.Hash]string{}
|
|
for _, j := range jobs {
|
|
h, ok := j.dl.(*httpdl.Download)
|
|
if !ok || j.dl.Stat().Status != download.Complete {
|
|
continue
|
|
}
|
|
// j.source is the TAB-joined mirror group; the primary (uris[0]) is what
|
|
// build() classified and named the download on, so test its suffix.
|
|
primary, _, _ := strings.Cut(j.source, "\t")
|
|
if !strings.HasSuffix(strings.ToLower(primary), ".torrent") {
|
|
continue
|
|
}
|
|
path := h.Path()
|
|
if _, err := bt.Files(path); err != nil {
|
|
continue // downloaded file is not a valid torrent
|
|
}
|
|
// Two fetched .torrent files that name the same torrent would share one
|
|
// torrent on the refcount-less client: the first to finish Drops it out
|
|
// from under the other, stranding it at 0%. build() dedupes pass-1 sources
|
|
// but cannot reach a not-yet-fetched HTTP .torrent, so dedupe here too.
|
|
if first, dup := markInfoHash(seen, path); dup {
|
|
out = append(out, job{source: path, dl: &failedDownload{
|
|
name: path,
|
|
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
|
}})
|
|
continue
|
|
}
|
|
out = append(out, job{source: path, dl: bt.New(client, path, true, bo)})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// saveCookies asks each HTTP download to write its cookie jar back to the
|
|
// --save-cookies file, persisting cookies at session end. A download with no
|
|
// --save-cookies configured is a no-op.
|
|
func saveCookies(jobs []job) {
|
|
for _, j := range jobs {
|
|
if h, ok := j.dl.(*httpdl.Download); ok {
|
|
if err := h.SaveCookies(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// target is one download: a single torrent/magnet/URL, or several HTTP mirror
|
|
// URLs that serve identical bytes (uris[0] is the primary).
|
|
type target struct{ uris []string }
|
|
|
|
// allURIs flattens targets to a plain URL list for the per-URL helpers
|
|
// (dedupe, needsBT, show-files) that don't care about grouping.
|
|
func allURIs(ts []target) []string {
|
|
var u []string
|
|
for _, t := range ts {
|
|
u = append(u, t.uris...)
|
|
}
|
|
return u
|
|
}
|
|
|
|
// gatherURIs collects download targets from the command line, --torrent-file,
|
|
// and --input-file. Plain command-line http/https URIs are grouped into ONE
|
|
// mirrored download by default (every command-line URI names the same file);
|
|
// -Z/--force-sequential makes each its own download. Torrents, magnets, and
|
|
// .torrent URLs (fetched over HTTP then followed) are always one target each,
|
|
// since distinct torrents are not mirrors of one another; --input-file groups
|
|
// TAB-separated URIs per line.
|
|
func gatherURIs(opts *cli.Options, cmdURIs []string) []target {
|
|
var ts []target
|
|
follow := opts.Bool("follow-torrent")
|
|
seq := opts.Bool("force-sequential")
|
|
var httpGroup []string
|
|
for _, u := range cmdURIs {
|
|
// Only plain HTTP URLs collapse into one mirror download (alternate sources
|
|
// for the same file); -Z opts out. A .torrent URL (kindFollow), like a
|
|
// torrent or magnet, is its own download — distinct torrents are never
|
|
// mirrors of one another.
|
|
if uriKind(u, follow) == kindHTTP && !seq {
|
|
httpGroup = append(httpGroup, u)
|
|
} else {
|
|
ts = append(ts, target{uris: []string{u}})
|
|
}
|
|
}
|
|
if len(httpGroup) > 0 {
|
|
ts = append(ts, target{uris: httpGroup})
|
|
}
|
|
if tf := opts.Str("torrent-file"); tf != "" {
|
|
ts = append(ts, target{uris: []string{tf}})
|
|
}
|
|
if in := opts.Str("input-file"); in != "" {
|
|
ts = append(ts, readInputFile(in)...)
|
|
}
|
|
return ts
|
|
}
|
|
|
|
// readInputFile parses an input file: one download per line, and TAB-separated
|
|
// URIs on a line are mirrors of that one download. Blank lines and '#' comments
|
|
// are skipped.
|
|
func readInputFile(path string) []target {
|
|
f := os.Stdin
|
|
if path != "-" {
|
|
var err error
|
|
if f, err = os.Open(path); err != nil {
|
|
// A missing input file is fine: it just means "nothing to resume
|
|
// yet", which is what the load+save-session idiom needs on first run.
|
|
if !os.IsNotExist(err) {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
}
|
|
return nil
|
|
}
|
|
defer f.Close()
|
|
}
|
|
var ts []target
|
|
sc := bufio.NewScanner(f)
|
|
// Magnet links with many trackers (and long signed URLs) can exceed the
|
|
// 64KB default token size, so raise the cap.
|
|
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
|
for sc.Scan() {
|
|
line := sc.Text()
|
|
if t := strings.TrimSpace(line); t == "" || strings.HasPrefix(t, "#") {
|
|
continue
|
|
}
|
|
var g []string
|
|
for _, field := range strings.Split(line, "\t") {
|
|
if field = strings.TrimSpace(field); field != "" {
|
|
g = append(g, field)
|
|
}
|
|
}
|
|
if len(g) > 0 {
|
|
ts = append(ts, target{uris: g})
|
|
}
|
|
}
|
|
if err := sc.Err(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
}
|
|
return ts
|
|
}
|
|
|
|
func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.Config {
|
|
out := ""
|
|
if single {
|
|
out = opts.Str("out")
|
|
}
|
|
// max-connection-per-server caps connections per host. Its default is 1,
|
|
// which on its own would clamp --split to a single connection, so a bare
|
|
// `got URL` would download single-threaded. Honor the literal default of 1
|
|
// only when the user actually set -x; otherwise let --split drive per-host
|
|
// parallelism, capped at the -x ceiling of 16. -x explicitly set keeps the
|
|
// exact min(split, mirrors*x) behavior.
|
|
perHost := opts.Int("max-connection-per-server")
|
|
if !opts.IsSet("max-connection-per-server") {
|
|
perHost = min(opts.Int("split"), 16)
|
|
}
|
|
return httpdl.Config{
|
|
Dir: opts.Str("dir"),
|
|
Out: out,
|
|
Split: opts.Int("split"),
|
|
MaxConnPerServer: perHost,
|
|
MinSplit: opts.Size("min-split-size"),
|
|
Tries: opts.Int("max-tries"),
|
|
Timeout: time.Duration(opts.Int("timeout")) * time.Second,
|
|
FileAlloc: opts.Str("file-allocation"),
|
|
Continue: opts.Bool("continue"),
|
|
AllowOverwrite: opts.Bool("allow-overwrite"),
|
|
AutoRename: opts.Bool("auto-file-renaming"),
|
|
Headers: opts.List("header"),
|
|
UserAgent: opts.Str("user-agent"),
|
|
Referer: opts.Str("referer"),
|
|
Proxy: opts.Str("all-proxy"),
|
|
CheckCert: opts.Bool("check-certificate"),
|
|
Limit: opts.Size("max-download-limit"),
|
|
OverallLimiter: overallDL,
|
|
RetryWait: time.Duration(opts.Int("retry-wait")) * time.Second,
|
|
AutoSaveInterval: time.Duration(opts.Int("auto-save-interval")) * time.Second,
|
|
ConnectTimeout: time.Duration(opts.Int("connect-timeout")) * time.Second,
|
|
LoadCookies: opts.Str("load-cookies"),
|
|
SaveCookies: opts.Str("save-cookies"),
|
|
ConditionalGet: opts.Bool("conditional-get"),
|
|
RemoteTime: opts.Bool("remote-time"),
|
|
HTTPUser: opts.Str("http-user"),
|
|
HTTPPasswd: opts.Str("http-passwd"),
|
|
LowestSpeedLimit: opts.Size("lowest-speed-limit"),
|
|
Checksum: opts.Str("checksum"),
|
|
DryRun: opts.Bool("dry-run"),
|
|
CACert: opts.Str("ca-certificate"),
|
|
DisableIPv6: opts.Bool("disable-ipv6"),
|
|
}
|
|
}
|
|
|
|
// btClientConfig is the run-wide configuration for the one shared torrent client.
|
|
func btClientConfig(opts *cli.Options, overallDL, overallUL *rate.Limiter) bt.ClientConfig {
|
|
return bt.ClientConfig{
|
|
Dir: opts.Str("dir"),
|
|
ListenPortSpec: opts.Str("listen-port"),
|
|
DHT: opts.Bool("enable-dht"),
|
|
MaxPeers: opts.Int("bt-max-peers"),
|
|
OverallDown: overallDL,
|
|
OverallUp: overallUL,
|
|
DownLimit: opts.Size("max-download-limit"),
|
|
UpLimit: opts.Size("max-upload-limit"),
|
|
UserAgent: opts.Str("user-agent"),
|
|
DisableIPv6: opts.Bool("disable-ipv6"),
|
|
}
|
|
}
|
|
|
|
// btOptions is the per-torrent configuration.
|
|
func btOptions(opts *cli.Options) bt.Options {
|
|
return bt.Options{
|
|
SeedTimeSet: opts.IsSet("seed-time"),
|
|
SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)),
|
|
SeedRatio: opts.Float("seed-ratio"),
|
|
StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second,
|
|
SelectFiles: parseSelect(opts.Str("select-file")),
|
|
CheckIntegrity: opts.Bool("check-integrity"),
|
|
DryRun: opts.Bool("dry-run"),
|
|
}
|
|
}
|
|
|
|
// showFiles prints the file list of each torrent argument and returns an exit
|
|
// code. Magnets need their metadata fetched first, which we don't do here.
|
|
func showFiles(uris []string) int {
|
|
printed := false
|
|
for _, u := range uris {
|
|
if !strings.HasSuffix(strings.ToLower(u), ".torrent") {
|
|
if strings.HasPrefix(u, "magnet:") {
|
|
fmt.Printf("%s: --show-files needs a .torrent (magnet metadata is fetched at download time)\n", u)
|
|
}
|
|
continue
|
|
}
|
|
files, err := bt.Files(u)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
return 1
|
|
}
|
|
fmt.Printf("%s:\n", u)
|
|
for _, f := range files {
|
|
fmt.Printf(" %d %s (%d bytes)\n", f.Index, f.Path, f.Length)
|
|
}
|
|
printed = true
|
|
}
|
|
if !printed {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// reportDryRun prints what --dry-run learned about each source — that it is
|
|
// reachable and how big it is — none of it having been downloaded, and returns
|
|
// the exit code: 0 if every source was reachable, else the dominant error's
|
|
// code (so a 404 under --dry-run still exits 3, a refused host 6, etc.).
|
|
func reportDryRun(results []download.Result) int {
|
|
var lastReal error
|
|
for _, r := range results {
|
|
switch {
|
|
case r.Err == nil:
|
|
if r.Stat.Total >= 0 {
|
|
fmt.Printf("DRY %s (%d bytes)\n", r.Name, r.Stat.Total)
|
|
} else {
|
|
fmt.Printf("DRY %s (size unknown)\n", r.Name)
|
|
}
|
|
case errors.Is(r.Err, context.Canceled):
|
|
fmt.Printf("STOP %s\n", r.Name)
|
|
default:
|
|
fmt.Printf("FAIL %s: %v\n", r.Name, r.Err)
|
|
lastReal = r.Err
|
|
}
|
|
}
|
|
if lastReal != nil {
|
|
return exitCode(lastReal)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// report prints a one-line outcome per download and returns the process exit
|
|
// code, mapped to the documented exit-status table:
|
|
// 0 all OK, 7 cancelled with at least one completed, 1 cancelled with nothing
|
|
// completed, else the code of the dominant (last) real error, mapped to an
|
|
// exit code by exitCode.
|
|
func report(results []download.Result) int {
|
|
completed, cancelled := 0, 0
|
|
var lastReal error
|
|
for _, r := range results {
|
|
switch {
|
|
case r.Err == nil:
|
|
fmt.Printf("OK %s\n", r.Name)
|
|
completed++
|
|
case errors.Is(r.Err, context.Canceled):
|
|
fmt.Printf("STOP %s (interrupted%s)\n", r.Name, progressNote(r.Stat))
|
|
cancelled++
|
|
default:
|
|
fmt.Printf("FAIL %s: %v\n", r.Name, r.Err)
|
|
lastReal = r.Err
|
|
}
|
|
}
|
|
|
|
// Exit code 7 is returned when the only failures were interruptions and at
|
|
// least one download finished: unfinished downloads remained in the queue when
|
|
// the process exited by pressing Ctrl-C. With no real error and nothing
|
|
// complete, fall back to 1.
|
|
if lastReal == nil {
|
|
switch {
|
|
case cancelled > 0 && completed > 0:
|
|
return 7
|
|
case cancelled > 0:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
return exitCode(lastReal)
|
|
}
|
|
|
|
// exitCode maps a real (non-cancellation) download error to the documented
|
|
// exit-status codes. The retry path wraps the underlying cause with %w, so we can inspect the
|
|
// chain: an existing output file (13) and a 404 (3) are classified by their
|
|
// httpdl sentinel type; name resolution (19), timeout (2), and a connection-level
|
|
// network problem (6) are classified by typed errors with a substring fallback
|
|
// for third-party net text; everything else is the generic unknown error (1).
|
|
// Telling 19 from 6 from 2 apart is the point — it says whether the user's own
|
|
// network, the remote host, or a stall is to blame.
|
|
func exitCode(err error) int {
|
|
switch {
|
|
case errors.Is(err, httpdl.ErrOutputExists):
|
|
return 13
|
|
case isNameResolution(err):
|
|
return 19
|
|
case isTimeout(err):
|
|
return 2
|
|
case isNetwork(err):
|
|
return 6
|
|
case errors.Is(err, httpdl.ErrNotFound):
|
|
return 3
|
|
case errors.Is(err, httpdl.ErrTooSlow):
|
|
return 5
|
|
case errors.Is(err, httpdl.ErrChecksum):
|
|
return 32
|
|
case errors.Is(err, errDuplicate):
|
|
return 12
|
|
}
|
|
return 1
|
|
}
|
|
|
|
// isNameResolution reports whether err is (or wraps) a DNS lookup failure —
|
|
// "name resolution failed" (19), which usually points at the user's own
|
|
// network or resolver rather than a dead remote. The stdlib reports every such
|
|
// failure (no-such-host, server-misbehaving) as *net.DNSError.
|
|
func isNameResolution(err error) bool {
|
|
var de *net.DNSError
|
|
return errors.As(err, &de)
|
|
}
|
|
|
|
// isNetwork reports whether err is a connection-level failure — refused, reset,
|
|
// or an unreachable host/network — a "network problem" (6). A dial error from
|
|
// net/http wraps the syscall errno, so the typed check sees through it.
|
|
func isNetwork(err error) bool {
|
|
return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) ||
|
|
errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTUNREACH)
|
|
}
|
|
|
|
// isTimeout reports whether err is (or wraps) a timeout: a deadline exceeded, a
|
|
// net.Error that timed out, or our own idle-read timeout (httpdl.ErrTimeout).
|
|
func isTimeout(err error) bool {
|
|
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, httpdl.ErrTimeout) {
|
|
return true
|
|
}
|
|
var ne net.Error
|
|
return errors.As(err, &ne) && ne.Timeout()
|
|
}
|
|
|
|
// saveSession writes the sources of every download that did not finish to path,
|
|
// one per line, so a later `got -i path` resumes them (HTTP via -c, BitTorrent
|
|
// via anacrolix's recheck of existing data). An all-done run leaves the file
|
|
// empty.
|
|
func saveSession(path string, jobs []job) error {
|
|
lines := unfinishedSources(jobs)
|
|
data := ""
|
|
if len(lines) > 0 {
|
|
data = strings.Join(lines, "\n") + "\n"
|
|
}
|
|
// Write to a temp file in the same directory, then rename over the target so
|
|
// a crash or a force-quit mid-write can never leave a half-written (and thus
|
|
// truncated) resume list. The rename is atomic within one filesystem.
|
|
dir := filepath.Dir(path)
|
|
tmp, err := os.CreateTemp(dir, ".got-session-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmpName := tmp.Name()
|
|
if _, err := tmp.WriteString(data); err != nil {
|
|
tmp.Close()
|
|
os.Remove(tmpName)
|
|
return err
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
os.Remove(tmpName)
|
|
return err
|
|
}
|
|
if err := os.Rename(tmpName, path); err != nil {
|
|
os.Remove(tmpName)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// unfinishedSources returns the source string of each download not in the
|
|
// Complete state.
|
|
func unfinishedSources(jobs []job) []string {
|
|
var out []string
|
|
for _, j := range jobs {
|
|
if j.dl.Stat().Status != download.Complete {
|
|
out = append(out, j.source)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// progressNote describes how far an interrupted download got, so the user knows
|
|
// resuming is worthwhile.
|
|
func progressNote(s download.Stat) string {
|
|
if s.Total > 0 && s.Completed > 0 {
|
|
return fmt.Sprintf("; %d%% done", int(s.Completed*100/s.Total))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// handleSignals owns the one registered signal channel. The first Ctrl-C (or
|
|
// TERM) cancels the graceful context so run() can wind down and save normally;
|
|
// a second forces exit, but first writes the session best-effort so the resume
|
|
// list survives a force-quit.
|
|
func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *atomic.Pointer[[]job]) {
|
|
sig := make(chan os.Signal, 2)
|
|
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
|
<-sig // first: cancel the graceful context
|
|
cancel()
|
|
<-sig // second: force exit
|
|
if sessionFile != "" {
|
|
var js []job // nil until the first publish; saveSession handles an empty list
|
|
if p := jobs.Load(); p != nil {
|
|
js = *p
|
|
}
|
|
if err := saveSession(sessionFile, js); err != nil {
|
|
fmt.Fprintln(os.Stderr, "got:", err)
|
|
}
|
|
}
|
|
fmt.Fprintln(os.Stderr, "\ngot: forced exit")
|
|
os.Exit(130)
|
|
}
|
|
|
|
func makeLimiter(bps int64) *rate.Limiter {
|
|
if bps <= 0 {
|
|
return nil
|
|
}
|
|
return rate.NewLimiter(rate.Limit(bps), download.LimiterBurst(bps))
|
|
}
|
|
|
|
// maxSelectIndex bounds how far a --select-file range is expanded: a torrent has
|
|
// at most a few thousand files, so a degenerate spec like "1-9999999999" must not
|
|
// spin or balloon the map. Indexes past the real file count are ignored anyway.
|
|
const maxSelectIndex = 1 << 20
|
|
|
|
// parseSelect parses "1,3-5" into a set of 1-based file indexes.
|
|
func parseSelect(s string) map[int]bool {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return nil
|
|
}
|
|
out := map[int]bool{}
|
|
for _, part := range strings.Split(s, ",") {
|
|
lo, hi, ok := strings.Cut(part, "-")
|
|
a, _ := strconv.Atoi(strings.TrimSpace(lo))
|
|
if !ok {
|
|
if a > 0 {
|
|
out[a] = true
|
|
}
|
|
continue
|
|
}
|
|
b, _ := strconv.Atoi(strings.TrimSpace(hi))
|
|
if a < 1 {
|
|
a = 1
|
|
}
|
|
if b > maxSelectIndex {
|
|
b = maxSelectIndex
|
|
}
|
|
for i := a; i <= b; i++ {
|
|
out[i] = true
|
|
}
|
|
}
|
|
return out
|
|
}
|