first commit
This commit is contained in:
826
main.go
Normal file
826
main.go
Normal file
@@ -0,0 +1,826 @@
|
||||
// Command got is a Rob-Pike-style command-line download tool: a small
|
||||
// multi-protocol downloader. It downloads HTTP(S) URLs over several connections
|
||||
// and BitTorrent magnets/torrents, with a comprehensive set of command-line
|
||||
// options. Each download runs as one goroutine; a scheduler bounds how many run
|
||||
// at once; a single goroutine draws the progress line.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/anacrolix/torrent"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/hanbok/got/bt"
|
||||
"github.com/hanbok/got/cli"
|
||||
"github.com/hanbok/got/download"
|
||||
"github.com/hanbok/got/httpdl"
|
||||
"github.com/hanbok/got/progress"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:]))
|
||||
}
|
||||
|
||||
func run(args []string) int {
|
||||
res, err := cli.Parse(args)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
return 1
|
||||
}
|
||||
switch res.Action {
|
||||
case cli.ShowVersion:
|
||||
cli.PrintVersion(os.Stdout)
|
||||
return 0
|
||||
case cli.ShowHelp:
|
||||
cli.PrintHelp(os.Stdout, res.HelpTag)
|
||||
return 0
|
||||
}
|
||||
|
||||
opts := res.Options
|
||||
|
||||
// --checksum is validated up front so a bad type or digest aborts before any
|
||||
// download, rejecting it at option-parse time rather than failing mid-run
|
||||
// (and once, not once per HTTP download that shares the spec).
|
||||
if cs := opts.Str("checksum"); cs != "" {
|
||||
if err := httpdl.ValidChecksum(cs); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
targets := gatherURIs(opts, res.URIs)
|
||||
if len(targets) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "got: no URI, torrent or magnet given (try --help)")
|
||||
return 1
|
||||
}
|
||||
|
||||
if opts.Bool("show-files") {
|
||||
return showFiles(allURIs(targets))
|
||||
}
|
||||
|
||||
jobs, btClient, err := build(opts, targets)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
return 1
|
||||
}
|
||||
if btClient != nil {
|
||||
defer btClient.Close()
|
||||
}
|
||||
|
||||
// A single registered signal channel handles both presses: the first cancels
|
||||
// the graceful context, the second saves the session best-effort and forces
|
||||
// exit. Using one channel avoids a startup race where two separate handlers
|
||||
// could split the two presses and need a third to actually quit.
|
||||
sessionFile := opts.Str("save-session")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// --stop: cancel everything after N seconds, exactly like a Ctrl-C, so an
|
||||
// unattended run can't hang forever (0 = off).
|
||||
if stop := opts.Int("stop"); stop > 0 {
|
||||
t := time.AfterFunc(time.Duration(stop)*time.Second, cancel)
|
||||
defer t.Stop()
|
||||
}
|
||||
jobsRef := &jobsHolder{}
|
||||
go handleSignals(cancel, sessionFile, jobsRef)
|
||||
|
||||
eng := download.NewEngine(opts.Int("max-concurrent-downloads"))
|
||||
|
||||
uiCtx, uiCancel := context.WithCancel(context.Background())
|
||||
if !opts.Bool("quiet") && !opts.Bool("dry-run") {
|
||||
rep := progress.New(eng.Snapshot, opts.Bool("human-readable"))
|
||||
go rep.Run(uiCtx)
|
||||
}
|
||||
|
||||
jobsRef.set(jobs)
|
||||
results := runJobs(ctx, eng, jobs)
|
||||
|
||||
// --follow-torrent: a .torrent fetched over HTTP becomes a BitTorrent
|
||||
// download of its content (on by default). Done as a second pass so the
|
||||
// engine and scheduler stay simple.
|
||||
if opts.Bool("follow-torrent") && ctx.Err() == nil && !opts.Bool("dry-run") {
|
||||
if follow := followUps(jobs, btClient, btOptions(opts)); len(follow) > 0 {
|
||||
// The engine numbers each Result by its position in the slice it was
|
||||
// passed, so the follow pass restarts at Index 0. Offset those by the
|
||||
// initial job count to give the concatenated slice one contiguous
|
||||
// Index space, so the sort below restores input order deterministically.
|
||||
nInitial := len(jobs)
|
||||
jobs = append(jobs, follow...)
|
||||
jobsRef.set(jobs)
|
||||
followResults := runJobs(ctx, eng, follow)
|
||||
for i := range followResults {
|
||||
followResults[i].Index += nInitial
|
||||
}
|
||||
results = append(results, followResults...)
|
||||
}
|
||||
}
|
||||
|
||||
uiCancel()
|
||||
time.Sleep(20 * time.Millisecond) // let the renderer clear its line
|
||||
|
||||
saveCookies(jobs)
|
||||
|
||||
if sessionFile != "" {
|
||||
if err := saveSession(sessionFile, jobs); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
}
|
||||
}
|
||||
// Restore input order so the OK/FAIL lines and the exit code are
|
||||
// deterministic regardless of which download finished first.
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].Index < results[j].Index })
|
||||
if opts.Bool("dry-run") {
|
||||
return reportDryRun(results)
|
||||
}
|
||||
return report(results)
|
||||
}
|
||||
|
||||
// jobsHolder lets the signal handler reach the latest job list for a
|
||||
// best-effort session save on a forced (second-Ctrl-C) exit.
|
||||
type jobsHolder struct {
|
||||
mu sync.Mutex
|
||||
jobs []job
|
||||
}
|
||||
|
||||
func (h *jobsHolder) set(jobs []job) {
|
||||
h.mu.Lock()
|
||||
h.jobs = jobs
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *jobsHolder) get() []job {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.jobs
|
||||
}
|
||||
|
||||
// job pairs a download with the source string that produced it, so we can write
|
||||
// the still-unfinished ones back to a session file.
|
||||
type job struct {
|
||||
source string
|
||||
dl download.Download
|
||||
}
|
||||
|
||||
// build turns each URI into a Download, classified by scheme, and creates the
|
||||
// one shared BitTorrent client the run needs (nil if no torrents are involved).
|
||||
// The overall speed limits are one shared limiter each, applied to both HTTP
|
||||
// downloads and the BitTorrent client so the cap is truly global.
|
||||
// errDuplicate marks a source that names the same torrent as an earlier one.
|
||||
// exitCode maps it to the duplicate-infohash exit code (12) via errors.Is.
|
||||
var errDuplicate = errors.New("duplicate download")
|
||||
|
||||
// failedDownload is a Download that does no work and fails immediately with a
|
||||
// fixed error. build() uses it for a duplicate torrent source, so the duplicate
|
||||
// is reported as a failed download without ever obtaining — or Dropping — a
|
||||
// torrent on the shared client.
|
||||
type failedDownload struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f failedDownload) Name() string { return f.name }
|
||||
func (f failedDownload) Run(context.Context) error { return f.err }
|
||||
func (f failedDownload) Stat() download.Stat {
|
||||
return download.Stat{Name: f.name, Status: download.Errored, Total: -1}
|
||||
}
|
||||
|
||||
// dedupeTorrents finds torrent/magnet sources that name the same torrent as an
|
||||
// earlier source (same infohash), returning each duplicate URI mapped to the
|
||||
// earlier source it repeats. HTTP(S) URLs and any source whose infohash can't be
|
||||
// read without a network fetch are never duplicates here. The shared client keys
|
||||
// torrents by infohash and does not refcount, so build() turns each duplicate
|
||||
// into a failed download (duplicate infohash) rather than letting two jobs
|
||||
// share — and Drop — one torrent.
|
||||
func dedupeTorrents(uris []string, follow bool) map[string]string {
|
||||
seen := map[metainfo.Hash]string{}
|
||||
dup := map[string]string{}
|
||||
for _, u := range uris {
|
||||
k := uriKind(u, follow)
|
||||
if k != kindMagnet && k != kindTorrent {
|
||||
continue
|
||||
}
|
||||
h, ok := bt.SourceInfoHash(u, k == kindTorrent)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if first, isDup := seen[h]; isDup {
|
||||
dup[u] = first
|
||||
} else {
|
||||
seen[h] = u
|
||||
}
|
||||
}
|
||||
return dup
|
||||
}
|
||||
|
||||
func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) {
|
||||
flat := allURIs(targets)
|
||||
// Two sources naming the same torrent (a magnet and its own .torrent, say)
|
||||
// would share one torrent in the client, which does not refcount: the first to
|
||||
// finish would Drop it out from under the other. The duplicate is reported as
|
||||
// a failed download (duplicate infohash, exit 12); the loop below does this,
|
||||
// so the duplicate never obtains or Drops a shared torrent.
|
||||
dups := dedupeTorrents(flat, opts.Bool("follow-torrent"))
|
||||
|
||||
overallDL := makeLimiter(opts.Size("max-overall-download-limit"))
|
||||
overallUL := makeLimiter(opts.Size("max-overall-upload-limit"))
|
||||
// --out names a single output file, so it only applies when exactly one HTTP
|
||||
// download (one mirror group) is queued; warn rather than silently misname.
|
||||
nhttp := countHTTP(targets)
|
||||
single := nhttp == 1
|
||||
if nhttp > 1 && opts.Str("out") != "" {
|
||||
fmt.Fprintln(os.Stderr, "got: --out ignored: it applies only to a single HTTP download")
|
||||
}
|
||||
hc := httpConfig(opts, single, overallDL)
|
||||
bo := btOptions(opts)
|
||||
|
||||
// One client serves every torrent, shared across the session. A failure to start
|
||||
// it (e.g. the listen port is taken) disables BitTorrent but still lets HTTP
|
||||
// downloads run; those torrents then report the error.
|
||||
var client *torrent.Client
|
||||
if needsBT(opts, flat) {
|
||||
c, err := bt.NewClient(btClientConfig(opts, overallDL, overallUL))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got: bittorrent disabled:", err)
|
||||
} else {
|
||||
client = c
|
||||
}
|
||||
}
|
||||
|
||||
var out []job
|
||||
for _, t := range targets {
|
||||
u := t.uris[0] // primary: classifies the target and names the session source
|
||||
src := strings.Join(t.uris, "\t")
|
||||
if first, isDup := dups[u]; isDup {
|
||||
out = append(out, job{source: src, dl: failedDownload{
|
||||
name: u,
|
||||
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
||||
}})
|
||||
continue
|
||||
}
|
||||
var dl download.Download
|
||||
switch uriKind(u, opts.Bool("follow-torrent")) {
|
||||
case kindMagnet:
|
||||
dl = bt.New(client, u, false, bo)
|
||||
case kindHTTP, kindFollow:
|
||||
dl = httpdl.New(t.uris, hc) // every mirror URL for this one file
|
||||
case kindTorrent:
|
||||
dl = bt.New(client, u, true, bo)
|
||||
default:
|
||||
if client != nil {
|
||||
client.Close()
|
||||
}
|
||||
return nil, nil, fmt.Errorf("unsupported URI: %s", u)
|
||||
}
|
||||
out = append(out, job{source: src, dl: dl})
|
||||
}
|
||||
return out, client, nil
|
||||
}
|
||||
|
||||
// uriKind classifies a URI by scheme into the taxonomy build() and needsBT()
|
||||
// both switch on. Scheme wins over suffix: an http URL ending in .torrent is a
|
||||
// file to fetch over HTTP, and with --follow-torrent it becomes a torrent only
|
||||
// after the fetch (kindFollow), so it still downloads over HTTP first.
|
||||
func uriKind(u string, follow bool) kind {
|
||||
lu := strings.ToLower(u)
|
||||
switch {
|
||||
case strings.HasPrefix(u, "magnet:"):
|
||||
return kindMagnet
|
||||
case strings.HasPrefix(u, "http://"), strings.HasPrefix(u, "https://"):
|
||||
if follow && strings.HasSuffix(lu, ".torrent") {
|
||||
return kindFollow
|
||||
}
|
||||
return kindHTTP
|
||||
case strings.HasSuffix(lu, ".torrent"):
|
||||
return kindTorrent
|
||||
}
|
||||
return kindUnsupported
|
||||
}
|
||||
|
||||
type kind int
|
||||
|
||||
const (
|
||||
kindUnsupported kind = iota
|
||||
kindMagnet // magnet: link, a BitTorrent download
|
||||
kindHTTP // plain HTTP(S) download
|
||||
kindTorrent // local .torrent file, a BitTorrent download
|
||||
kindFollow // HTTP(S) .torrent fetched, then followed as BitTorrent
|
||||
)
|
||||
|
||||
// needsBT reports whether the run will involve any torrent, so we only start a
|
||||
// client when one is actually needed.
|
||||
func needsBT(opts *cli.Options, uris []string) bool {
|
||||
follow := opts.Bool("follow-torrent")
|
||||
for _, u := range uris {
|
||||
switch uriKind(u, follow) {
|
||||
case kindMagnet, kindTorrent, kindFollow:
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// countHTTP counts HTTP download targets. A mirror group is one output file, so
|
||||
// it counts groups, not URLs, for the --out single-file decision.
|
||||
func countHTTP(targets []target) int {
|
||||
n := 0
|
||||
for _, t := range targets {
|
||||
u := t.uris[0]
|
||||
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// runJobs runs a set of jobs through the engine and returns their results.
|
||||
func runJobs(ctx context.Context, eng *download.Engine, jobs []job) []download.Result {
|
||||
dls := make([]download.Download, len(jobs))
|
||||
for i := range jobs {
|
||||
dls[i] = jobs[i].dl
|
||||
}
|
||||
return eng.Run(ctx, dls)
|
||||
}
|
||||
|
||||
// followUps turns each completed HTTP .torrent download into a BitTorrent
|
||||
// download of its content. A file is followed only if it parses as a torrent,
|
||||
// so a regular download that merely ends in ".torrent" is left alone.
|
||||
func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
||||
var out []job
|
||||
for _, j := range jobs {
|
||||
h, ok := j.dl.(*httpdl.Download)
|
||||
if !ok || j.dl.Stat().Status != download.Complete {
|
||||
continue
|
||||
}
|
||||
// j.source is the TAB-joined mirror group; the primary (uris[0]) is what
|
||||
// build() classified and named the download on, so test its suffix.
|
||||
primary, _, _ := strings.Cut(j.source, "\t")
|
||||
if !strings.HasSuffix(strings.ToLower(primary), ".torrent") {
|
||||
continue
|
||||
}
|
||||
path := h.Path()
|
||||
if _, err := bt.Files(path); err != nil {
|
||||
continue // downloaded file is not a valid torrent
|
||||
}
|
||||
out = append(out, job{source: path, dl: bt.New(client, path, true, bo)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// saveCookies asks each HTTP download to write its cookie jar back to the
|
||||
// --save-cookies file, persisting cookies at session end. A download with no
|
||||
// --save-cookies configured is a no-op.
|
||||
func saveCookies(jobs []job) {
|
||||
for _, j := range jobs {
|
||||
if h, ok := j.dl.(*httpdl.Download); ok {
|
||||
if err := h.SaveCookies(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// target is one download: a single torrent/magnet/URL, or several HTTP mirror
|
||||
// URLs that serve identical bytes (uris[0] is the primary).
|
||||
type target struct{ uris []string }
|
||||
|
||||
// allURIs flattens targets to a plain URL list for the per-URL helpers
|
||||
// (dedupe, needsBT, show-files) that don't care about grouping.
|
||||
func allURIs(ts []target) []string {
|
||||
var u []string
|
||||
for _, t := range ts {
|
||||
u = append(u, t.uris...)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// gatherURIs collects download targets from the command line, --torrent-file,
|
||||
// and --input-file. Plain command-line http/https URIs are grouped into ONE
|
||||
// mirrored download by default (every command-line URI names the same file);
|
||||
// -Z/--force-sequential makes each its own download. Torrents and magnets
|
||||
// are always one target each; --input-file groups TAB-separated URIs per line.
|
||||
func gatherURIs(opts *cli.Options, cmdURIs []string) []target {
|
||||
var ts []target
|
||||
follow := opts.Bool("follow-torrent")
|
||||
seq := opts.Bool("force-sequential")
|
||||
var httpGroup []string
|
||||
for _, u := range cmdURIs {
|
||||
switch uriKind(u, follow) {
|
||||
case kindHTTP, kindFollow:
|
||||
if seq {
|
||||
ts = append(ts, target{uris: []string{u}})
|
||||
} else {
|
||||
httpGroup = append(httpGroup, u)
|
||||
}
|
||||
default:
|
||||
ts = append(ts, target{uris: []string{u}})
|
||||
}
|
||||
}
|
||||
if len(httpGroup) > 0 {
|
||||
ts = append(ts, target{uris: httpGroup})
|
||||
}
|
||||
if tf := opts.Str("torrent-file"); tf != "" {
|
||||
ts = append(ts, target{uris: []string{tf}})
|
||||
}
|
||||
if in := opts.Str("input-file"); in != "" {
|
||||
ts = append(ts, readInputFile(in)...)
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
// readInputFile parses an input file: one download per line, and TAB-separated
|
||||
// URIs on a line are mirrors of that one download. Blank lines and '#' comments
|
||||
// are skipped.
|
||||
func readInputFile(path string) []target {
|
||||
f := os.Stdin
|
||||
if path != "-" {
|
||||
var err error
|
||||
if f, err = os.Open(path); err != nil {
|
||||
// A missing input file is fine: it just means "nothing to resume
|
||||
// yet", which is what the load+save-session idiom needs on first run.
|
||||
if !os.IsNotExist(err) {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
}
|
||||
var ts []target
|
||||
sc := bufio.NewScanner(f)
|
||||
// Magnet links with many trackers (and long signed URLs) can exceed the
|
||||
// 64KB default token size, so raise the cap.
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if t := strings.TrimSpace(line); t == "" || strings.HasPrefix(t, "#") {
|
||||
continue
|
||||
}
|
||||
var g []string
|
||||
for _, field := range strings.Split(line, "\t") {
|
||||
if field = strings.TrimSpace(field); field != "" {
|
||||
g = append(g, field)
|
||||
}
|
||||
}
|
||||
if len(g) > 0 {
|
||||
ts = append(ts, target{uris: g})
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.Config {
|
||||
out := ""
|
||||
if single {
|
||||
out = opts.Str("out")
|
||||
}
|
||||
return httpdl.Config{
|
||||
Dir: opts.Str("dir"),
|
||||
Out: out,
|
||||
Split: opts.Int("split"),
|
||||
MaxConnPerServer: opts.Int("max-connection-per-server"),
|
||||
MinSplit: opts.Size("min-split-size"),
|
||||
Tries: opts.Int("max-tries"),
|
||||
Timeout: time.Duration(opts.Int("timeout")) * time.Second,
|
||||
FileAlloc: opts.Str("file-allocation"),
|
||||
Continue: opts.Bool("continue"),
|
||||
AllowOverwrite: opts.Bool("allow-overwrite"),
|
||||
AutoRename: opts.Bool("auto-file-renaming"),
|
||||
Headers: opts.List("header"),
|
||||
UserAgent: opts.Str("user-agent"),
|
||||
Referer: opts.Str("referer"),
|
||||
Proxy: opts.Str("all-proxy"),
|
||||
CheckCert: opts.Bool("check-certificate"),
|
||||
Limit: opts.Size("max-download-limit"),
|
||||
OverallLimiter: overallDL,
|
||||
RetryWait: time.Duration(opts.Int("retry-wait")) * time.Second,
|
||||
AutoSaveInterval: time.Duration(opts.Int("auto-save-interval")) * time.Second,
|
||||
ConnectTimeout: time.Duration(opts.Int("connect-timeout")) * time.Second,
|
||||
LoadCookies: opts.Str("load-cookies"),
|
||||
SaveCookies: opts.Str("save-cookies"),
|
||||
ConditionalGet: opts.Bool("conditional-get"),
|
||||
RemoteTime: opts.Bool("remote-time"),
|
||||
HTTPUser: opts.Str("http-user"),
|
||||
HTTPPasswd: opts.Str("http-passwd"),
|
||||
LowestSpeedLimit: opts.Size("lowest-speed-limit"),
|
||||
Checksum: opts.Str("checksum"),
|
||||
DryRun: opts.Bool("dry-run"),
|
||||
CACert: opts.Str("ca-certificate"),
|
||||
DisableIPv6: opts.Bool("disable-ipv6"),
|
||||
}
|
||||
}
|
||||
|
||||
// btClientConfig is the run-wide configuration for the one shared torrent client.
|
||||
func btClientConfig(opts *cli.Options, overallDL, overallUL *rate.Limiter) bt.ClientConfig {
|
||||
return bt.ClientConfig{
|
||||
Dir: opts.Str("dir"),
|
||||
ListenPortSpec: opts.Str("listen-port"),
|
||||
DHT: opts.Bool("enable-dht"),
|
||||
MaxPeers: opts.Int("bt-max-peers"),
|
||||
OverallDown: overallDL,
|
||||
OverallUp: overallUL,
|
||||
DownLimit: opts.Size("max-download-limit"),
|
||||
UpLimit: opts.Size("max-upload-limit"),
|
||||
UserAgent: opts.Str("user-agent"),
|
||||
DisableIPv6: opts.Bool("disable-ipv6"),
|
||||
}
|
||||
}
|
||||
|
||||
// btOptions is the per-torrent configuration.
|
||||
func btOptions(opts *cli.Options) bt.Options {
|
||||
return bt.Options{
|
||||
SeedTimeSet: opts.IsSet("seed-time"),
|
||||
SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)),
|
||||
SeedRatio: opts.Float("seed-ratio"),
|
||||
StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second,
|
||||
MetaTimeout: time.Duration(opts.Int("bt-metadata-timeout")) * time.Second,
|
||||
SelectFiles: parseSelect(opts.Str("select-file")),
|
||||
CheckIntegrity: opts.Bool("check-integrity"),
|
||||
DryRun: opts.Bool("dry-run"),
|
||||
}
|
||||
}
|
||||
|
||||
// showFiles prints the file list of each torrent argument and returns an exit
|
||||
// code. Magnets need their metadata fetched first, which we don't do here.
|
||||
func showFiles(uris []string) int {
|
||||
printed := false
|
||||
for _, u := range uris {
|
||||
if !strings.HasSuffix(strings.ToLower(u), ".torrent") {
|
||||
if strings.HasPrefix(u, "magnet:") {
|
||||
fmt.Printf("%s: --show-files needs a .torrent (magnet metadata is fetched at download time)\n", u)
|
||||
}
|
||||
continue
|
||||
}
|
||||
files, err := bt.Files(u)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("%s:\n", u)
|
||||
for _, f := range files {
|
||||
fmt.Printf(" %d %s (%d bytes)\n", f.Index, f.Path, f.Length)
|
||||
}
|
||||
printed = true
|
||||
}
|
||||
if !printed {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// reportDryRun prints what --dry-run learned about each source — that it is
|
||||
// reachable and how big it is — none of it having been downloaded, and returns
|
||||
// the exit code: 0 if every source was reachable, else the dominant error's
|
||||
// code (so a 404 under --dry-run still exits 3, a refused host 6, etc.).
|
||||
func reportDryRun(results []download.Result) int {
|
||||
var lastReal error
|
||||
for _, r := range results {
|
||||
switch {
|
||||
case r.Err == nil:
|
||||
if r.Stat.Total >= 0 {
|
||||
fmt.Printf("DRY %s (%d bytes)\n", r.Name, r.Stat.Total)
|
||||
} else {
|
||||
fmt.Printf("DRY %s (size unknown)\n", r.Name)
|
||||
}
|
||||
case errors.Is(r.Err, context.Canceled):
|
||||
fmt.Printf("STOP %s\n", r.Name)
|
||||
default:
|
||||
fmt.Printf("FAIL %s: %v\n", r.Name, r.Err)
|
||||
lastReal = r.Err
|
||||
}
|
||||
}
|
||||
if lastReal != nil {
|
||||
return exitCode(lastReal)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// report prints a one-line outcome per download and returns the process exit
|
||||
// code, mapped to the documented exit-status table:
|
||||
// 0 all OK, 7 cancelled with at least one completed, 1 cancelled with nothing
|
||||
// completed, else the code of the dominant (last) real error, mapped to an
|
||||
// exit code by exitCode.
|
||||
func report(results []download.Result) int {
|
||||
completed, cancelled := 0, 0
|
||||
var lastReal error
|
||||
for _, r := range results {
|
||||
switch {
|
||||
case r.Err == nil:
|
||||
fmt.Printf("OK %s\n", r.Name)
|
||||
completed++
|
||||
case errors.Is(r.Err, context.Canceled):
|
||||
fmt.Printf("STOP %s (interrupted%s)\n", r.Name, progressNote(r.Stat))
|
||||
cancelled++
|
||||
default:
|
||||
fmt.Printf("FAIL %s: %v\n", r.Name, r.Err)
|
||||
lastReal = r.Err
|
||||
}
|
||||
}
|
||||
|
||||
// Exit code 7 is returned when the only failures were interruptions and at
|
||||
// least one download finished: unfinished downloads remained in the queue when
|
||||
// the process exited by pressing Ctrl-C. With no real error and nothing
|
||||
// complete, fall back to 1.
|
||||
if lastReal == nil {
|
||||
switch {
|
||||
case cancelled > 0 && completed > 0:
|
||||
return 7
|
||||
case cancelled > 0:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return exitCode(lastReal)
|
||||
}
|
||||
|
||||
// exitCode maps a real (non-cancellation) download error to the documented
|
||||
// exit-status codes. The retry path wraps the underlying cause with %w, so we can inspect the
|
||||
// chain: an existing output file (13) and a 404 (3) are classified by their
|
||||
// httpdl sentinel type; name resolution (19), timeout (2), and a connection-level
|
||||
// network problem (6) are classified by typed errors with a substring fallback
|
||||
// for third-party net text; everything else is the generic unknown error (1).
|
||||
// Telling 19 from 6 from 2 apart is the point — it says whether the user's own
|
||||
// network, the remote host, or a stall is to blame.
|
||||
func exitCode(err error) int {
|
||||
switch {
|
||||
case errors.Is(err, httpdl.ErrOutputExists):
|
||||
return 13
|
||||
case isNameResolution(err):
|
||||
return 19
|
||||
case isTimeout(err):
|
||||
return 2
|
||||
case isNetwork(err):
|
||||
return 6
|
||||
case errors.Is(err, httpdl.ErrNotFound):
|
||||
return 3
|
||||
case errors.Is(err, httpdl.ErrTooSlow):
|
||||
return 5
|
||||
case errors.Is(err, httpdl.ErrChecksum):
|
||||
return 32
|
||||
case errors.Is(err, errDuplicate):
|
||||
return 12
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// isNameResolution reports whether err is (or wraps) a DNS lookup failure —
|
||||
// "name resolution failed" (19), which usually points at the user's own
|
||||
// network or resolver rather than a dead remote.
|
||||
func isNameResolution(err error) bool {
|
||||
var de *net.DNSError
|
||||
if errors.As(err, &de) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "no such host") || strings.Contains(msg, "server misbehaving")
|
||||
}
|
||||
|
||||
// isNetwork reports whether err is a connection-level failure — refused, reset,
|
||||
// or an unreachable host/network — a "network problem" (6).
|
||||
func isNetwork(err error) bool {
|
||||
if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) ||
|
||||
errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTUNREACH) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "connection refused") ||
|
||||
strings.Contains(msg, "connection reset") ||
|
||||
strings.Contains(msg, "network is unreachable") ||
|
||||
strings.Contains(msg, "no route to host")
|
||||
}
|
||||
|
||||
// isTimeout reports whether err is (or wraps) a timeout: a deadline exceeded, a
|
||||
// net.Error that timed out, or an idle-read timeout reported as such.
|
||||
func isTimeout(err error) bool {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "timeout") || strings.Contains(msg, "timed out")
|
||||
}
|
||||
|
||||
// saveSession writes the sources of every download that did not finish to path,
|
||||
// one per line, so a later `got -i path` resumes them (HTTP via -c, BitTorrent
|
||||
// via anacrolix's recheck of existing data). An all-done run leaves the file
|
||||
// empty.
|
||||
func saveSession(path string, jobs []job) error {
|
||||
lines := unfinishedSources(jobs)
|
||||
data := ""
|
||||
if len(lines) > 0 {
|
||||
data = strings.Join(lines, "\n") + "\n"
|
||||
}
|
||||
// Write to a temp file in the same directory, then rename over the target so
|
||||
// a crash or a force-quit mid-write can never leave a half-written (and thus
|
||||
// truncated) resume list. The rename is atomic within one filesystem.
|
||||
dir := filepath.Dir(path)
|
||||
tmp, err := os.CreateTemp(dir, ".got-session-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
if _, err := tmp.WriteString(data); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unfinishedSources returns the source string of each download not in the
|
||||
// Complete state.
|
||||
func unfinishedSources(jobs []job) []string {
|
||||
var out []string
|
||||
for _, j := range jobs {
|
||||
if j.dl.Stat().Status != download.Complete {
|
||||
out = append(out, j.source)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// progressNote describes how far an interrupted download got, so the user knows
|
||||
// resuming is worthwhile.
|
||||
func progressNote(s download.Stat) string {
|
||||
if s.Total > 0 && s.Completed > 0 {
|
||||
return fmt.Sprintf("; %d%% done", int(s.Completed*100/s.Total))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// handleSignals owns the one registered signal channel. The first Ctrl-C (or
|
||||
// TERM) cancels the graceful context so run() can wind down and save normally;
|
||||
// a second forces exit, but first writes the session best-effort so the resume
|
||||
// list survives a force-quit.
|
||||
func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *jobsHolder) {
|
||||
sig := make(chan os.Signal, 2)
|
||||
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
||||
<-sig // first: cancel the graceful context
|
||||
cancel()
|
||||
<-sig // second: force exit
|
||||
if sessionFile != "" {
|
||||
if err := saveSession(sessionFile, jobs.get()); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "got:", err)
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "\ngot: forced exit")
|
||||
os.Exit(130)
|
||||
}
|
||||
|
||||
func makeLimiter(bps int64) *rate.Limiter {
|
||||
if bps <= 0 {
|
||||
return nil
|
||||
}
|
||||
return rate.NewLimiter(rate.Limit(bps), download.LimiterBurst(bps))
|
||||
}
|
||||
|
||||
// parseSelect parses "1,3-5" into a set of 1-based file indexes.
|
||||
func parseSelect(s string) map[int]bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
out := map[int]bool{}
|
||||
for _, part := range strings.Split(s, ",") {
|
||||
lo, hi, ok := strings.Cut(part, "-")
|
||||
a, _ := strconv.Atoi(strings.TrimSpace(lo))
|
||||
if !ok {
|
||||
if a > 0 {
|
||||
out[a] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
b, _ := strconv.Atoi(strings.TrimSpace(hi))
|
||||
for i := a; i <= b; i++ {
|
||||
if i > 0 {
|
||||
out[i] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user